#!/usr/bin/python
+import datetime
import logging
import math
import socket
+import time
+import tornado.httpclient
+import urlparse
from . import base
from . import logs
+log = logging.getLogger("mirrors")
+log.propagate = 1
+
from .decorators import lazy_property
class Mirrors(base.Object):
- def get_all(self):
- mirrors = []
+ def __iter__(self):
+ res = self.db.query("SELECT * FROM mirrors \
+ WHERE deleted IS FALSE ORDER BY hostname")
- for mirror in self.db.query("SELECT id FROM mirrors \
- WHERE NOT status = 'deleted' ORDER BY hostname"):
- mirror = Mirror(self.pakfire, mirror.id)
+ mirrors = []
+ for row in res:
+ mirror = Mirror(self.backend, row.id, data=row)
mirrors.append(mirror)
- return mirrors
+ return iter(mirrors)
- def count(self, status=None):
- query = "SELECT COUNT(*) AS count FROM mirrors"
- args = []
+ def _get_mirror(self, query, *args):
+ res = self.db.get(query, *args)
+
+ if res:
+ return Mirror(self.backend, res.id, data=res)
- if status:
- query += " WHERE status = %s"
- args.append(status)
+ def create(self, hostname, path="", owner=None, contact=None, user=None):
+ mirror = self._get_mirror("INSERT INTO mirrors(hostname, path, owner, contact) \
+ VALUES(%s, %s, %s, %s) RETURNING *", hostname, path, owner, contact)
- query = self.db.get(query, *args)
+ # Log creation
+ mirror.log("created", user=user)
- return query.count
+ return mirror
def get_random(self, limit=None):
query = "SELECT id FROM mirrors WHERE status = 'enabled' ORDER BY RAND()"
return mirrors
def get_by_id(self, id):
- mirror = self.db.get("SELECT id FROM mirrors WHERE id = %s", id)
- if not mirror:
- return
-
- return Mirror(self.pakfire, mirror.id)
+ return self._get_mirror("SELECT * FROM mirrors WHERE id = %s", id)
def get_by_hostname(self, hostname):
- mirror = self.db.get("SELECT id FROM mirrors WHERE NOT status = 'deleted' \
- AND hostname = %s", hostname)
-
- if not mirror:
- return
+ return self._get_mirror("SELECT * FROM mirrors \
+ WHERE hostname = %s AND deleted IS FALSE", hostname)
- return Mirror(self.pakfire, mirror.id)
-
- def get_for_location(self, addr):
- country_code = self.backend.geoip.guess_from_address(addr)
+ def get_for_location(self, address):
+ country_code = self.backend.geoip.guess_from_address(address)
# Cannot return any good mirrors if location is unknown
if not country_code:
return entries
+ def check(self, **kwargs):
+ """
+ Runs the mirror check for all mirrors
+ """
+ for mirror in self:
+ with self.db.transaction():
+ mirror.check(**kwargs)
-class Mirror(base.Object):
- def __init__(self, pakfire, id):
- base.Object.__init__(self, pakfire)
-
- self.id = id
-
- # Cache.
- self._data = None
- self._location = None
- def __cmp__(self, other):
- return cmp(self.id, other.id)
+class Mirror(base.DataObject):
+ table = "mirrors"
- @classmethod
- def create(cls, pakfire, hostname, path="", owner=None, contact=None, user=None):
- id = pakfire.db.execute("INSERT INTO mirrors(hostname, path, owner, contact) \
- VALUES(%s, %s, %s, %s)", hostname, path, owner, contact)
-
- mirror = cls(pakfire, id)
- mirror.log("created", user=user)
-
- return mirror
+ def __eq__(self, other):
+ if isinstance(other, self.__class__):
+ return self.id == other.id
def log(self, action, user=None):
user_id = None
self.db.execute("INSERT INTO mirrors_history(mirror_id, action, user_id, time) \
VALUES(%s, %s, %s, NOW())", self.id, action, user_id)
- @property
- def data(self):
- if self._data is None:
- self._data = \
- self.db.get("SELECT * FROM mirrors WHERE id = %s", self.id)
-
- return self._data
-
- def set_status(self, status, user=None):
- assert status in ("enabled", "disabled", "deleted")
-
- if self.status == status:
- return
-
- self.db.execute("UPDATE mirrors SET status = %s WHERE id = %s",
- status, self.id)
-
- if self._data:
- self._data["status"] = status
-
- # Log the status change.
- self.log(status, user=user)
-
def set_hostname(self, hostname):
- if self.hostname == hostname:
- return
-
- self.db.execute("UPDATE mirrors SET hostname = %s WHERE id = %s",
- hostname, self.id)
-
- if self._data:
- self._data["hostname"] = hostname
+ self._set_attribute("hostname", hostname)
hostname = property(lambda self: self.data.hostname, set_hostname)
return self.data.path
def set_path(self, path):
- if self.path == path:
- return
-
- self.db.execute("UPDATE mirrors SET path = %s WHERE id = %s",
- path, self.id)
-
- if self._data:
- self._data["path"] = path
+ self._set_attribute("path", path)
path = property(lambda self: self.data.path, set_path)
@property
def url(self):
- ret = "http://%s" % self.hostname
+ return self.make_url()
- if self.path:
- path = self.path
+ def make_url(self, path=""):
+ url = "http://%s%s" % (self.hostname, self.path)
- if not self.path.startswith("/"):
- path = "/%s" % path
+ if path.startswith("/"):
+ path = path[1:]
- if self.path.endswith("/"):
- path = path[:-1]
+ return urlparse.urljoin(url, path)
- ret += path
+ def set_owner(self, owner):
+ self._set_attribute("owner", owner)
- return ret
+ owner = property(lambda self: self.data.owner or "", set_owner)
- def set_owner(self, owner):
- if self.owner == owner:
- return
+ def set_contact(self, contact):
+ self._set_attribute("contact", contact)
- self.db.execute("UPDATE mirrors SET owner = %s WHERE id = %s",
- owner, self.id)
+ contact = property(lambda self: self.data.contact or "", set_contact)
- if self._data:
- self._data["owner"] = owner
+ def check(self, connect_timeout=10, request_timeout=10):
+ log.info("Running mirror check for %s" % self.hostname)
- owner = property(lambda self: self.data.owner or "", set_owner)
+ client = tornado.httpclient.HTTPClient()
- def set_contact(self, contact):
- if self.contact == contact:
- return
+ # Get URL for .timestamp
+ url = self.make_url(".timestamp")
+ log.debug(" Fetching %s..." % url)
- self.db.execute("UPDATE mirrors SET contact = %s WHERE id = %s",
- contact, self.id)
+ # Record start time
+ time_start = time.time()
- if self._data:
- self._data["contact"] = contact
+ http_status = None
+ last_sync_at = None
+ status = "OK"
- contact = property(lambda self: self.data.contact or "", set_contact)
+ # XXX needs to catch connection resets, DNS errors, etc.
- @property
- def status(self):
- return self.data.status
+ try:
+ response = client.fetch(url,
+ connect_timeout=connect_timeout,
+ request_timeout=request_timeout)
- @property
- def enabled(self):
- return self.status == "enabled"
+ # We expect the response to be an integer
+ # which holds the timestamp of the last sync
+ # in seconds since epoch UTC
+ try:
+ timestamp = int(response.body)
+ except ValueError:
+ raise
+
+ # Convert to datetime
+ last_sync_at = datetime.datetime.utcfromtimestamp(timestamp)
+
+ # Must have synced within 24 hours
+ now = datetime.datetime.utcnow()
+ if now - last_sync_at >= datetime.timedelta(hours=24):
+ status = "OUTOFSYNC"
+
+ except tornado.httpclient.HTTPError as e:
+ http_status = e.code
+ status = "ERROR"
+
+ finally:
+ response_time = time.time() - time_start
+
+ # Log check
+ self.db.execute("INSERT INTO mirrors_checks(mirror_id, response_time, \
+ http_status, last_sync_at, status) VALUES(%s, %s, %s, %s, %s)",
+ self.id, response_time, http_status, last_sync_at, status)
+
+ @lazy_property
+ def last_check(self):
+ res = self.db.get("SELECT * FROM mirrors_checks \
+ WHERE mirror_id = %s ORDER BY timestamp DESC LIMIT 1", self.id)
+
+ return res
@property
- def check_status(self):
- return self.data.check_status
+ def status(self):
+ if self.last_check:
+ return self.last_check.status
@property
- def last_check(self):
- return self.data.last_check
+ def average_response_time(self):
+ res = self.db.get("SELECT AVG(response_time) AS response_time \
+ FROM mirrors_checks WHERE mirror_id = %s \
+ AND timestamp >= NOW() - '24 hours'::interval", self.id)
+
+ return res.response_time
@property
def address(self):
path text NOT NULL,
owner text,
contact text,
- status mirrors_status DEFAULT 'disabled'::mirrors_status NOT NULL,
- check_status mirrors_check_status DEFAULT 'UNKNOWN'::mirrors_check_status NOT NULL,
- last_check timestamp without time zone
+ deleted boolean DEFAULT false NOT NULL
);
ALTER TABLE mirrors OWNER TO pakfire;
+--
+-- Name: mirrors_checks; Type: TABLE; Schema: public; Owner: pakfire; Tablespace:
+--
+
+CREATE TABLE mirrors_checks (
+ id integer NOT NULL,
+ mirror_id integer NOT NULL,
+ "timestamp" timestamp without time zone DEFAULT now() NOT NULL,
+ response_time double precision,
+ http_status integer,
+ last_sync_at timestamp without time zone,
+ status text DEFAULT 'OK'::text NOT NULL
+);
+
+
+ALTER TABLE mirrors_checks OWNER TO pakfire;
+
+--
+-- Name: mirrors_checks_id_seq; Type: SEQUENCE; Schema: public; Owner: pakfire
+--
+
+CREATE SEQUENCE mirrors_checks_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+
+ALTER TABLE mirrors_checks_id_seq OWNER TO pakfire;
+
+--
+-- Name: mirrors_checks_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: pakfire
+--
+
+ALTER SEQUENCE mirrors_checks_id_seq OWNED BY mirrors_checks.id;
+
+
--
-- Name: mirrors_history; Type: TABLE; Schema: public; Owner: pakfire; Tablespace:
--
ALTER TABLE ONLY mirrors ALTER COLUMN id SET DEFAULT nextval('mirrors_id_seq'::regclass);
+--
+-- Name: id; Type: DEFAULT; Schema: public; Owner: pakfire
+--
+
+ALTER TABLE ONLY mirrors_checks ALTER COLUMN id SET DEFAULT nextval('mirrors_checks_id_seq'::regclass);
+
+
--
-- Name: id; Type: DEFAULT; Schema: public; Owner: pakfire
--
ADD CONSTRAINT jobs_packages_unique UNIQUE (job_id, pkg_id);
+--
+-- Name: mirrors_checks_pkey; Type: CONSTRAINT; Schema: public; Owner: pakfire; Tablespace:
+--
+
+ALTER TABLE ONLY mirrors_checks
+ ADD CONSTRAINT mirrors_checks_pkey PRIMARY KEY (id);
+
+
--
-- Name: sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: pakfire; Tablespace:
--
CREATE INDEX jobs_buildroots_pkg_uuid ON jobs_buildroots USING btree (pkg_uuid);
+--
+-- Name: mirrors_checks_sort; Type: INDEX; Schema: public; Owner: pakfire; Tablespace:
+--
+
+CREATE INDEX mirrors_checks_sort ON mirrors_checks USING btree (mirror_id, "timestamp");
+
+ALTER TABLE mirrors_checks CLUSTER ON mirrors_checks_sort;
+
+
--
-- Name: on_update_current_timestamp; Type: TRIGGER; Schema: public; Owner: pakfire
--
ADD CONSTRAINT logfiles_job_id FOREIGN KEY (job_id) REFERENCES jobs(id);
+--
+-- Name: mirrors_checks_mirror_id; Type: FK CONSTRAINT; Schema: public; Owner: pakfire
+--
+
+ALTER TABLE ONLY mirrors_checks
+ ADD CONSTRAINT mirrors_checks_mirror_id FOREIGN KEY (mirror_id) REFERENCES mirrors(id);
+
+
--
-- Name: mirrors_history_mirror_id; Type: FK CONSTRAINT; Schema: public; Owner: pakfire
--
projects / people / jschlag / pbs.git / commitdiff
