From: Michael Tremer Date: Sat, 7 Oct 2017 15:00:15 +0000 (+0100) Subject: Refactor mirrors X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=c660ff5933a6e0930a3af1633bafa94d63a118ff;p=pbs.git Refactor mirrors This adds checks twice an hour to see if the mirror is responding correctly, etc. Signed-off-by: Michael Tremer --- diff --git a/src/buildservice/mirrors.py b/src/buildservice/mirrors.py index 5c8bf140..58a0ba88 100644 --- a/src/buildservice/mirrors.py +++ b/src/buildservice/mirrors.py @@ -1,36 +1,47 @@ #!/usr/bin/python +import datetime import logging import math import socket +import time +import tornado.httpclient +import urlparse from . import base from . import logs +log = logging.getLogger("mirrors") +log.propagate = 1 + from .decorators import lazy_property class Mirrors(base.Object): - def get_all(self): - mirrors = [] + def __iter__(self): + res = self.db.query("SELECT * FROM mirrors \ + WHERE deleted IS FALSE ORDER BY hostname") - for mirror in self.db.query("SELECT id FROM mirrors \ - WHERE NOT status = 'deleted' ORDER BY hostname"): - mirror = Mirror(self.pakfire, mirror.id) + mirrors = [] + for row in res: + mirror = Mirror(self.backend, row.id, data=row) mirrors.append(mirror) - return mirrors + return iter(mirrors) - def count(self, status=None): - query = "SELECT COUNT(*) AS count FROM mirrors" - args = [] + def _get_mirror(self, query, *args): + res = self.db.get(query, *args) + + if res: + return Mirror(self.backend, res.id, data=res) - if status: - query += " WHERE status = %s" - args.append(status) + def create(self, hostname, path="", owner=None, contact=None, user=None): + mirror = self._get_mirror("INSERT INTO mirrors(hostname, path, owner, contact) \ + VALUES(%s, %s, %s, %s) RETURNING *", hostname, path, owner, contact) - query = self.db.get(query, *args) + # Log creation + mirror.log("created", user=user) - return query.count + return mirror def get_random(self, limit=None): query = "SELECT id FROM mirrors WHERE status = 'enabled' ORDER BY RAND()" @@ -48,23 +59,14 @@ class Mirrors(base.Object): return mirrors def get_by_id(self, id): - mirror = self.db.get("SELECT id FROM mirrors WHERE id = %s", id) - if not mirror: - return - - return Mirror(self.pakfire, mirror.id) + return self._get_mirror("SELECT * FROM mirrors WHERE id = %s", id) def get_by_hostname(self, hostname): - mirror = self.db.get("SELECT id FROM mirrors WHERE NOT status = 'deleted' \ - AND hostname = %s", hostname) - - if not mirror: - return + return self._get_mirror("SELECT * FROM mirrors \ + WHERE hostname = %s AND deleted IS FALSE", hostname) - return Mirror(self.pakfire, mirror.id) - - def get_for_location(self, addr): - country_code = self.backend.geoip.guess_from_address(addr) + def get_for_location(self, address): + country_code = self.backend.geoip.guess_from_address(address) # Cannot return any good mirrors if location is unknown if not country_code: @@ -118,29 +120,21 @@ class Mirrors(base.Object): return entries + def check(self, **kwargs): + """ + Runs the mirror check for all mirrors + """ + for mirror in self: + with self.db.transaction(): + mirror.check(**kwargs) -class Mirror(base.Object): - def __init__(self, pakfire, id): - base.Object.__init__(self, pakfire) - - self.id = id - - # Cache. - self._data = None - self._location = None - def __cmp__(self, other): - return cmp(self.id, other.id) +class Mirror(base.DataObject): + table = "mirrors" - @classmethod - def create(cls, pakfire, hostname, path="", owner=None, contact=None, user=None): - id = pakfire.db.execute("INSERT INTO mirrors(hostname, path, owner, contact) \ - VALUES(%s, %s, %s, %s)", hostname, path, owner, contact) - - mirror = cls(pakfire, id) - mirror.log("created", user=user) - - return mirror + def __eq__(self, other): + if isinstance(other, self.__class__): + return self.id == other.id def log(self, action, user=None): user_id = None @@ -150,38 +144,8 @@ class Mirror(base.Object): self.db.execute("INSERT INTO mirrors_history(mirror_id, action, user_id, time) \ VALUES(%s, %s, %s, NOW())", self.id, action, user_id) - @property - def data(self): - if self._data is None: - self._data = \ - self.db.get("SELECT * FROM mirrors WHERE id = %s", self.id) - - return self._data - - def set_status(self, status, user=None): - assert status in ("enabled", "disabled", "deleted") - - if self.status == status: - return - - self.db.execute("UPDATE mirrors SET status = %s WHERE id = %s", - status, self.id) - - if self._data: - self._data["status"] = status - - # Log the status change. - self.log(status, user=user) - def set_hostname(self, hostname): - if self.hostname == hostname: - return - - self.db.execute("UPDATE mirrors SET hostname = %s WHERE id = %s", - hostname, self.id) - - if self._data: - self._data["hostname"] = hostname + self._set_attribute("hostname", hostname) hostname = property(lambda self: self.data.hostname, set_hostname) @@ -190,73 +154,102 @@ class Mirror(base.Object): return self.data.path def set_path(self, path): - if self.path == path: - return - - self.db.execute("UPDATE mirrors SET path = %s WHERE id = %s", - path, self.id) - - if self._data: - self._data["path"] = path + self._set_attribute("path", path) path = property(lambda self: self.data.path, set_path) @property def url(self): - ret = "http://%s" % self.hostname + return self.make_url() - if self.path: - path = self.path + def make_url(self, path=""): + url = "http://%s%s" % (self.hostname, self.path) - if not self.path.startswith("/"): - path = "/%s" % path + if path.startswith("/"): + path = path[1:] - if self.path.endswith("/"): - path = path[:-1] + return urlparse.urljoin(url, path) - ret += path + def set_owner(self, owner): + self._set_attribute("owner", owner) - return ret + owner = property(lambda self: self.data.owner or "", set_owner) - def set_owner(self, owner): - if self.owner == owner: - return + def set_contact(self, contact): + self._set_attribute("contact", contact) - self.db.execute("UPDATE mirrors SET owner = %s WHERE id = %s", - owner, self.id) + contact = property(lambda self: self.data.contact or "", set_contact) - if self._data: - self._data["owner"] = owner + def check(self, connect_timeout=10, request_timeout=10): + log.info("Running mirror check for %s" % self.hostname) - owner = property(lambda self: self.data.owner or "", set_owner) + client = tornado.httpclient.HTTPClient() - def set_contact(self, contact): - if self.contact == contact: - return + # Get URL for .timestamp + url = self.make_url(".timestamp") + log.debug(" Fetching %s..." % url) - self.db.execute("UPDATE mirrors SET contact = %s WHERE id = %s", - contact, self.id) + # Record start time + time_start = time.time() - if self._data: - self._data["contact"] = contact + http_status = None + last_sync_at = None + status = "OK" - contact = property(lambda self: self.data.contact or "", set_contact) + # XXX needs to catch connection resets, DNS errors, etc. - @property - def status(self): - return self.data.status + try: + response = client.fetch(url, + connect_timeout=connect_timeout, + request_timeout=request_timeout) - @property - def enabled(self): - return self.status == "enabled" + # We expect the response to be an integer + # which holds the timestamp of the last sync + # in seconds since epoch UTC + try: + timestamp = int(response.body) + except ValueError: + raise + + # Convert to datetime + last_sync_at = datetime.datetime.utcfromtimestamp(timestamp) + + # Must have synced within 24 hours + now = datetime.datetime.utcnow() + if now - last_sync_at >= datetime.timedelta(hours=24): + status = "OUTOFSYNC" + + except tornado.httpclient.HTTPError as e: + http_status = e.code + status = "ERROR" + + finally: + response_time = time.time() - time_start + + # Log check + self.db.execute("INSERT INTO mirrors_checks(mirror_id, response_time, \ + http_status, last_sync_at, status) VALUES(%s, %s, %s, %s, %s)", + self.id, response_time, http_status, last_sync_at, status) + + @lazy_property + def last_check(self): + res = self.db.get("SELECT * FROM mirrors_checks \ + WHERE mirror_id = %s ORDER BY timestamp DESC LIMIT 1", self.id) + + return res @property - def check_status(self): - return self.data.check_status + def status(self): + if self.last_check: + return self.last_check.status @property - def last_check(self): - return self.data.last_check + def average_response_time(self): + res = self.db.get("SELECT AVG(response_time) AS response_time \ + FROM mirrors_checks WHERE mirror_id = %s \ + AND timestamp >= NOW() - '24 hours'::interval", self.id) + + return res.response_time @property def address(self): diff --git a/src/crontab/pakfire-build-service b/src/crontab/pakfire-build-service index e98eb806..bf9ab184 100644 --- a/src/crontab/pakfire-build-service +++ b/src/crontab/pakfire-build-service @@ -6,3 +6,6 @@ # Cleanup expired sessions 0 0 * * * nobody pakfire-build-service cleanup-sessions &>/dev/null + +# Run mirror check +*/30 * * * * nobody pakfire-build-service check-mirrors &>/dev/null diff --git a/src/database.sql b/src/database.sql index d6f9cbef..00c36fab 100644 --- a/src/database.sql +++ b/src/database.sql @@ -1602,14 +1602,50 @@ CREATE TABLE mirrors ( path text NOT NULL, owner text, contact text, - status mirrors_status DEFAULT 'disabled'::mirrors_status NOT NULL, - check_status mirrors_check_status DEFAULT 'UNKNOWN'::mirrors_check_status NOT NULL, - last_check timestamp without time zone + deleted boolean DEFAULT false NOT NULL ); ALTER TABLE mirrors OWNER TO pakfire; +-- +-- Name: mirrors_checks; Type: TABLE; Schema: public; Owner: pakfire; Tablespace: +-- + +CREATE TABLE mirrors_checks ( + id integer NOT NULL, + mirror_id integer NOT NULL, + "timestamp" timestamp without time zone DEFAULT now() NOT NULL, + response_time double precision, + http_status integer, + last_sync_at timestamp without time zone, + status text DEFAULT 'OK'::text NOT NULL +); + + +ALTER TABLE mirrors_checks OWNER TO pakfire; + +-- +-- Name: mirrors_checks_id_seq; Type: SEQUENCE; Schema: public; Owner: pakfire +-- + +CREATE SEQUENCE mirrors_checks_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE mirrors_checks_id_seq OWNER TO pakfire; + +-- +-- Name: mirrors_checks_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: pakfire +-- + +ALTER SEQUENCE mirrors_checks_id_seq OWNED BY mirrors_checks.id; + + -- -- Name: mirrors_history; Type: TABLE; Schema: public; Owner: pakfire; Tablespace: -- @@ -2396,6 +2432,13 @@ ALTER TABLE ONLY logfiles ALTER COLUMN id SET DEFAULT nextval('logfiles_id_seq': ALTER TABLE ONLY mirrors ALTER COLUMN id SET DEFAULT nextval('mirrors_id_seq'::regclass); +-- +-- Name: id; Type: DEFAULT; Schema: public; Owner: pakfire +-- + +ALTER TABLE ONLY mirrors_checks ALTER COLUMN id SET DEFAULT nextval('mirrors_checks_id_seq'::regclass); + + -- -- Name: id; Type: DEFAULT; Schema: public; Owner: pakfire -- @@ -2788,6 +2831,14 @@ ALTER TABLE ONLY jobs_packages ADD CONSTRAINT jobs_packages_unique UNIQUE (job_id, pkg_id); +-- +-- Name: mirrors_checks_pkey; Type: CONSTRAINT; Schema: public; Owner: pakfire; Tablespace: +-- + +ALTER TABLE ONLY mirrors_checks + ADD CONSTRAINT mirrors_checks_pkey PRIMARY KEY (id); + + -- -- Name: sessions_pkey; Type: CONSTRAINT; Schema: public; Owner: pakfire; Tablespace: -- @@ -3091,6 +3142,15 @@ CREATE INDEX idx_2198256_user_id ON users_emails USING btree (user_id); CREATE INDEX jobs_buildroots_pkg_uuid ON jobs_buildroots USING btree (pkg_uuid); +-- +-- Name: mirrors_checks_sort; Type: INDEX; Schema: public; Owner: pakfire; Tablespace: +-- + +CREATE INDEX mirrors_checks_sort ON mirrors_checks USING btree (mirror_id, "timestamp"); + +ALTER TABLE mirrors_checks CLUSTER ON mirrors_checks_sort; + + -- -- Name: on_update_current_timestamp; Type: TRIGGER; Schema: public; Owner: pakfire -- @@ -3354,6 +3414,14 @@ ALTER TABLE ONLY logfiles ADD CONSTRAINT logfiles_job_id FOREIGN KEY (job_id) REFERENCES jobs(id); +-- +-- Name: mirrors_checks_mirror_id; Type: FK CONSTRAINT; Schema: public; Owner: pakfire +-- + +ALTER TABLE ONLY mirrors_checks + ADD CONSTRAINT mirrors_checks_mirror_id FOREIGN KEY (mirror_id) REFERENCES mirrors(id); + + -- -- Name: mirrors_history_mirror_id; Type: FK CONSTRAINT; Schema: public; Owner: pakfire -- diff --git a/src/scripts/pakfire-build-service b/src/scripts/pakfire-build-service index c6a23c91..2fb2cc07 100644 --- a/src/scripts/pakfire-build-service +++ b/src/scripts/pakfire-build-service @@ -14,6 +14,9 @@ class Cli(object): self.backend = pakfire.buildservice.Backend(*args, **kwargs) self._commands = { + # Run mirror check + "check-mirrors" : self.backend.mirrors.check, + # Cleanup sessions "cleanup-sessions" : self.backend.sessions.cleanup, diff --git a/src/templates/mirrors-detail.html b/src/templates/mirrors-detail.html index 4751d2dd..4b9ac38e 100644 --- a/src/templates/mirrors-detail.html +++ b/src/templates/mirrors-detail.html @@ -75,21 +75,48 @@

{{ _("Status information") }}

- - - - + {% if not mirror.status == "OK" %} + + + + - - - - + {% if mirror.status == "ERROR" %} + + + + + {% end %} + + {% if mirror.last_check.last_sync_at %} + + + + + {% end %} + + + + + + {% end %} + + {% if mirror.average_response_time %} + + + + + {% end %}
{{ _("Status") }}{{ mirror.status }}
{{ _("Status") }}{{ mirror.status }}
{{ _("Last check") }} - {% if mirror.last_check %} - {{ format_date(mirror.last_check) }} - {% else %} - {{ _("Never") }} - {% end %} -
{{ _("HTTP Response Code") }}{{ mirror.last_check.http_status }}
{{ _("Last sync") }} + {{ locale.format_date(mirror.last_check.last_sync_at) }} +
{{ _("Last check") }} + {% if mirror.last_check %} + {{ format_date(mirror.last_check.timestamp) }} + {% else %} + {{ _("Never") }} + {% end %} +
{{ _("Average Response Time") }} + {{ "%.2fms" % (mirror.average_response_time * 1000) }} +
diff --git a/src/templates/mirrors-list.html b/src/templates/mirrors-list.html index 4f67e109..bbf03f1d 100644 --- a/src/templates/mirrors-list.html +++ b/src/templates/mirrors-list.html @@ -59,11 +59,15 @@ [{{ mirror.country_code }}] - - {% if mirror.check_status == "UP" %} + {% if mirror.status == "OK" %} {{ _("Up") }} - {% elif mirror.check_status == "DOWN" %} + {% elif mirror.status == "OUTOFSYNC" %} + + {{ _("Out Of Sync") }} + + {% elif mirror.status == "ERROR" %} {{ _("Down") }} @@ -76,7 +80,7 @@ {% if mirror.last_check %} - {{ format_date(mirror.last_check, relative=True) }} + {{ format_date(mirror.last_check.timestamp, relative=True) }} {% else %} {{ _("N/A") }} {% end %} diff --git a/src/web/handlers.py b/src/web/handlers.py index e523b8c6..49e5e52f 100644 --- a/src/web/handlers.py +++ b/src/web/handlers.py @@ -1,5 +1,6 @@ #!/usr/bin/python +import random import tornado.web from .handlers_auth import * @@ -225,19 +226,7 @@ class RepositoryMirrorlistHandler(BaseHandler): # on mirror servers. if repo.mirrored: - # See how many mirrors we can max. find. - num_mirrors = self.mirrors.count(status="enabled") - assert num_mirrors > 0 - - # Create a list with all mirrors that is up to 50 mirrors long. - # First add all preferred mirrors and then fill the rest up - # with other mirrors. - if num_mirrors >= 10: - MAX_MIRRORS = 10 - else: - MAX_MIRRORS = num_mirrors - - + # Select a list of preferred mirrors for mirror in self.mirrors.get_for_location(self.current_address): mirrors.append({ "url" : "/".join((mirror.url, distro.identifier, repo.identifier, arch.name)), @@ -245,17 +234,16 @@ class RepositoryMirrorlistHandler(BaseHandler): "preferred" : 1, }) - while MAX_MIRRORS - len(mirrors) > 0: - mirror = self.mirrors.get_random(limit=1)[0] + # Add all other mirrors at the end in a random order + remaining_mirrors = [m for m in self.backend.mirrors if not m in mirrors] + random.shuffle(remaining_mirrors) - mirror = { + for mirror in remaining_mirrors: + mirrors.append({ "url" : "/".join((mirror.url, distro.identifier, repo.identifier, arch.name)), "location" : mirror.country_code, "preferred" : 0, - } - - if not mirror in mirrors: - mirrors.append(mirror) + }) else: repo_baseurl = self.pakfire.settings.get("repository_baseurl") diff --git a/src/web/handlers_mirrors.py b/src/web/handlers_mirrors.py index 4e4d7e35..42d986a3 100644 --- a/src/web/handlers_mirrors.py +++ b/src/web/handlers_mirrors.py @@ -8,7 +8,7 @@ from .handlers_base import BaseHandler class MirrorListHandler(BaseHandler): def get(self): - mirrors = self.pakfire.mirrors.get_all() + mirrors = self.pakfire.mirrors mirrors_nearby = self.pakfire.mirrors.get_for_location(self.current_address) mirrors_worldwide = []