From: Michael Tremer Date: Tue, 16 May 2023 13:49:19 +0000 (+0000) Subject: mirrors: Refactor everything X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=1523bc410536d7f7999c6fc47562f1f5a85363c3;p=pbs.git mirrors: Refactor everything Signed-off-by: Michael Tremer --- diff --git a/Makefile.am b/Makefile.am index b13acdf8..72c4e250 100644 --- a/Makefile.am +++ b/Makefile.am @@ -268,13 +268,17 @@ templates_messagesdir = $(templatesdir)/messages dist_templates_mirrors_DATA = \ src/templates/mirrors/delete.html \ - src/templates/mirrors/detail.html \ src/templates/mirrors/edit.html \ - src/templates/mirrors/list.html \ - src/templates/mirrors/new.html + src/templates/mirrors/index.html \ + src/templates/mirrors/show.html templates_mirrorsdir = $(templatesdir)/mirrors +dist_templates_mirrors_modules_DATA = \ + src/templates/mirrors/modules/list.html + +templates_mirrors_modulesdir = $(templates_mirrorsdir)/modules + dist_templates_modules_DATA = \ src/templates/modules/commits-table.html \ src/templates/modules/commit-message.html \ diff --git a/src/buildservice/mirrors.py b/src/buildservice/mirrors.py index feeea0dc..2c0bcdf7 100644 --- a/src/buildservice/mirrors.py +++ b/src/buildservice/mirrors.py @@ -1,26 +1,24 @@ #!/usr/bin/python +import asyncio import datetime import logging -import math +import random import socket -import time -import tornado.httpclient +import tornado.netutil import urllib.parse import location from . import base +from . import httpclient -from .decorators import lazy_property +from .decorators import * # Setup logging log = logging.getLogger("pbs.mirrors") class Mirrors(base.Object): - def init(self): - self.location = location.Database("/var/lib/location/database.db") - def _get_mirror(self, query, *args): res = self.db.get(query, *args) @@ -34,99 +32,145 @@ class Mirrors(base.Object): yield Mirror(self.backend, row.id, data=row) def __iter__(self): - mirrors = self._get_mirrors("SELECT * FROM mirrors \ - WHERE deleted IS FALSE ORDER BY hostname") + mirrors = self._get_mirrors(""" + SELECT + * + FROM + mirrors + WHERE + deleted_at IS NULL + ORDER BY + hostname + """, + ) return iter(mirrors) - @property - def random(self): + async def create(self, hostname, path, owner, contact, notes, user=None, check=True): """ - Returns all mirrors in a random order + Creates a new mirror """ - return self._get_mirrors("SELECT * FROM mirrors \ - WHERE deleted IS FALSE ORDER BY RANDOM()") + mirror = self._get_mirror(""" + INSERT INTO + mirrors + ( + hostname, + path, + owner, + contact, + notes, + created_by + ) + VALUES( + %s, %s, %s, %s, %s, %s + ) + RETURNING + * + """, hostname, path, owner, contact, notes, user, + ) - def create(self, hostname, path="", owner=None, contact=None, user=None): - mirror = self._get_mirror("INSERT INTO mirrors(hostname, path, owner, contact) \ - VALUES(%s, %s, %s, %s) RETURNING *", hostname, path, owner, contact) + log.info("Mirror %s has been created" % mirror) + + # Perform the first check + if check: + await mirror.check(force=True) return mirror def get_by_id(self, id): - return self._get_mirror("SELECT * FROM mirrors WHERE id = %s", id) + return self._get_mirror(""" + SELECT + * + FROM + mirrors + WHERE + id = %s + """, id, + ) def get_by_hostname(self, hostname): - return self._get_mirror("SELECT * FROM mirrors \ - WHERE hostname = %s AND deleted IS FALSE", hostname) - - def make_mirrorlist(self, client_address=None): - network = self.location.lookup(client_address) - - # Walk through all mirrors in a random order - # and put all preferred mirrors to the front of the list - # and everything else at the end - mirrors = [] - for mirror in self.random: - if mirror.is_preferred_for_country(network.country_code if network else None): - mirrors.insert(0, mirror) - else: - mirrors.append(mirror) + return self._get_mirror(""" + SELECT + * + FROM + mirrors + WHERE + deleted_at IS NULL + AND + hostname = %s + """, hostname, + ) + + def get_mirrors_for_address(self, address): + """ + Returns all mirrors in random order with preferred mirrors first + """ + # Lookup the client + network = self.location.lookup(address) + + def __sort(mirror): + # Generate some random value for each mirror + r = random.random() + + # Put preferred mirrors first + if network and mirror.is_preferred_for_network(network): + r += 1 + + return r - return mirrors + # Fetch all mirrors and shuffle them, but put preferred mirrors first + return sorted(self, key=__sort) - def check(self, **kwargs): + @lazy_property + def location(self): + """ + The location database + """ + return location.Database("/var/lib/location/database.db") + + @lazy_property + def resolver(self): + """ + A DNS resolver + """ + return tornado.netutil.ThreadedResolver() + + async def check(self, *args, **kwargs): """ Runs the mirror check for all mirrors """ - for mirror in self: - with self.db.transaction(): - mirror.check(**kwargs) + # Check all mirrors concurrently + async with asyncio.TaskGroup() as tg: + for mirror in self: + tg.create_task(mirror.check(*args, **kwargs)) class Mirror(base.DataObject): table = "mirrors" + def __str__(self): + return self.hostname + def __lt__(self, other): if isinstance(other, self.__class__): return self.hostname < other.hostname return NotImplemented - def set_hostname(self, hostname): - self._set_attribute("hostname", hostname) - - hostname = property(lambda self: self.data.hostname, set_hostname) - - def set_deleted(self, deleted): - self._set_attribute("deleted", deleted) - - deleted = property(lambda s: s.data.deleted, set_deleted) - - def has_perm(self, user): - # Anonymous users have no permission - if not user: - return False - - # Admins have all permissions - return user.is_admin() + @property + def hostname(self): + return self.data.hostname @property def path(self): return self.data.path - def set_path(self, path): - self._set_attribute("path", path) - - path = property(lambda self: self.data.path, set_path) - @property def url(self): return self.make_url() def make_url(self, path=""): - url = "%s://%s%s" % ( - "https" if self.supports_https else "http", + url = "https://%s%s" % ( self.hostname, self.path ) @@ -136,111 +180,301 @@ class Mirror(base.DataObject): return urllib.parse.urljoin(url, path) - def set_supports_https(self, supports_https): - self._set_attribute("supports_https", supports_https) + @property + def last_check_success(self): + """ + True if the last check was successful + """ + return self.data.last_check_success + + @property + def last_check_at(self): + """ + The timestamp of the last check + """ + return self.data.last_check_at + + @property + def error(self): + """ + The error message of the last unsuccessful check + """ + return self.data.error + + @property + def created_at(self): + return self.data.created_at + + # Delete + + def delete(self, user): + """ + Deleted this mirror + """ + self._set_attribute_now("deleted_at") + if user: + self._set_attribute("deleted_by", user) + + # Log the event + log.info("Mirror %s has been deleted" % self) + + def has_perm(self, user): + # Anonymous users have no permission + if not user: + return False + + # Admins have all permissions + return user.is_admin() + + # Owner - supports_https = property(lambda s: s.data.supports_https, set_supports_https) + def get_owner(self): + return self.data.owner def set_owner(self, owner): self._set_attribute("owner", owner) - owner = property(lambda self: self.data.owner or "", set_owner) + owner = property(get_owner, set_owner) + + # Contact + + def get_contact(self): + return self.data.contact def set_contact(self, contact): self._set_attribute("contact", contact) - contact = property(lambda self: self.data.contact or "", set_contact) + contact = property(get_contact, set_contact) - def check(self, connect_timeout=10, request_timeout=10): - log.info("Running mirror check for %s" % self.hostname) + # Notes - client = tornado.httpclient.HTTPClient() + def get_notes(self): + return self.data.notes - # Get URL for .timestamp - url = self.make_url(".timestamp") - log.debug(" Fetching %s..." % url) + def set_notes(self, notes): + self._set_attribute("notes", notes or "") - # Record start time - time_start = time.time() + notes = property(get_notes, set_notes) - http_status = None - last_sync_at = None - status = "OK" + # Country Code & ASN - # XXX needs to catch connection resets, DNS errors, etc. + @property + def country_code(self): + """ + The country code + """ + return self.data.country_code + + @lazy_property + def asn(self): + """ + The Autonomous System + """ + if self.data.asn: + return self.backend.mirrors.location.get_as(self.data.asn) + async def _update_country_code_and_asn(self): + """ + Updates the country code of this mirror + """ + # Resolve the hostname try: - response = client.fetch(url, - connect_timeout=connect_timeout, - request_timeout=request_timeout) + addresses = await self.backend.mirrors.resolver.resolve(self.hostname, port=443) - # We expect the response to be an integer - # which holds the timestamp of the last sync - # in seconds since epoch UTC - try: - timestamp = int(response.body) + # XXX Catch this! + except socket.gaierror as e: + # Name or service not known + if e.errno == -2: + log.error("Could not resolve %s: %s" % (self, e)) + return - # If we could not parse the timestamp, we probably got - # an error page or something similar. - # So that's an error then... - except ValueError: - status = "ERROR" + # Raise anything else + raise e - # Timestamp seems to be okay - else: - # Convert to datetime - last_sync_at = datetime.datetime.utcfromtimestamp(timestamp) + for family, address in addresses: + # Extract the address + address = address[0] - # Must have synced within 24 hours - now = datetime.datetime.utcnow() - if now - last_sync_at >= datetime.timedelta(hours=24): - status = "OUTOFSYNC" + # Lookup the address + network = self.backend.mirrors.location.lookup(address) + if not network or not network.country_code: + continue - except tornado.httpclient.HTTPError as e: - http_status = e.code - status = "ERROR" + # Store the country code + self._set_attribute("country_code", network.country_code) - finally: - response_time = time.time() - time_start + # Store the ASN + self._set_attribute("asn", network.asn) - # Log check - self.db.execute("INSERT INTO mirrors_checks(mirror_id, response_time, \ - http_status, last_sync_at, status) VALUES(%s, %s, %s, %s, %s)", - self.id, response_time, http_status, last_sync_at, status) + # Once is enough + break - @lazy_property - def last_check(self): - res = self.db.get("SELECT * FROM mirrors_checks \ - WHERE mirror_id = %s ORDER BY timestamp DESC LIMIT 1", self.id) + def is_preferred_for_network(self, network): + """ + Returns True if this mirror is preferred for clients on the given network. + """ + # If the AS matches, we will prefer this + if self.asn and self.asn.number == network.asn: + return True - return res + # If the mirror and client are in the same country, we prefer this + if self.country_code and self.country_code == network.country_code: + return True - @property - def status(self): - if self.last_check: - return self.last_check.status + return False - @property - def average_response_time(self): - res = self.db.get("SELECT AVG(response_time) AS response_time \ - FROM mirrors_checks WHERE mirror_id = %s \ - AND timestamp >= NOW() - '24 hours'::interval", self.id) + # Check - return res.response_time + async def check(self, force=False): + t = datetime.datetime.utcnow() - @property - def address(self): - return socket.gethostbyname(self.hostname) + # Ratelimit checks somewhat + if not force: + # Check mirrors that are up only once an hour + if self.last_check_success is True: + if self.last_check_at + datetime.timedelta(hours=1) > t: + log.debug("Skipping check for %s" % self) + return - def is_preferred_for_country(self, country_code): - if country_code and self.country_code: - return self.country_code == country_code + # Check mirrors that are down once every 15 minutes + elif self.last_check_success is False: + if self.last_check_at + datetime.timedelta(minutes=15) > t: + log.debug("Skipping check for %s" % self) + return - @lazy_property - def country_code(self): - network = self.backend.mirrors.location.lookup(self.address) + log.debug("Running mirror check for %s" % self.hostname) + + # Wrap this into one large transaction + with self.db.transaction(): + # Update the country code & ASN + await self._update_country_code_and_asn() + + # Make URL for .timestamp + url = self.make_url(".timestamp") + + response = None + error = None + + # Was this check successful? + success = False + + # When was the last sync? + timestamp = None + + try: + response = await self.backend.httpclient.fetch( + url, - if network: - return network.country_code + # Allow a moment to connect and get a response + connect_timeout=10, + request_timeout=10, + ) + + # Try to parse the response + try: + timestamp = int(response.body) + + except (TypeError, ValueError) as e: + log.error("%s responded with an invalid timestamp") + + raise ValueError("Invalid timestamp received") from e + + # Convert into datetime + timestamp = datetime.datetime.utcfromtimestamp(timestamp) + + # Catch anything that isn't 200 OK + except httpclient.HTTPError as e: + log.error("%s: %s" % (self, e)) + error = "%s" % e + + # Catch DNS Errors + except socket.gaierror as e: + # Name or service not known + if e.code == -2: + log.error("Could not resolve %s: %s" % (self, e)) + + # Store the error + error = "%s" % e + + # Raise anything else + else: + raise e + + # Success! + else: + # This check was successful! + success = True + + # Log this check + self.db.execute(""" + INSERT INTO + mirror_checks + ( + mirror_id, + success, + response_time, + http_status, + last_sync_at, + error + ) + VALUES + ( + %s, %s, %s, %s, %s, %s + ) + """, + self.id, + success, + response.request_time if response else None, + response.code if response else None, + timestamp, + error, + ) + + # Update the main table + self._set_attribute_now("last_check_at") + self._set_attribute("last_check_success", success) + self._set_attribute("last_sync_at", timestamp) + self._set_attribute("error", error) + + def get_uptime_since(self, t): + # Convert timedeltas to absolute time + if isinstance(t, datetime.timedelta): + t = datetime.datetime.utcnow() - t + + res = self.db.get(""" + -- SELECT all successful checks and find out when the next one failed + WITH uptimes AS ( + SELECT + success, + LEAST( + LEAD(checked_at, 1, CURRENT_TIMESTAMP) + OVER (ORDER BY checked_at ASC) + - checked_at, + INTERVAL '1 hour' + ) AS uptime + FROM + mirror_checks + WHERE + mirror_id = %s + AND + checked_at >= %s + ) + SELECT + ( + EXTRACT( + epoch FROM SUM(uptime) FILTER (WHERE success IS TRUE) + ) + / + EXTRACT( + epoch FROM SUM(uptime) + ) + ) AS uptime + FROM + uptimes + """, self.id, t, + ) + + if res: + return res.uptime or 0 - return "UNKNOWN" + return 0 diff --git a/src/crontab/pakfire-build-service b/src/crontab/pakfire-build-service index e747b72f..e08f397f 100644 --- a/src/crontab/pakfire-build-service +++ b/src/crontab/pakfire-build-service @@ -20,6 +20,3 @@ MAILTO=pakfire@ipfire.org # Send updates to Bugzilla #*/5 * * * * _pakfire pakfire-build-service send-bug-updates &>/dev/null - -# Run mirror check -#*/30 * * * * _pakfire pakfire-build-service check-mirrors &>/dev/null diff --git a/src/database.sql b/src/database.sql index b39c0d43..fa6fbd2c 100644 --- a/src/database.sql +++ b/src/database.sql @@ -41,6 +41,61 @@ SET default_tablespace = ''; SET default_table_access_method = heap; +-- +-- Name: build_comments; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.build_comments ( + id integer NOT NULL, + build_id integer NOT NULL, + user_id integer NOT NULL, + text text DEFAULT ''::text NOT NULL, + created_at timestamp without time zone DEFAULT now() NOT NULL, + deleted boolean DEFAULT false NOT NULL +); + + +-- +-- Name: build_groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.build_groups ( + id integer NOT NULL, + uuid uuid DEFAULT gen_random_uuid() NOT NULL, + created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + created_by integer, + deleted_at timestamp without time zone, + deleted_by integer, + finished_at timestamp without time zone, + failed boolean DEFAULT false NOT NULL, + tested_build_id integer +); + + +-- +-- Name: build_points; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.build_points ( + build_id integer NOT NULL, + created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + points integer DEFAULT 0 NOT NULL, + user_id integer +); + + +-- +-- Name: build_watchers; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.build_watchers ( + build_id integer NOT NULL, + user_id integer NOT NULL, + added_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + deleted_at timestamp without time zone +); + + -- -- Name: builds; Type: TABLE; Schema: public; Owner: - -- @@ -100,33 +155,17 @@ CREATE TABLE public.jobs ( -- --- Name: build_comments; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.build_comments ( - id integer NOT NULL, - build_id integer NOT NULL, - user_id integer NOT NULL, - text text DEFAULT ''::text NOT NULL, - created_at timestamp without time zone DEFAULT now() NOT NULL, - deleted boolean DEFAULT false NOT NULL -); - - --- --- Name: build_groups; Type: TABLE; Schema: public; Owner: - +-- Name: repository_builds; Type: TABLE; Schema: public; Owner: - -- -CREATE TABLE public.build_groups ( +CREATE TABLE public.repository_builds ( id integer NOT NULL, - uuid uuid DEFAULT gen_random_uuid() NOT NULL, - created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, - created_by integer, - deleted_at timestamp without time zone, - deleted_by integer, - finished_at timestamp without time zone, - failed boolean DEFAULT false NOT NULL, - tested_build_id integer + repo_id integer NOT NULL, + build_id bigint NOT NULL, + added_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + added_by integer, + removed_at timestamp without time zone, + removed_by integer ); @@ -162,18 +201,6 @@ CREATE TABLE public.build_packages ( ); --- --- Name: build_points; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.build_points ( - build_id integer NOT NULL, - created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, - points integer DEFAULT 0 NOT NULL, - user_id integer -); - - -- -- Name: build_test_builds; Type: VIEW; Schema: public; Owner: - -- @@ -187,18 +214,6 @@ CREATE VIEW public.build_test_builds AS WHERE ((builds.deleted_at IS NULL) AND (build_groups.deleted_at IS NULL)); --- --- Name: build_watchers; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.build_watchers ( - build_id integer NOT NULL, - user_id integer NOT NULL, - added_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, - deleted_at timestamp without time zone -); - - -- -- Name: builder_stats; Type: TABLE; Schema: public; Owner: - -- @@ -529,52 +544,42 @@ ALTER SEQUENCE public.messages_id_seq OWNED BY public.messages.id; -- --- Name: mirrors; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.mirrors ( - id integer NOT NULL, - hostname text NOT NULL, - path text NOT NULL, - owner text, - contact text, - deleted boolean DEFAULT false NOT NULL, - supports_https boolean DEFAULT false NOT NULL -); - - --- --- Name: mirrors_checks; Type: TABLE; Schema: public; Owner: - +-- Name: mirror_checks; Type: TABLE; Schema: public; Owner: - -- -CREATE TABLE public.mirrors_checks ( - id integer NOT NULL, +CREATE TABLE public.mirror_checks ( mirror_id integer NOT NULL, - "timestamp" timestamp without time zone DEFAULT now() NOT NULL, + checked_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + success boolean DEFAULT false NOT NULL, response_time double precision, http_status integer, last_sync_at timestamp without time zone, - status text DEFAULT 'OK'::text NOT NULL + error text ); -- --- Name: mirrors_checks_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.mirrors_checks_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: mirrors_checks_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- Name: mirrors; Type: TABLE; Schema: public; Owner: - -- -ALTER SEQUENCE public.mirrors_checks_id_seq OWNED BY public.mirrors_checks.id; +CREATE TABLE public.mirrors ( + id integer NOT NULL, + hostname text NOT NULL, + path text NOT NULL, + owner text, + contact text, + created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + created_by integer NOT NULL, + deleted_at timestamp without time zone, + deleted_by integer, + last_check_success boolean, + last_check_at timestamp without time zone, + last_sync_at timestamp without time zone, + country_code text, + error text, + asn integer, + notes text DEFAULT ''::text NOT NULL +); -- @@ -878,21 +883,6 @@ CREATE SEQUENCE public.repositories_id_seq ALTER SEQUENCE public.repositories_id_seq OWNED BY public.repositories.id; --- --- Name: repository_builds; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.repository_builds ( - id integer NOT NULL, - repo_id integer NOT NULL, - build_id bigint NOT NULL, - added_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, - added_by integer, - removed_at timestamp without time zone, - removed_by integer -); - - -- -- Name: repository_builds_id_seq; Type: SEQUENCE; Schema: public; Owner: - -- @@ -1211,13 +1201,6 @@ ALTER TABLE ONLY public.messages ALTER COLUMN id SET DEFAULT nextval('public.mes ALTER TABLE ONLY public.mirrors ALTER COLUMN id SET DEFAULT nextval('public.mirrors_id_seq'::regclass); --- --- Name: mirrors_checks id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.mirrors_checks ALTER COLUMN id SET DEFAULT nextval('public.mirrors_checks_id_seq'::regclass); - - -- -- Name: packages id; Type: DEFAULT; Schema: public; Owner: - -- @@ -1344,14 +1327,6 @@ ALTER TABLE ONLY public.images_types ADD CONSTRAINT idx_2198057_primary PRIMARY KEY (id); --- --- Name: mirrors idx_2198115_primary; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.mirrors - ADD CONSTRAINT idx_2198115_primary PRIMARY KEY (id); - - -- -- Name: users idx_2198244_primary; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -1385,11 +1360,11 @@ ALTER TABLE ONLY public.messages -- --- Name: mirrors_checks mirrors_checks_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- Name: mirrors mirrors_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.mirrors_checks - ADD CONSTRAINT mirrors_checks_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.mirrors + ADD CONSTRAINT mirrors_pkey PRIMARY KEY (id); -- @@ -1670,12 +1645,17 @@ CREATE INDEX messages_queued ON public.messages USING btree (priority DESC, queu -- --- Name: mirrors_checks_sort; Type: INDEX; Schema: public; Owner: - +-- Name: mirror_checks_search; Type: INDEX; Schema: public; Owner: - -- -CREATE INDEX mirrors_checks_sort ON public.mirrors_checks USING btree (mirror_id, "timestamp"); +CREATE INDEX mirror_checks_search ON public.mirror_checks USING btree (mirror_id, checked_at); + + +-- +-- Name: mirrors_hostname; Type: INDEX; Schema: public; Owner: - +-- -ALTER TABLE public.mirrors_checks CLUSTER ON mirrors_checks_sort; +CREATE UNIQUE INDEX mirrors_hostname ON public.mirrors USING btree (hostname) WHERE (deleted_at IS NULL); -- @@ -2044,11 +2024,27 @@ ALTER TABLE ONLY public.keys -- --- Name: mirrors_checks mirrors_checks_mirror_id; Type: FK CONSTRAINT; Schema: public; Owner: - +-- Name: mirror_checks mirror_checks_mirror_id; Type: FK CONSTRAINT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.mirrors_checks - ADD CONSTRAINT mirrors_checks_mirror_id FOREIGN KEY (mirror_id) REFERENCES public.mirrors(id); +ALTER TABLE ONLY public.mirror_checks + ADD CONSTRAINT mirror_checks_mirror_id FOREIGN KEY (mirror_id) REFERENCES public.mirrors(id); + + +-- +-- Name: mirrors mirrors_created_by; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.mirrors + ADD CONSTRAINT mirrors_created_by FOREIGN KEY (created_by) REFERENCES public.users(id); + + +-- +-- Name: mirrors mirrors_deleted_by; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.mirrors + ADD CONSTRAINT mirrors_deleted_by FOREIGN KEY (deleted_by) REFERENCES public.users(id); -- diff --git a/src/scripts/pakfire-build-service b/src/scripts/pakfire-build-service index ac5256e2..8622a426 100644 --- a/src/scripts/pakfire-build-service +++ b/src/scripts/pakfire-build-service @@ -37,6 +37,9 @@ class Cli(object): # Messages "messages:queue:send" : self.backend.messages.queue.send, + # Mirrors + "mirrors:check" : self._mirrors_check, + # Release Monitoring "releasemonitoring:update" : self._release_monitoring_update, @@ -49,9 +52,6 @@ class Cli(object): # Sync "sync" : self.backend.sync, - # Run mirror check - #"check-mirrors" : self.backend.mirrors.check, - # Dist #"dist" : self.backend.sources.dist, @@ -191,6 +191,12 @@ class Cli(object): if release: print(" Found new release: %s" % release) + async def _mirrors_check(self): + """ + Forces a check on all mirrors + """ + return await self.backend.mirrors.check(force=True) + async def main(): cli = Cli() diff --git a/src/templates/mirrors/delete.html b/src/templates/mirrors/delete.html index 53b6fcac..8961e1b1 100644 --- a/src/templates/mirrors/delete.html +++ b/src/templates/mirrors/delete.html @@ -1,51 +1,43 @@ -{% extends "../base.html" %} +{% extends "../modal.html" %} -{% block title %}{{ _("Delete mirror %s") % mirror.hostname }}{% end block %} +{% block title %}{{ _("Delete Mirror") }} - {{ mirror }}{% end block %} -{% block body %} -
-
- -
-
+{% block breadcrumbs %} + +{% end block %} -
-
- -
-
+{% block modal_title %} +

{{ _("Delete Mirror") }}

+
{{ mirror }}
+{% end block %} + +{% block modal %} +
+ {% raw xsrf_form_html() %} -
-
- +
+

+ {{ _("Are you sure you want to delete %s?") % mirror }} +

-
+ {# Submit! #} +
+ +
+ {% end block %} diff --git a/src/templates/mirrors/detail.html b/src/templates/mirrors/detail.html deleted file mode 100644 index 40c632c3..00000000 --- a/src/templates/mirrors/detail.html +++ /dev/null @@ -1,136 +0,0 @@ -{% extends "../base.html" %} - -{% block title %}{{ _("Mirror: %s") % mirror.hostname }}{% end block %} - -{% block body %} -
-
- -
-
- - -
-
-

- {{ _("Mirror: %s") % mirror.hostname }}
- {{ mirror.owner }} -

-
- {% if mirror.has_perm(current_user) %} -
- -
- - {{ _("Add new mirror") }} - - - - {{ _("Edit settings") }} - - - - {{ _("Delete mirror") }} - -
-
- {% end %} -
- -
-
-

{{ _("General") }}

-
- - - - - - - - - - - - {% if mirror.has_perm(current_user) %} - - - - - {% end %} - -
{{ _("Hostname") }}{{ mirror.hostname }}
{{ _("Location") }}{{ _("The location of the mirror server could not be estimated.") }}
{{ _("Contact") }} - {% if mirror.contact %} - {{ mirror.contact }} - {% else %} - {{ _("N/A") }} - {% end %} -
-
-
-
- -
-
-

{{ _("Status information") }}

-
- - - {% if not mirror.status == "OK" %} - - - - - - {% if mirror.status == "ERROR" %} - - - - - {% end %} - - {% if mirror.last_check and mirror.last_check.last_sync_at %} - - - - - {% end %} - - - - - - {% end %} - - {% if mirror.average_response_time %} - - - - - {% end %} - -
{{ _("Status") }}{{ mirror.status }}
{{ _("HTTP Response Code") }}{{ mirror.last_check.http_status }}
{{ _("Last sync") }} - {{ locale.format_date(mirror.last_check.last_sync_at) }} -
{{ _("Last check") }} - {% if mirror.last_check %} - {{ format_date(mirror.last_check.timestamp) }} - {% else %} - {{ _("Never") }} - {% end %} -
{{ _("Average Response Time") }} - {{ "%.2fms" % (mirror.average_response_time * 1000) }} -
-
-
-
-{% end block %} diff --git a/src/templates/mirrors/edit.html b/src/templates/mirrors/edit.html index a17aa03d..368e9512 100644 --- a/src/templates/mirrors/edit.html +++ b/src/templates/mirrors/edit.html @@ -1,78 +1,110 @@ -{% extends "../base.html" %} +{% extends "../modal.html" %} -{% block title %}{{ _("Manage mirror %s") % mirror.hostname }}{% end block %} +{% block title %} + {% if mirror %} + {{ _("Edit Mirror") }} - {{ mirror }} + {% else %} + {{ _("Create Mirror") }} + {% end %} +{% end block %} + +{% block breadcrumbs %} + +{% end block %} + +{% block modal_title %} + {% if mirror %} +

{{ _("Edit Mirror") }}

+
{{ mirror }}
+ {% else %} +

{{ _("Create A New Mirror") }}

+ {% end %} +{% end block %} + +{% block modal %} +
+ {% raw xsrf_form_html() %} + + {# Hostname & Path can only be set once #} + {% if not mirror %} + {# Hostname #} +
+ +
+ +
+
-{% block body %} -
-
- + {# Path #} +
+ +
+ +
+
+ {% end %} + + {# Owner #} +
+ +
+ +
-
-
-
-

- {{ _("Manage mirror: %s") % mirror.hostname }} -

+ {# Contact #} +
+ +
+ +
-
-
-
- - {% raw xsrf_form_html() %} -
-
- - - - {{ _("The canonical hostname.") }} - -
-
- -
-
-
- {{ _("Contact information") }} -
- - - - {{ _("The owner of the mirror server.") }} - -
-
- - - - {{ _("An email address to contact an administrator of the mirror.") }} -
- {{ _("This won't be made public.") }} - -
+ {# Notes #} +
+ +
+ +
+
-
- - {{ _("Cancel") }} - + {# Submit! #} +
+ {% if mirror %} + + {% else %} + + {% end %}
-
+ {% end block %} diff --git a/src/templates/mirrors/index.html b/src/templates/mirrors/index.html new file mode 100644 index 00000000..96e82217 --- /dev/null +++ b/src/templates/mirrors/index.html @@ -0,0 +1,37 @@ +{% extends "../base.html" %} + +{% block title %}{{ _("Mirrors") }}{% end block %} + +{% block body %} +
+
+
+ + +

{{ _("Mirrors") }}

+
+
+
+ +
+
+ {% module MirrorsList(mirrors) %} + + {% if current_user and current_user.is_admin() %} +
+ + {{ _("Create Mirror") }} + +
+ {% end %} +
+
+{% end block %} diff --git a/src/templates/mirrors/list.html b/src/templates/mirrors/list.html deleted file mode 100644 index 27e4161a..00000000 --- a/src/templates/mirrors/list.html +++ /dev/null @@ -1,107 +0,0 @@ -{% extends "../base.html" %} - -{% block title %}{{ _("Mirrors") }}{% end block %} - -{% block body %} -
-
- -
-
- -
-
-

{{ _("Mirrors") }}

-
- {% if current_user and current_user.is_admin() %} -
- -
- - {{ _("Add new mirror") }} - -
-
- {% end %} -
- -
-
-

- {{ _("On this page, you will see a list of all mirror servers.") }} -

-
-
- - {% if mirrors %} -
-
-
- - - - - - - - - - {% for mirror in mirrors %} - - - - - - - {% end %} - -
{{ _("Hostname") }} / {{ _("Owner") }}{{ _("Last check") }}
- - {{ mirror.hostname }} - -

- {{ mirror.owner or _("N/A") }} -

- 		- [{{ mirror.country_code }}] - - - {% if mirror.status == "OK" %} - - {{ _("Up") }} - - {% elif mirror.status == "OUTOFSYNC" %} - - {{ _("Out Of Sync") }} - - {% elif mirror.status == "ERROR" %} - - {{ _("Down") }} - - {% else %} - - {{ _("Unknown") }} - - {% end %} - - {% if mirror.last_check %} - {{ format_date(mirror.last_check.timestamp, relative=True) }} - {% else %} - {{ _("N/A") }} - {% end %} -
-
- {% else %} -

- {{ _("There are no mirrors configured, yet.") }} -

-
-
- {% end %} -{% end block %} diff --git a/src/templates/mirrors/modules/list.html b/src/templates/mirrors/modules/list.html new file mode 100644 index 00000000..edccfd4f --- /dev/null +++ b/src/templates/mirrors/modules/list.html @@ -0,0 +1,15 @@ +
+ {% for mirror in mirrors %} +
+
+ + {{ mirror }} + +
+ + {% if mirror.owner %} +
{{ mirror.owner }}
+ {% end %} +
+ {% end %} +
diff --git a/src/templates/mirrors/new.html b/src/templates/mirrors/new.html deleted file mode 100644 index c9446960..00000000 --- a/src/templates/mirrors/new.html +++ /dev/null @@ -1,55 +0,0 @@ -{% extends "../base.html" %} - -{% block title %}{{ _("Add new mirror") }}{% end block %} - -{% block body %} -
-
- -
-
- -
-
-

- {{ _("Add a new mirror") }} -

-
-
- -
-
-
- {% raw xsrf_form_html() %} -
-
- - - - {{ _("Enter the canonical hostname of the mirror.") }} - -
- -
- - - - {{ _("The path to the files on the server.") }} - -
- -
-
-
-
-{% end block %} diff --git a/src/templates/mirrors/show.html b/src/templates/mirrors/show.html new file mode 100644 index 00000000..2ca2106d --- /dev/null +++ b/src/templates/mirrors/show.html @@ -0,0 +1,153 @@ +{% extends "../base.html" %} + +{% block title %}{{ _("Mirrors") }} - {{ mirror }}{% end block %} + +{% block body %} +
+
+
+ + +

{{ mirror }}

+ + {% if mirror.owner %} +

{{ mirror.owner }}

+ {% end %} + +
+
+
+

{{ _("Status") }}

+

+ {% if mirror.last_check_success is True %} + {{ _("Online") }} + {% elif mirror.last_check_success is False %} + {{ _("Offline") }} + {% else %} + {{ _("Pending") }} + {% end %} +

+
+
+ + {# ASN #} + {% if mirror.asn %} +
+
+

{{ _("Autonomous System") }}

+

+ {{ mirror.asn }} +

+
+
+ {% end %} + + {# Country Code #} + {% if mirror.country_code %} +
+
+

{{ _("Location") }}

+

+ {{ mirror.country_code }} +

+
+
+ {% end %} + + {# Last Check #} + {% if mirror.last_check_at %} +
+
+

{{ _("Last Check") }}

+

+ {{ locale.format_date(mirror.last_check_at, shorter=True) }} +

+
+
+ {% end %} + + {# Uptime #} + {% set uptime = mirror.get_uptime_since(datetime.timedelta(days=30)) %} + {% if uptime is not None %} +
+
+

{{ _("Uptime In The Last 30 Days") }}

+

+ {% if uptime >= 0.99 %} + {{ "%.4f%%" % (uptime * 100) }} + {% elif uptime >= 0.90 %} + {{ "%.4f%%" % (uptime * 100) }} + {% else %} + {{ "%.4f%%" % (uptime * 100) }} + {% end %} +

+
+
+ {% end %} +
+
+
+
+ + {% if mirror.has_perm(current_user) %} +
+
+ {# Errors #} + {% if mirror.last_check_success is False %} +
+
+
+

{{ _("Mirror Check Failed") }}

+
+
+ {{ mirror.error }} +
+
+
+ {% end %} + + {# Notes #} + {% if mirror.notes %} +
+
+ {% module Text(mirror.notes) %} +
+
+ {% end %} + +
+
+ {% raw xsrf_form_html() %} +
+ + + + {% if mirror.contact %} + + {{ _("Contact Owner") }} + + {% end %} + + + {{ _("Edit") }} + + + + {{ _("Delete") }} + +
+
+
+ {% end %} +{% end block %} diff --git a/src/web/__init__.py b/src/web/__init__.py index aef1d435..5d244269 100644 --- a/src/web/__init__.py +++ b/src/web/__init__.py @@ -67,6 +67,9 @@ class Application(tornado.web.Application): "JobsList" : jobs.ListModule, "JobsLogStream" : jobs.LogStreamModule, + # Mirrors + "MirrorsList" : mirrors.ListModule, + # Packages "PackageInfo" : packages.InfoModule, "PackageDependencies": packages.DependenciesModule, @@ -187,11 +190,12 @@ class Application(tornado.web.Application): repos.MirrorlistHandler), # Mirrors - (r"/mirrors", mirrors.MirrorListHandler), - (r"/mirror/new", mirrors.MirrorNewHandler), - (r"/mirror/([\w\-\.]+)/delete", mirrors.MirrorDeleteHandler), - (r"/mirror/([\w\-\.]+)/edit", mirrors.MirrorEditHandler), - (r"/mirror/([\w\-\.]+)", mirrors.MirrorDetailHandler), + (r"/mirrors", mirrors.IndexHandler), + (r"/mirrors/create", mirrors.CreateHandler), + (r"/mirrors/([\w\-\.]+)", mirrors.ShowHandler), + (r"/mirrors/([\w\-\.]+)/check", mirrors.CheckHandler), + (r"/mirrors/([\w\-\.]+)/delete", mirrors.DeleteHandler), + (r"/mirrors/([\w\-\.]+)/edit", mirrors.EditHandler), # Keys (r"/keys/([A-Z0-9]+)", keys.DownloadHandler), @@ -217,6 +221,9 @@ class Application(tornado.web.Application): self.backend.run_task(self.backend.builders.sync) self.backend.run_task(self.backend.builders.autoscale) + # Regularly check the mirrors + self.backend.run_periodic_task(300, self.backend.mirrors.check) + ## UI methods def extract_hostname(self, handler, url): diff --git a/src/web/mirrors.py b/src/web/mirrors.py index 602d1b49..7419bbda 100644 --- a/src/web/mirrors.py +++ b/src/web/mirrors.py @@ -3,61 +3,80 @@ import tornado.web from . import base +from . import ui_modules -class MirrorListHandler(base.BaseHandler): +class IndexHandler(base.BaseHandler): def get(self): - self.render("mirrors/list.html", mirrors=self.backend.mirrors) + self.render("mirrors/index.html", mirrors=self.backend.mirrors) -class MirrorDetailHandler(base.BaseHandler): +class ShowHandler(base.BaseHandler): def get(self, hostname): mirror = self.backend.mirrors.get_by_hostname(hostname) if not mirror: - raise tornado.web.HTTPError(404, "Could not find mirror: %s" % hostname) + raise tornado.web.HTTPError(404, "Could not find mirror %s" % hostname) - self.render("mirrors/detail.html", mirror=mirror) + self.render("mirrors/show.html", mirror=mirror) -class MirrorNewHandler(base.BaseHandler): - # XXX everyone can perform this task because of a lacking permission check - +class CheckHandler(base.BaseHandler): @tornado.web.authenticated - def get(self, hostname="", path="", hostname_missing=False, path_invalid=False): - self.render("mirrors/new.html", _hostname=hostname, path=path, - hostname_missing=hostname_missing, path_invalid=path_invalid) + async def post(self, hostname): + mirror = self.backend.mirrors.get_by_hostname(hostname) + if not mirror: + raise tornado.web.HTTPError(404, "Could not find mirror %s" % hostname) - @tornado.web.authenticated - def post(self): - errors = {} + # Check permissions + if not mirror.has_perm(self.current_user): + raise tornado.web.HTTPError(403, "%s has no permission for %s" \ + % (self.current_user, mirror)) - hostname = self.get_argument("name", None) - if not hostname: - errors["hostname_missing"] = True + # check() creates its own transaction + await mirror.check(force=True) - path = self.get_argument("path", "") - if path is None: - errors["path_invalid"] = True + # Redirect back to the mirror + self.redirect("/mirrors/%s" % mirror.hostname) - if errors: - errors.update({ - "hostname" : hostname, - "path" : path, - }) - return self.get(**errors) - # Create mirror +class CreateHandler(base.BaseHandler): + def prepare(self): + # Admin permissions are required + if not self.current_user.is_admin(): + raise tornado.web.HTTPError(403) + + @tornado.web.authenticated + def get(self): + self.render("mirrors/edit.html", mirror=None) + + @tornado.web.authenticated + async def post(self): + # Fetch all values + hostname = self.get_argument("hostname") + path = self.get_argument("path") + owner = self.get_argument("owner") + contact = self.get_argument("contact") + notes = self.get_argument("notes", None) + + # Create the mirror with self.db.transaction(): - mirror = self.backend.mirrors.create(hostname, path, user=self.current_user) + mirror = await self.backend.mirrors.create( + hostname, + path, + owner, + contact, + user=self.current_user, + ) - self.redirect("/mirror/%s" % mirror.hostname) + # Redirect the user back + self.redirect("/mirrors/%s" % mirror.hostname) -class MirrorEditHandler(base.BaseHandler): +class EditHandler(base.BaseHandler): @tornado.web.authenticated def get(self, hostname): mirror = self.backend.mirrors.get_by_hostname(hostname) if not mirror: - raise tornado.web.HTTPError(404, "Could not find mirror: %s" % hostname) + raise tornado.web.HTTPError(404, "Could not find mirror %s" % hostname) # Check permissions if not mirror.has_perm(self.current_user): @@ -69,39 +88,50 @@ class MirrorEditHandler(base.BaseHandler): def post(self, hostname): mirror = self.backend.mirrors.get_by_hostname(hostname) if not mirror: - raise tornado.web.HTTPError(404, "Could not find mirror: %s" % hostname) + raise tornado.web.HTTPError(404, "Could not find mirror %s" % hostname) # Check permissions if not mirror.has_perm(self.current_user): raise tornado.web.HTTPError(403) with self.db.transaction(): - mirror.hostname = self.get_argument("name") - mirror.path = self.get_argument("path", "") - mirror.owner = self.get_argument("owner", None) - mirror.contact = self.get_argument("contact", None) - mirror.supports_https = self.get_argument("supports_https", False) + mirror.owner = self.get_argument("owner") + mirror.contact = self.get_argument("contact") + mirror.notes = self.get_argument("notes", None) - self.redirect("/mirror/%s" % mirror.hostname) + self.redirect("/mirrors/%s" % mirror.hostname) -class MirrorDeleteHandler(base.BaseHandler): +class DeleteHandler(base.BaseHandler): @tornado.web.authenticated def get(self, hostname): mirror = self.backend.mirrors.get_by_hostname(hostname) if not mirror: - raise tornado.web.HTTPError(404, "Could not find mirror: %s" % hostname) + raise tornado.web.HTTPError(404, "Could not find mirror %s" % hostname) + + # Check permissions + if not mirror.has_perm(self.current_user): + raise tornado.web.HTTPError(403) + + self.render("mirrors/delete.html", mirror=mirror) + + @tornado.web.authenticated + def post(self, hostname): + mirror = self.backend.mirrors.get_by_hostname(hostname) + if not mirror: + raise tornado.web.HTTPError(404, "Could not find mirror %s" % hostname) # Check permissions if not mirror.has_perm(self.current_user): raise tornado.web.HTTPError(403) - confirmed = self.get_argument("confirmed", None) - if confirmed: - with self.db.transaction(): - mirror.deleted = True + with self.db.transaction(): + mirror.delete(user=self.current_user) - self.redirect("/mirrors") - return + # Redirect back to all mirrors + self.redirect("/mirrors") - self.render("mirrors/delete.html", mirror=mirror) + +class ListModule(ui_modules.UIModule): + def render(self, mirrors): + return self.render_string("mirrors/modules/list.html", mirrors=mirrors) diff --git a/src/web/repos.py b/src/web/repos.py index a9bae9c7..e00f9c16 100644 --- a/src/web/repos.py +++ b/src/web/repos.py @@ -173,7 +173,7 @@ class MirrorlistHandler(BaseHandler): mirrors = [] # Fetch mirrors - for mirror in self.backend.mirrors.make_mirrorlist(self.current_address): + for mirror in self.backend.mirrors.get_mirrors_for_address(self.current_address): mirrors.append({ "url" : "/".join((mirror.url, repo.path, arch)), "location" : mirror.country_code,