# #
###############################################################################
+import asyncio
import json
import logging
import urllib.parse
# Setup logging
log = logging.getLogger("pbs.releasemonitoring")
-class ReleaseMonitoring(base.Object):
+# Only send up to four simultaneous API requests
+ratelimiter = asyncio.Semaphore(value=4)
+
+BUG_DESCRIPTION = """\
+A new version of %(name)s has been released: %(version)s\
+"""
+
+BUG_DESCRIPTION_WITH_BUILD = BUG_DESCRIPTION + """\n
+XXX PUT STUFF ABOUT THE BUILD HERE
+"""
+
+class ReleaseExistsError(Exception):
+ """
+ Raised if a release already exists
+ """
+ pass
+
+
+class BuildExistsError(Exception):
+ """
+ Raised if the build or a newer one already exists
+ """
+ pass
+
+
+class Monitorings(base.Object):
baseurl = "https://release-monitoring.org"
@property
return self.settings.get("release-monitoring-api-key")
async def _request(self, method, url, data=None):
+ body = {}
+
# Authenticate to the API
headers = {
"Authorization" : "Token %s" % self.api_key,
# Decode JSON response
if res.body:
- return json.loads(res.body)
+ body = json.loads(res.body)
- # Return an empty response
- return {}
+ # Check if we have received an error
+ error = body.get("error")
- # Packages
+ # Raise the error
+ if error:
+ raise RuntimeError(error)
- def _get_package(self, query, *args):
+ return body
+
+ def _get_monitoring(self, query, *args):
res = self.db.get(query, *args)
if res:
- return Package(self.backend, id=res.id, data=res)
+ return Monitoring(self.backend, res.id, res)
+
+ def _get_monitorings(self, query, *args):
+ res = self.db.query(query, *args)
+
+ for row in res:
+ yield Monitoring(self.backend, row.id, row)
- def get_package(self, distro, name):
- return self._get_package("""
+ def get_by_id(self, id):
+ return self._get_monitoring("""
SELECT
*
FROM
- release_monitoring_packages
+ release_monitorings
WHERE
- deleted_at IS NULL
- AND
- distro = %s
- AND
- name = %s
- """, distro, name,
+ id = %s
+ """, id,
)
- def create_package(self, distro, name):
- pass
+ async def create(self, distro, name, created_by, project_id,
+ follow="mainline", create_builds=True):
+ monitoring = self._get_monitoring("""
+ INSERT INTO
+ release_monitorings
+ (
+ distro_id,
+ name,
+ created_by,
+ project_id,
+ follow,
+ create_builds
+ )
+ VALUES(
+ %s, %s, %s, %s, %s, %s
+ )
+ RETURNING
+ *
+ """, distro, name, created_by, project_id, follow,
+ )
+
+ # Schedule the first check in the background
+ self.backend.run_task(monitoring.check)
+
+ return monitoring
async def search(self, name):
"""
# Return all packages
return [database.Row(item) for item in response.get("items")]
+ async def check(self, limit=None):
+ """
+ Perform checks on monitorings
+ """
+ # Fetch all monitorings that were never checked or checked longer than 24 hours ago
+ monitorings = self._get_monitorings("""
+ SELECT
+ *
+ FROM
+ release_monitorings
+ WHERE
+ deleted_at IS NULL
+ AND
+ (
+ last_check_at IS NULL
+ OR
+ last_check_at <= CURRENT_TIMESTAMP - INTERVAL '24 hours'
+ )
+ ORDER BY
+ last_check_at ASC NULLS FIRST
+ LIMIT
+ %s
+ """, limit,
+ )
+
+ # Perform all checks concurrently
+ async with asyncio.TaskGroup() as tg:
+ for monitoring in monitorings:
+ tg.create_task(monitoring.check())
-class Package(base.DataObject):
- table = "release_monitoring_packages"
+
+class Monitoring(base.DataObject):
+ table = "release_monitorings"
+
+ def __str__(self):
+ return "%s - %s" % (self.distro, self.name)
@property
def distro(self):
- return self.data.distro
+ """
+ The distribution
+ """
+ return self.backend.distros.get_by_id(self.data.distro_id)
@property
def name(self):
+ """
+ The package name
+ """
return self.data.name
+ @property
+ def project_id(self):
+ return self.data.project_id
+
+ @property
+ def follow(self):
+ return self.data.follow
+
+ @property
+ def create_builds(self):
+ return self.data.create_builds
+
+ # Check
+
+ async def check(self):
+ # Wait until we are allowed to send an API request
+ async with ratelimiter:
+ log.info("Checking for new releases for %s" % self)
+
+ # Fetch the current versions
+ versions = await self._fetch_versions()
+
+ with self.db.transaction():
+ try:
+ if self.follow == "mainline":
+ await self._follow_mainline(versions)
+ else:
+ raise ValueError("Cannot handle follow: %s" % self.follow)
+
+ # If the release exists, do nothing
+ except ReleaseExistsError as e:
+ log.debug("Release %s already exists" % e)
+
+ async def _fetch_versions(self):
+ """
+ Fetches all versions for this project
+ """
+ response = await self.backend.monitorings._request(
+ "GET", "/api/v2/versions/", {
+ "project_id" : self.project_id,
+ },
+ )
+
+ # Parse the response as JSON and return it
+ return database.Row(response)
+
+ async def _follow_mainline(self, versions):
+ """
+ This will follow "mainline" i.e. the latest version
+ """
+ return await self.create_release(versions.latest_version)
+
# Releases
def _get_releases(self, query, *args):
- res = self.db.query(query, *args)
-
- for row in res:
- yield Release(self.backend, id=row.id, data=row)
+ return self.db.fetch_many(Release, query, *args, monitoring=self)
def _get_release(self, query, *args):
- res = self.db.get(query, *args)
-
- if res:
- return Release(self.backend, id=res.id, data=res)
+ return self.db.fetch_one(Release, query, *args, monitoring=self)
@property
def latest_release(self):
WHERE
deleted_at IS NULL
AND
- package_id = %s
+ monitoring_id = %s
ORDER BY
created_at DESC
LIMIT 1
""", self.id,
)
- @property
+ @lazy_property
def releases(self):
releases = self._get_releases("""
SELECT
FROM
release_monitoring_releases
WHERE
- deleted_at IS NULL
- AND
- package_id = %s
+ monitoring_id = %s
ORDER BY
created_at DESC
""", self.id,
return list(releases)
- def _create_release(self, version):
+ def _release_exists(self, version):
+ """
+ Returns True if this version already exists
+ """
+ return version in [release.version for release in self.releases]
+
+ async def create_release(self, version):
"""
Creates a new release for this package
"""
# XXX Do we need to check whether we are going backwards?
+ # Raise an error if the release already exists
+ if self._release_exists(version):
+ raise ReleaseExistsError(version)
+
+ # Raise an error if we already have a newer build
+ elif self._build_exists(version):
+ raise BuildExistsError(version)
+
+ log.info("%s: Creating new release %s" % (self, version))
+
release = self._get_release("""
INSERT INTO
release_monitoring_releases
(
- package_id,
+ monitoring_id,
version
)
VALUES
(
%s, %s
)
- ON CONFLICT
- (package_id, version)
- WHERE
- deleted_at IS NULL
- DO
- NOTHING
RETURNING
*
""", self.id, version,
)
+ # Add the release to cache
+ self.releases.append(release)
+
+ # Create a bug report
+ await release._create_bug()
+
+ # Create a build
+ if self.data.create_builds:
+ await release._create_build()
+
# Return the release
return release
- # Update
+ # Builds
- async def _fetch(self):
+ def _build_exists(self, version):
"""
- Fetches all sorts of information about a package
+ Returns True if a build with this version already exists
"""
- response = await self.backend.releasemonitoring._request(
- "GET", "/api/v2/packages/", {
- "distribution" : self.distro,
- "name" : self.name,
- },
- )
+ # XXX needs to check if we already have a newer version
- # Fetch all received items
- items = response.get("items")
+ return version in [build.package.version for build in self.builds]
- # Is the package known?
- if not items:
- # Automatically try to create the package
- if await self._auto_create_package():
- return await self._fetch()
+ @property
+ def builds(self):
+ return self.distro.get_builds_by_name(self.name)
- raise PackageNotFoundError(self.name)
+ @lazy_property
+ def latest_build(self):
+ builds = self.distro.get_builds_by_name(self.name, limit=1)
- # Did we receive more than one item?
- elif len(items) > 1:
- raise RuntimeError("More than one item received for %s" % name)
+ for build in builds:
+ return build
- for item in items:
- return database.Row({
- "version" : item.get("version"),
- "stable_version" : item.get("stable_version"),
- })
- async def _auto_create_package(self):
- """
- Tries to automatically create a package
- """
- packages = await self.backend.releasemonitoring.search(self.name)
+class Release(base.DataObject):
+ table = "release_monitoring_releases"
- # Use the first exact match
- for package in packages:
- if not package.name == self.name:
- continue
+ def __str__(self):
+ return "%s %s" % (self.monitoring.name, self.version)
- print(package)
+ # Monitoring
- # Automatically create a mapping
- await self.update_mapping(
- project_name=package.name,
- project_ecosystem=package.ecosystem,
- )
-
- return True
+ @lazy_property
+ def monitoring(self):
+ return self.backend.monitorings.get_by_id(self.data.monitoring_id)
- # We could not create a mapping
- return False
+ # Version
- async def update_mapping(self, project_name=None, project_ecosystem=None):
- if project_name is None:
- project_name = self.name
+ @property
+ def version(self):
+ return self.data.version
- # Send request
- await self.backend.releasemonitoring._request("POST", "/api/v2/packages/",
- {
- "distribution" : self.distro,
- "package_name" : self.name,
- "project_name" : project_name,
- "project_ecosystem" : project_ecosystem,
- },
- )
+ # Bug
- async def update(self):
+ async def _create_bug(self):
"""
- Update the information we have about this package
+ Creates a new bug report about this release
"""
- # Pull information from the API
- package = await self._fetch()
+ args = {
+ # Product, Version & Component
+ "product" : self.monitoring.distro.bugzilla_product,
+ "version" : self.monitoring.distro.bugzilla_version,
+ "component" : self.monitoring.name,
+
+ # Summary & Description
+ "summary" : "%s has been released" % self,
+ "description" : BUG_DESCRIPTION % \
+ {
+ "name" : self.monitoring.name,
+ "version" : self.version
+ },
+
+ # Keywords
+ "keywords" : [
+ # Mark this bug as created automatically
+ "Monitoring",
+
+ # Mark this bug as a new release
+ "NewRelease",
+ ],
+ }
- # Try to create a new release
- release = self._create_release(package.stable_version)
+ # If we have a build, include it in the bug description
+ if self.build:
+ args |= {
+ "description" : BUG_DESCRIPTION_WITH_BUILD % \
+ {
+ "name" : self.monitoring.name,
+ "version" : self.version
+ },
+ }
- # Return the new release
- return release
+ # Create the bug
+ bug = await self.backend.bugzilla.create_bug(**args)
+ # Store the bug ID
+ self._set_attribute("bug_id", bug.id)
-class Release(base.DataObject):
- table = "release_monitoring_releases"
+ return bug
- def __str__(self):
- return self.version
+ # Build
- # Version
+ def get_build(self):
+ if self.data.build_id:
+ return self.backend.builds.get_by_id(self.data.build_id)
- @property
- def version(self):
- return self.data.version
+ def set_build(self, build):
+ if self.build:
+ raise AttributeError("Cannot reset build")
- # Builds
+ self._set_attribute("build_id", build)
- @lazy_property
- def builds(self):
- builds = self.backend.builds._get_builds("""
- SELECT
- builds.*
- FROM
- release_monitoring_release_builds
- LEFT JOIN
- builds ON release_monitoring_release_builds.build_id = builds.id
- WHERE
- release_monitoring_release_builds.release_id = %s
- AND
- builds.deleted_at IS NULL
- """, self.id,
- )
+ build = lazy_property(get_build, set_build)
+
+ async def _create_build(self):
+ """
+ Creates a build
+ """
+ if self.build:
+ raise RuntimeError("Build already exists")
+
+ # XXX since we cannot yet update any builds, we will simply clone the latest one
+ # to test the tooling
+ #print(self.monitoring.latest_build)
- # Return builds by distribution
- return { build.distro : build for build in builds }
+ # XXX does this need an owner?
+ #self.build = await self.monitoring.latest_build.create(
+ # repo=None, package=self.monitoring.latest_build.package)
--
--- Name: release_monitoring_packages; Type: TABLE; Schema: public; Owner: -
+-- Name: release_monitoring_releases; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE public.release_monitoring_packages (
+CREATE TABLE public.release_monitoring_releases (
id integer NOT NULL,
- distro text NOT NULL,
- name text NOT NULL,
+ monitoring_id integer NOT NULL,
+ version text NOT NULL,
created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
- deleted_at timestamp without time zone,
- auto_update boolean DEFAULT true NOT NULL
+ bug_id integer,
+ build_id integer
);
--
--- Name: release_monitoring_packages_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+-- Name: release_monitoring_releases_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE public.release_monitoring_packages_id_seq
+CREATE SEQUENCE public.release_monitoring_releases_id_seq
AS integer
START WITH 1
INCREMENT BY 1
--
--- Name: release_monitoring_packages_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+-- Name: release_monitoring_releases_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE public.release_monitoring_packages_id_seq OWNED BY public.release_monitoring_packages.id;
+ALTER SEQUENCE public.release_monitoring_releases_id_seq OWNED BY public.release_monitoring_releases.id;
--
--- Name: release_monitoring_release_builds; Type: TABLE; Schema: public; Owner: -
+-- Name: release_monitorings; Type: TABLE; Schema: public; Owner: -
--
-CREATE TABLE public.release_monitoring_release_builds (
+CREATE TABLE public.release_monitorings (
id integer NOT NULL,
- release_id integer NOT NULL,
distro_id integer NOT NULL,
- build_id integer NOT NULL
-);
-
-
---
--- Name: release_monitoring_releases; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.release_monitoring_releases (
- id integer NOT NULL,
- package_id integer NOT NULL,
- version text NOT NULL,
+ name text NOT NULL,
created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
- deleted_at timestamp without time zone
+ created_by integer NOT NULL,
+ deleted_at timestamp without time zone,
+ deleted_by integer,
+ project_id integer NOT NULL,
+ follow text NOT NULL,
+ last_check_at timestamp without time zone,
+ create_builds boolean DEFAULT true NOT NULL
);
--
--- Name: release_monitoring_releases_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+-- Name: release_monitorings_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
-CREATE SEQUENCE public.release_monitoring_releases_id_seq
+CREATE SEQUENCE public.release_monitorings_id_seq
AS integer
START WITH 1
INCREMENT BY 1
--
--- Name: release_monitoring_releases_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+-- Name: release_monitorings_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
-ALTER SEQUENCE public.release_monitoring_releases_id_seq OWNED BY public.release_monitoring_releases.id;
+ALTER SEQUENCE public.release_monitorings_id_seq OWNED BY public.release_monitorings.id;
--
--
--- Name: release_monitoring_packages id; Type: DEFAULT; Schema: public; Owner: -
+-- Name: release_monitoring_releases id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY public.release_monitoring_packages ALTER COLUMN id SET DEFAULT nextval('public.release_monitoring_packages_id_seq'::regclass);
+ALTER TABLE ONLY public.release_monitoring_releases ALTER COLUMN id SET DEFAULT nextval('public.release_monitoring_releases_id_seq'::regclass);
--
--- Name: release_monitoring_releases id; Type: DEFAULT; Schema: public; Owner: -
+-- Name: release_monitorings id; Type: DEFAULT; Schema: public; Owner: -
--
-ALTER TABLE ONLY public.release_monitoring_releases ALTER COLUMN id SET DEFAULT nextval('public.release_monitoring_releases_id_seq'::regclass);
+ALTER TABLE ONLY public.release_monitorings ALTER COLUMN id SET DEFAULT nextval('public.release_monitorings_id_seq'::regclass);
--
--
--- Name: release_monitoring_packages release_monitoring_packages_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.release_monitoring_packages
- ADD CONSTRAINT release_monitoring_packages_pkey PRIMARY KEY (id);
-
-
---
--- Name: release_monitoring_release_builds release_monitoring_release_builds_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+-- Name: release_monitoring_releases release_monitoring_releases_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY public.release_monitoring_release_builds
- ADD CONSTRAINT release_monitoring_release_builds_pkey PRIMARY KEY (id);
+ALTER TABLE ONLY public.release_monitoring_releases
+ ADD CONSTRAINT release_monitoring_releases_pkey PRIMARY KEY (id);
--
--- Name: release_monitoring_releases release_monitoring_releases_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+-- Name: release_monitorings release_monitorings_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY public.release_monitoring_releases
- ADD CONSTRAINT release_monitoring_releases_pkey PRIMARY KEY (id);
+ALTER TABLE ONLY public.release_monitorings
+ ADD CONSTRAINT release_monitorings_pkey PRIMARY KEY (id);
--
--
--- Name: release_monitoring_packages_unique; Type: INDEX; Schema: public; Owner: -
+-- Name: release_monitoring_releases_search; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX release_monitoring_packages_unique ON public.release_monitoring_packages USING btree (distro, name) WHERE (deleted_at IS NULL);
+CREATE INDEX release_monitoring_releases_search ON public.release_monitoring_releases USING btree (monitoring_id, created_at);
--
--- Name: release_monitoring_release_builds_release_id; Type: INDEX; Schema: public; Owner: -
+-- Name: release_monitoring_releases_unique; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX release_monitoring_release_builds_release_id ON public.release_monitoring_release_builds USING btree (release_id);
+CREATE UNIQUE INDEX release_monitoring_releases_unique ON public.release_monitoring_releases USING btree (monitoring_id, version);
--
--- Name: release_monitoring_releases_unique; Type: INDEX; Schema: public; Owner: -
+-- Name: release_monitorings_unique; Type: INDEX; Schema: public; Owner: -
--
-CREATE UNIQUE INDEX release_monitoring_releases_unique ON public.release_monitoring_releases USING btree (package_id, version) WHERE (deleted_at IS NULL);
+CREATE UNIQUE INDEX release_monitorings_unique ON public.release_monitorings USING btree (distro_id, name) WHERE (deleted_at IS NULL);
--
--
--- Name: release_monitoring_release_builds release_monitoring_release_builds_build_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+-- Name: release_monitoring_releases release_monitoring_releases_monitoring_id; Type: FK CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY public.release_monitoring_release_builds
- ADD CONSTRAINT release_monitoring_release_builds_build_id FOREIGN KEY (build_id) REFERENCES public.builds(id);
+ALTER TABLE ONLY public.release_monitoring_releases
+ ADD CONSTRAINT release_monitoring_releases_monitoring_id FOREIGN KEY (monitoring_id) REFERENCES public.release_monitorings(id);
--
--- Name: release_monitoring_release_builds release_monitoring_release_builds_distro_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+-- Name: release_monitorings release_monitorings_created_by; Type: FK CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY public.release_monitoring_release_builds
- ADD CONSTRAINT release_monitoring_release_builds_distro_id FOREIGN KEY (distro_id) REFERENCES public.distributions(id);
+ALTER TABLE ONLY public.release_monitorings
+ ADD CONSTRAINT release_monitorings_created_by FOREIGN KEY (created_by) REFERENCES public.users(id);
--
--- Name: release_monitoring_release_builds release_monitoring_release_builds_release_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+-- Name: release_monitorings release_monitorings_deleted_by; Type: FK CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY public.release_monitoring_release_builds
- ADD CONSTRAINT release_monitoring_release_builds_release_id FOREIGN KEY (release_id) REFERENCES public.release_monitoring_releases(id);
+ALTER TABLE ONLY public.release_monitorings
+ ADD CONSTRAINT release_monitorings_deleted_by FOREIGN KEY (deleted_by) REFERENCES public.users(id);
--
--- Name: release_monitoring_releases release_monitoring_releases_package_id; Type: FK CONSTRAINT; Schema: public; Owner: -
+-- Name: release_monitorings release_monitorings_distro_id; Type: FK CONSTRAINT; Schema: public; Owner: -
--
-ALTER TABLE ONLY public.release_monitoring_releases
- ADD CONSTRAINT release_monitoring_releases_package_id FOREIGN KEY (package_id) REFERENCES public.release_monitoring_packages(id);
+ALTER TABLE ONLY public.release_monitorings
+ ADD CONSTRAINT release_monitorings_distro_id FOREIGN KEY (distro_id) REFERENCES public.distributions(id);
--
projects / pbs.git / commitdiff
