From: Michael Tremer Date: Sun, 9 Feb 2025 13:14:30 +0000 (+0000) Subject: registry: Build a basic read-only OCI registry X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=cb2a6697de2e46f301b861d8145f4431f1ff737e;p=pbs.git registry: Build a basic read-only OCI registry This allows us to easily run any OCI images in Docker/podman/etc. for CI testing without engaging any external storage, etc. Signed-off-by: Michael Tremer --- diff --git a/Makefile.am b/Makefile.am index 8a584848..e1b118c0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -103,6 +103,7 @@ pkgpython_PYTHON = \ src/buildservice/misc.py \ src/buildservice/packages.py \ src/buildservice/ratelimiter.py \ + src/buildservice/registry.py \ src/buildservice/releases.py \ src/buildservice/releasemonitoring.py \ src/buildservice/repos.py \ @@ -135,6 +136,7 @@ web_PYTHON = \ src/web/mirrors.py \ src/web/monitorings.py \ src/web/packages.py \ + src/web/registry.py \ src/web/repos.py \ src/web/search.py \ src/web/sources.py \ diff --git a/src/buildservice/registry.py b/src/buildservice/registry.py new file mode 100644 index 00000000..1b920ff4 --- /dev/null +++ b/src/buildservice/registry.py @@ -0,0 +1,84 @@ +############################################################################### +# # +# Pakfire - The IPFire package management system # +# Copyright (C) 2025 Pakfire development team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +import asyncio +import json +import logging +import tarfile + +# Setup logging +log = logging.getLogger("pbs.registry") + +class OCIImage(object): + """ + Takes an Image object and opens it + """ + def __init__(self, backend, image): + self.backend = backend + self.image = image + + async def open(self): + """ + Opens the image as a tar file + """ + return await asyncio.to_thread(self._open, self.image.path) + + def _open(self, path): + log.debug("Opening %s..." % path) + + # Open the tar file + return tarfile.open(path) + + # Get Manifest + + async def get_manifests(self): + """ + Returns the manifests of index.json in OCI images + """ + with await self.open() as f: + # Open index.json + f = f.extractfile("index.json") + + # Parse the JSON object + index = json.load(f) + + # Return all manifests + return index.get("manifests") + + # Get Blob + + async def get_blob(self, reference): + """ + Returns the blob as a file-like object or None if not found + """ + # Split the reference + type, _, digest = reference.partition(":") + + # Make the path + path = "blobs/%s/%s" % (type, digest) + + # Log action + log.debug("Looking up %s in %s" % (path, self.image)) + + # Open the tar file + f = await self.open() + + # Extract the blob + return f.extractfile(path) diff --git a/src/buildservice/releases.py b/src/buildservice/releases.py index 379e129d..ed955d7b 100644 --- a/src/buildservice/releases.py +++ b/src/buildservice/releases.py @@ -19,6 +19,7 @@ ############################################################################### import datetime +import functools import logging import sqlalchemy @@ -27,6 +28,7 @@ from sqlalchemy import ARRAY, Boolean, DateTime, Integer, Text from . import database from . import images +from . import registry # Setup logging log = logging.getLogger("pbs.releases") @@ -138,22 +140,21 @@ class Release(database.Base, database.BackendMixin, database.SoftDeleteMixin): "Image", back_populates="release", lazy="selectin", ) - @property - def XXXimages(self): - images = self.backend.distros.releases.images._get_images(""" - SELECT - * - FROM - release_images - WHERE - release_id = %s - AND - deleted_at IS NULL - """, self.id, - - # Populate cache - release=self, - ) - - # Return grouped by architecture - return misc.group(images, lambda image: image.arch) + # OCI Images + + @functools.cached_property + def oci_images(self): + """ + Returns all OCI images + """ + images = [] + + for image in self.images: + if not image.type == "oci": + continue + + # Make it an OCI image + image = registry.OCIImage(self.backend, image) + images.append(image) + + return images diff --git a/src/web/__init__.py b/src/web/__init__.py index 3f976c67..0d7d2b72 100644 --- a/src/web/__init__.py +++ b/src/web/__init__.py @@ -20,6 +20,7 @@ from . import jobs from . import mirrors from . import monitorings from . import packages +from . import registry from . import repos from . import search from . import sources @@ -195,6 +196,26 @@ class Application(tornado.web.Application): uploads.APIv1DetailHandler), ], default_handler_class=errors.Error404Handler, **settings) + # Add the registry + self.add_handlers("registry.pakfire.ipfire.org", [ + # Redirect anyone who is lost + (r"/", tornado.web.RedirectHandler, { "url" : "https://pakfire.ipfire.org/" }), + + # Version 2 + (r"/v2/", registry.IndexHandler), + + # Manifests + (r"/v2/([a-z0-9]+(?:(?:\.|_|__|-+)[a-z0-9]+)*(?:\/[a-z0-9]+(?:(?:\.|_|__|-+)[a-z0-9]+)*)*)/manifests/([a-zA-Z0-9_][a-zA-Z0-9._-]{0,127})", + registry.ManifestHandler), + + # Blobs & Manifests referenced by digest + (r"/v2/([a-z0-9]+(?:(?:\.|_|__|-+)[a-z0-9]+)*(?:\/[a-z0-9]+(?:(?:\.|_|__|-+)[a-z0-9]+)*)*)/(blobs|manifests)/(sha256:[a-f0-9]{64})", + registry.BlobOrManifestHandler), + + # Catch anything else + (r"/v2/.*", registry.NotFoundHandler), + ]) + # Launch backend & background tasks self.backend = Backend("/etc/pakfire/pbs.conf") self.backend.launch_background_tasks() diff --git a/src/web/base.py b/src/web/base.py index d064671f..62f07573 100644 --- a/src/web/base.py +++ b/src/web/base.py @@ -230,6 +230,20 @@ class BaseHandler(tornado.web.RequestHandler): """ return self.request.headers.get("User-Agent", None) + def client_accepts(self, type): + """ + Checks if type is in Accept-Encoding: header + """ + # Fetch the header + header = self.request.headers.get("Accept", "*/*") + + # Assume everything is accepted + if header == "*/*": + return True + + # Check if the type is accepted + return type in re.split(r",\s*", header) + def client_accepts_encoding(self, encoding): """ Checks if type is in Accept-Encoding: header diff --git a/src/web/registry.py b/src/web/registry.py new file mode 100644 index 00000000..3f0f3200 --- /dev/null +++ b/src/web/registry.py @@ -0,0 +1,161 @@ +############################################################################### +# # +# Pakfire - The IPFire package management system # +# Copyright (C) 2025 Pakfire development team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +# This file implements the OCI Registry API +# https://github.com/opencontainers/distribution-spec/blob/main/spec.md#api + +import itertools +import json +import tornado.web + +from . import base + +class BaseHandler(base.BaseHandler): + """ + A base handler for any registry stuff + """ + async def write_error(self, *args, **kwargs): + pass # Don't send any body + + +class NotFoundHandler(BaseHandler): + """ + A custom 404 error which does not any body. + """ + def prepare(self): + raise tornado.web.HTTPError(404) + + +class IndexHandler(BaseHandler): + """ + This simply has to return 200 OK + """ + def get(self): + pass + + +class ManifestHandler(BaseHandler): + async def head(self, *args, **kwargs): + return await self.get(*args, **kwargs, send_body=False) + + async def get(self, distro_slug, release_slug, send_body=True): + # Fetch the distribution + distro = await self.backend.distros.get_by_slug(distro_slug) + if not distro: + raise tornado.web.HTTPError(404) + + # Fetch the release + if release_slug == "latest": + release = await distro.get_latest_release() + else: + release = await distro.get_release(release_slug) + + # Fail if we could not find a release + if not release: + raise tornado.web.HTTPError(404) + + # Check if the client supports the index format + if not self.client_accepts("application/vnd.oci.image.index.v1+json"): + raise tornado.web.HTTPError(406, "Client does not support OCI index format") + + # Fail if there are no OCI images + if not release.oci_images: + raise tornado.web.HTTPError(404, "Release has no OCI images") + + manifests = [] + + # Collect all manifests + for image in release.oci_images: + manifests += await image.get_manifests() + + # Serialise the JSON (because self.finish() resets Content-Type) + index = json.dumps({ + "schemaVersion" : 2, + "mediaType" : "application/vnd.oci.image.index.v1+json", + "manifests" : manifests, + }) + + # Set Content-* headers + self.set_header("Content-Type", "application/vnd.oci.image.index.v1+json") + self.set_header("Content-Length", len(index)) + + # Send the response + if send_body: + self.finish(index) + + +class BlobOrManifestHandler(BaseHandler): + async def head(self, *args, **kwargs): + return await self.get(*args, **kwargs, send_body=False) + + async def get(self, distro_slug, type, reference, send_body=True): + # Fetch the distribution + distro = await self.backend.distros.get_by_slug(distro_slug) + if not distro: + raise tornado.web.HTTPError(404) + + # Fetch all releases + releases = await distro.get_releases() + + # This is a super naive approach because we don't have an index + # for the files. Considering how rarely we are running through this code, + # it might work out perfectly fine for us to just walk through all images + # until we find the correct file. As we are starting with the latest release, + # chances should be high that we don't have to iterate through all of this + # for too long. + + for release in releases: + for image in release.oci_images: + blob = await image.get_blob(reference) + + # Continue if there was no match + if blob is None: + continue + + # Send the blob! + return await self._send_blob(blob, type, send_body=send_body) + + # If we get here, we didn't find anything + raise tornado.web.HTTPError(404) + + async def _send_blob(self, blob, type, send_body=True): + """ + Sends the blob to the client + """ + # Set Content-Type + if type == "manifests": + self.set_header("Content-Type", "application/vnd.oci.image.manifest.v1+json") + else: + self.set_header("Content-Type", "application/octet-stream") + + # It would be nice to set Content-Length here, but there is no way to + # determine the length of the file without reading the whole thing first. + + # Done if we should not send the body + if not send_body: + return + + # Send the file + while True: + chunk = blob.read(1024 * 1024) + if not chunk: + break + + self.write(chunk)