dist_bin_SCRIPTS = \
src/scripts/pakfire-build-service \
- src/scripts/pakfire-hub \
src/scripts/pakfire-web
dist_configs_DATA = \
CLEANFILES += \
src/buildservice/constants.py
-hub_PYTHON = \
- src/hub/__init__.py \
- src/hub/builds.py \
- src/hub/handlers.py \
- src/hub/jobs.py \
- src/hub/queue.py \
- src/hub/uploads.py
-
-hubdir = $(buildservicedir)/hub
-
web_PYTHON = \
src/web/__init__.py \
src/web/auth.py \
src/web/repos.py \
src/web/search.py \
src/web/ui_modules.py \
+ src/web/uploads.py \
src/web/users.py
webdir = $(buildservicedir)/web
if HAVE_SYSTEMD
systemdsystemunit_DATA = \
- src/systemd/pakfire-hub.service \
src/systemd/pakfire-web.service
CLEANFILES += \
endif
EXTRA_DIST += \
- src/systemd/pakfire-hub.service.in \
src/systemd/pakfire-web.service.in
dist_database_DATA = \
src/buildservice/sources.py
src/buildservice/uploads.py
src/buildservice/users.py
-src/hub/__init__.py
-src/hub/builds.py
-src/hub/handlers.py
-src/hub/jobs.py
-src/hub/queue.py
-src/hub/uploads.py
src/static/robots.txt
src/templates/base.html
src/templates/bugs/modules/list.html
online_until = property(lambda s: s.data.online_until, set_online_until)
- def update_info(self, cpu_model=None, cpu_count=None, cpu_arch=None,
- pakfire_version=None, os_name=None):
+ def log_stats(self, cpu_model=None, cpu_count=None, cpu_arch=None, pakfire_version=None,
+ os_name=None, cpu_user=None, cpu_nice=None, cpu_system=None, cpu_idle=None,
+ cpu_iowait=None, cpu_irq=None, cpu_softirq=None, cpu_steal=None, cpu_guest=None,
+ cpu_guest_nice=None, loadavg1=None, loadavg5=None, loadavg15=None, mem_total=None,
+ mem_available=None, mem_used=None, mem_free=None, mem_active=None, mem_inactive=None,
+ mem_buffers=None, mem_cached=None, mem_shared=None, swap_total=None, swap_used=None,
+ swap_free=None, **kwargs):
+ """
+ Logs some stats about this builder
+ """
+ # Update information
self.db.execute("""
UPDATE
builders
self.id,
)
- def log_stats(self, cpu_user, cpu_nice, cpu_system, cpu_idle, cpu_iowait,
- cpu_irq, cpu_softirq, cpu_steal, cpu_guest, cpu_guest_nice,
- loadavg1, loadavg5, loadavg15, mem_total, mem_available, mem_used,
- mem_free, mem_active, mem_inactive, mem_buffers, mem_cached, mem_shared,
- swap_total, swap_used, swap_free):
- """
- Logs some stats about this builder
- """
+ # Log Stats
self.db.execute("""
INSERT INTO
builder_stats
log = logging.getLogger("pakfire.buildservice.jobqueue")
class JobQueue(base.Object):
+ # A list of all builders that have a connection
+ connections = []
+
def __iter__(self):
jobs = self.backend.jobs._get_jobs("SELECT jobs.* FROM job_queue queue \
LEFT JOIN jobs ON queue.job_id = jobs.id")
builder.supported_arches,
)
+ async def open(self, builder, connection):
+ """
+ Called when a builder opens a connection
+ """
+ log.debug("Connection opened by %s" % builder)
+
+ # Find any previous connections of this builder and close them
+ for c in self.connections:
+ if not c.builder == builder:
+ continue
+
+ log.warning("Closing connection to builder %s because it is being replaced" % builder)
+
+ # Close the previous connection
+ c.close(code=1000, reason="Replaced by a new connection")
+
+ # Add this connection to the list
+ self.connections.append(connection)
+
+ # Dispatch any jobs immediately
+ await self.dispatch_jobs()
+
+ def close(self, builder, connection):
+ log.debug("Connection to %s closed" % builder)
+
+ # Remove the connection
+ try:
+ self.connections.remove(connection)
+ except IndexError:
+ pass
+
+ async def dispatch_jobs(self):
+ """
+ Will be called regularly and will dispatch any pending jobs to any
+ available builders
+ """
+ log.debug("Dispatching jobs...")
+
+ # Exit if there are no builders connected
+ if not self.connections:
+ log.debug(" No connections open")
+ return
+
+ # Map all connections by builder
+ builders = { c.builder : c for c in self.connections }
+
+ # Process all builders and assign jobs
+ # We prioritize builders with fewer jobs
+ for builder in sorted(builders, key=lambda b: len(b.jobs)):
+ log.debug(" Processing builder %s" % builder)
+
+ # Find the connection
+ connection = builders[builder]
+
+ with self.backend.db.transaction():
+ if not builder.is_ready():
+ log.debug(" Builder %s is not ready" % builder)
+ continue
+
+ # We are ready for a new job
+ job = self.pop(builder)
+ if job:
+ connection.assign_job(job)
+ continue
+
+ log.debug(" No jobs processable for %s" % builder)
+
+ # If there is no job for the builder, we might as well shut it down
+ await builder.stop()
import shutil
from . import base
+from . import builders
from . import users
from .constants import *
from .decorators import *
""", uuid,
)
- def create(self, filename, size, builder=None, user=None):
+ def create(self, filename, size, uploader=None):
+ builder = None
+ user = None
+
+ # Check uploader type
+ if isinstance(uploader, builders.Builder):
+ builder = uploader
+ elif isinstance(uploader, users.User):
+ user = uploader
+
# Check quota for users
if user:
# This will raise an exception if the quota has been exceeded
+++ /dev/null
-#!/usr/bin/python3
-
-import logging
-import tornado.web
-
-from .. import Backend
-from . import builds
-from . import handlers
-from . import jobs
-from . import queue
-from . import uploads
-
-class Application(tornado.web.Application):
- def __init__(self, **settings):
- tornado.web.Application.__init__(self, [
- # Redirect stranded users
- (r"/", tornado.web.RedirectHandler, { "url" : "https://pakfire.ipfire.org/" }),
-
- # Builds
- (r"/builds", builds.CreateHandler),
- (r"/builds/(.*)", handlers.BuildsGetHandler),
-
- # Builders
- (r"/builders/info", handlers.BuildersInfoHandler),
- (r"/builders/stats", handlers.BuildersStatsHandler),
-
- # Jobs
- (r"/jobs/([0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12})/builder",
- jobs.BuilderHandler),
- (r"/jobs/([0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12})/finished",
- jobs.FinishedHandler),
-
- (r"/jobs/active", handlers.JobsGetActiveHandler),
- (r"/jobs/latest", handlers.JobsGetLatestHandler),
- (r"/jobs/queue", handlers.JobsGetQueueHandler),
- (r"/jobs/(.*)", handlers.JobsGetHandler),
-
- # Packages
- (r"/packages/(.*)", handlers.PackagesGetHandler),
-
- # Queue
- (r"/queue", queue.QueueHandler),
-
- # Test
- (r"/test", handlers.TestHandler),
-
- # Uploads
- (r"/uploads", uploads.IndexHandler),
- ],
-
- # Forward any other settings
- **settings,
- )
-
- # Launch backend
- self.backend = Backend("/etc/pakfire/pbs.conf")
-
- logging.info("Successfully initialied application")
-
- # Perform some initial tasks
- self.backend.run_task(self.backend.builders.sync)
- self.backend.run_task(self.backend.builders.autoscale)
+++ /dev/null
-#!/usr/bin/python3
-###############################################################################
-# #
-# Pakfire - The IPFire package management system #
-# Copyright (C) 2011 Pakfire development team #
-# #
-# This program is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published by #
-# the Free Software Foundation, either version 3 of the License, or #
-# (at your option) any later version. #
-# #
-# This program is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with this program. If not, see <http://www.gnu.org/licenses/>. #
-# #
-###############################################################################
-
-import tornado.web
-
-from ..errors import NoSuchDistroError
-
-from .handlers import BaseHandler
-from . import queue
-
-class CreateHandler(BaseHandler):
- async def post(self):
- # Fetch the upload
- upload = self.get_argument_upload("upload_id")
- if not upload:
- raise tornado.web.HTTPError(404, "Could not find upload")
-
- # Check permissions of the upload
- if not upload.has_perm(self.current_user):
- raise tornado.web.HTTPError(403, "No permission for using upload %s" % upload)
-
- # Fetch the repository
- repo_name = self.get_argument("repo", None)
-
- with self.db.transaction():
- # Import the package
- try:
- package = await self.backend.packages.create(upload)
-
- # If the distribution that is coded into the package could not be found,
- # we will send that error to the user...
- except NoSuchDistroError as e:
- raise tornado.web.HTTPError(404, "Could not find distribution: %s" % e)
-
- # Find the repository
- repo = self.current_user.get_repo(package.distro, repo_name)
- if not repo:
- raise tornado.web.HTTPError(404, "Could not find repository")
-
- # Create a new build
- build = await self.backend.builds.create(repo, package, owner=self.user)
-
- # Delete the upload
- await upload.delete()
-
- # Send some data about the build
- self.finish({
- "uuid" : build.uuid,
- "name" : "%s" % build,
- })
-
- # Run dependency check on all jobs
- await self.backend.jobs.depcheck(build.jobs)
-
- # Try to dispatch jobs
- await queue.dispatch_jobs(self.backend)
+++ /dev/null
-#!/usr/bin/python
-
-import json
-import logging
-import tornado.web
-
-from .. import builds
-from .. import builders
-from .. import users
-
-from ..web.auth import KerberosAuthMixin
-
-class AuthMixin(KerberosAuthMixin):
- # Allow users to authenticate
- allow_users = True
-
- """
- Requires a builder or user to authenticate
- """
- def get_current_user(self):
- # Fetch the Kerberos ticket
- principal = self.get_authenticated_user()
-
- # Return nothing if we did not receive any credentials
- if not principal:
- return
-
- logging.debug("Searching for principal %s..." % principal)
-
- # Strip the realm
- principal, delimiter, realm = principal.partition("@")
-
- # Return any builders
- if principal.startswith("host/"):
- hostname = principal.removeprefix("host/")
-
- return self.backend.builders.get_by_name(hostname)
-
- # End here if users are not allowed to authenticate
- if not self.allow_users:
- return
-
- # Return users
- return self.backend.users.get_by_name(principal)
-
-
-class BackendMixin(AuthMixin):
- @property
- def backend(self):
- """
- Shortcut handler to pakfire instance
- """
- return self.application.backend
-
- @property
- def db(self):
- return self.backend.db
-
-
-class BaseHandler(BackendMixin, tornado.web.RequestHandler):
- @property
- def builder(self):
- if isinstance(self.current_user, builders.Builder):
- return self.current_user
-
- @property
- def user(self):
- if isinstance(self.current_user, users.User):
- return self.current_user
-
- def get_argument_bool(self, *args, **kwargs):
- arg = self.get_argument(*args, **kwargs)
-
- if arg:
- return arg.lower() in ("on", "true", "1")
-
- def get_argument_int(self, *args, **kwargs):
- arg = self.get_argument(*args, **kwargs)
-
- try:
- return int(arg)
- except (TypeError, ValueError):
- return None
-
- def get_argument_float(self, *args, **kwargs):
- arg = self.get_argument(*args, **kwargs)
-
- try:
- return float(arg)
- except (TypeError, ValueError):
- return None
-
- def get_argument_json(self, *args, **kwargs):
- arg = self.get_argument(*args, **kwargs)
-
- if arg:
- return json.loads(arg)
-
- def get_argument_upload(self, *args, **kwargs):
- """
- Returns an upload
- """
- uuid = self.get_argument(*args, **kwargs)
-
- if uuid:
- return self.backend.uploads.get_by_uuid(uuid)
-
- def get_argument_uploads(self, *args, **kwargs):
- """
- Returns a list of uploads
- """
- uuids = self.get_arguments(*args, **kwargs)
-
- # Return all uploads
- return [self.backend.uploads.get_by_uuid(uuid) for uuid in uuids]
-
-
-# Hello World
-
-class TestHandler(BaseHandler):
- """
- This handler is for checking whether authentication works, etc...
- """
- @tornado.web.authenticated
- def get(self):
- # Send a message which is wrapped into some JSON
- self.finish({
- "message" : [
- "Hello, %s!" % self.current_user,
- ],
- })
-
-# Builds
-
-class BuildsGetHandler(BaseHandler):
- def get(self, build_uuid):
- build = self.backend.builds.get_by_uuid(build_uuid)
- if not build:
- raise tornado.web.HTTPError(404, "Could not find build: %s" % build_uuid)
-
- ret = {
- "distro" : build.distro.slug,
- "jobs" : [j.uuid for j in build.jobs],
- "name" : build.name,
- "package" : build.pkg.uuid,
- "priority" : build.priority,
- "score" : build.score,
- "severity" : build.severity,
- "sup_arches" : build.supported_arches,
- "time_created" : build.created.isoformat(),
- "type" : build.type,
- "uuid" : build.uuid,
- }
-
- # If the build is in a repository, update that bit.
- if build.repo:
- ret["repo"] = build.repo.identifier
-
- self.finish(ret)
-
-
-# Jobs
-
-class JobsBaseHandler(BaseHandler):
- def job2json(self, job):
- ret = {
- "arch" : job.arch,
- "build" : job.build.uuid,
- "duration" : job.duration,
- "name" : job.name,
- "packages" : [p.uuid for p in job.packages],
- "state" : job.state,
- "time_created" : job.time_created.isoformat(),
- "type" : "test" if job.test else "release",
- "uuid" : job.uuid,
- }
-
- if job.builder:
- ret["builder"] = job.builder.hostname
-
- if job.time_started:
- ret["time_started"] = job.time_started.isoformat()
-
- if job.time_finished:
- ret["time_finished"] = job.time_finished.isoformat()
-
- return ret
-
-
-class JobsGetActiveHandler(JobsBaseHandler):
- def get(self):
- # Get list of all active jobs.
- jobs = self.backend.jobs.get_active()
-
- args = {
- "jobs" : [self.job2json(j) for j in jobs],
- }
-
- self.finish(args)
-
-
-class JobsGetLatestHandler(JobsBaseHandler):
- def get(self):
- limit = self.get_argument_int("limit", 5)
-
- # Get the latest jobs.
- jobs = self.backend.jobs.get_recently_ended(limit=limit)
-
- args = {
- "jobs" : [self.job2json(j) for j in jobs],
- }
-
- self.finish(args)
-
-
-class JobsGetQueueHandler(JobsBaseHandler):
- def get(self):
- limit = self.get_argument_int("limit", 5)
-
- # Get the job queue.
- jobs = []
- for job in self.backend.jobqueue:
- jobs.append(job)
-
- limit -= 1
- if not limit: break
-
- args = {
- "jobs" : [self.job2json(j) for j in jobs],
- }
-
- self.finish(args)
-
-
-class JobsGetHandler(JobsBaseHandler):
- def get(self, job_uuid):
- job = self.backend.jobs.get_by_uuid(job_uuid)
- if not job:
- raise tornado.web.HTTPError(404, "Could not find job: %s" % job_uuid)
-
- ret = self.job2json(job)
- self.finish(ret)
-
-
-# Packages
-
-class PackagesGetHandler(BaseHandler):
- def get(self, package_uuid):
- pkg = self.backend.packages.get_by_uuid(package_uuid)
- if not pkg:
- raise tornado.web.HTTPError(404, "Could not find package: %s" % package_uuid)
-
- ret = {
- "arch" : pkg.arch,
- "build_id" : pkg.build_id,
- "build_host" : pkg.build_host,
- "build_time" : pkg.build_time.isoformat(),
- "description" : pkg.description,
- "epoch" : pkg.epoch,
- "filesize" : pkg.filesize,
- "friendly_name" : pkg.friendly_name,
- "friendly_version" : pkg.friendly_version,
- "groups" : pkg.groups,
- "hash_sha512" : pkg.hash_sha512,
- "license" : pkg.license,
- "name" : pkg.name,
- "release" : pkg.release,
- "size" : pkg.size,
- "summary" : pkg.summary,
- "type" : pkg.type,
- "url" : pkg.url,
- "uuid" : pkg.uuid,
- "version" : pkg.version,
-
- # Dependencies.
- "prerequires" : pkg.prerequires,
- "requires" : pkg.requires,
- "provides" : pkg.provides,
- "obsoletes" : pkg.obsoletes,
- "conflicts" : pkg.conflicts,
- }
-
- if pkg.type == "source":
- ret["supported_arches"] = pkg.supported_arches
-
- if pkg.distro:
- ret["distro"] = pkg.distro.slug
-
- self.finish(ret)
-
-
-# Builders
-
-class BuildersBaseHandler(BaseHandler):
- def prepare(self):
- # The request must come from an authenticated buider.
- if not self.builder:
- raise tornado.web.HTTPError(403, "Not authenticated as a builder")
-
-
-class BuildersInfoHandler(BuildersBaseHandler):
- @tornado.web.authenticated
- def post(self):
- args = {
- # CPU info
- "cpu_model" : self.get_argument("cpu_model", None),
- "cpu_count" : self.get_argument("cpu_count", None),
- "cpu_arch" : self.get_argument("cpu_arch", None),
-
- # Pakfire
- "pakfire_version" : self.get_argument("pakfire_version", None),
-
- # OS
- "os_name" : self.get_argument("os_name", None),
- }
-
- with self.db.transaction():
- self.builder.update_info(**args)
-
- # Send something back
- self.finish({
- "status" : "OK",
- })
-
-
-class BuildersStatsHandler(BuildersBaseHandler):
- @tornado.web.authenticated
- def post(self):
- args = {
- # CPU
- "cpu_user" : self.get_argument_float("cpu_user"),
- "cpu_nice" : self.get_argument_float("cpu_nice"),
- "cpu_system" : self.get_argument_float("cpu_system"),
- "cpu_idle" : self.get_argument_float("cpu_idle"),
- "cpu_iowait" : self.get_argument_float("cpu_iowait"),
- "cpu_irq" : self.get_argument_float("cpu_irq"),
- "cpu_softirq" : self.get_argument_float("cpu_softirq"),
- "cpu_steal" : self.get_argument_float("cpu_steal"),
- "cpu_guest" : self.get_argument_float("cpu_guest"),
- "cpu_guest_nice" : self.get_argument_float("cpu_guest_nice"),
-
- # Load average
- "loadavg1" : self.get_argument_float("loadavg1"),
- "loadavg5" : self.get_argument_float("loadavg5"),
- "loadavg15" : self.get_argument_float("loadavg15"),
-
- # Memory
- "mem_total" : self.get_argument_int("mem_total"),
- "mem_available" : self.get_argument_int("mem_available"),
- "mem_used" : self.get_argument_int("mem_used"),
- "mem_free" : self.get_argument_int("mem_free"),
- "mem_active" : self.get_argument_int("mem_active"),
- "mem_inactive" : self.get_argument_int("mem_inactive"),
- "mem_buffers" : self.get_argument_int("mem_buffers"),
- "mem_cached" : self.get_argument_int("mem_cached"),
- "mem_shared" : self.get_argument_int("mem_shared"),
-
- # Swap
- "swap_total" : self.get_argument_int("swap_total"),
- "swap_used" : self.get_argument_int("swap_used"),
- "swap_free" : self.get_argument_int("swap_free"),
-
- }
-
- with self.db.transaction():
- self.builder.log_stats(**args)
-
- # Send something back
- self.finish({
- "status" : "OK",
- })
+++ /dev/null
-#!/usr/bin/python3
-###############################################################################
-# #
-# Pakfire - The IPFire package management system #
-# Copyright (C) 2011 Pakfire development team #
-# #
-# This program is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published by #
-# the Free Software Foundation, either version 3 of the License, or #
-# (at your option) any later version. #
-# #
-# This program is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with this program. If not, see <http://www.gnu.org/licenses/>. #
-# #
-###############################################################################
-
-import json
-import logging
-import tornado.websocket
-
-from .handlers import BaseHandler, BackendMixin
-from . import queue
-
-class BuilderHandler(BackendMixin, tornado.websocket.WebSocketHandler):
- """
- Builders connect to this handler when they are running a build.
-
- We can pass information about this build around in real time.
- """
- # Don't allow users to authenticate
- allow_users = False
-
- @tornado.web.authenticated
- def open(self, job_id):
- self.job = self.backend.jobs.get_by_uuid(job_id)
- if not self.job:
- raise tornado.web.HTTPError(404, "Could not find job %s" % job_id)
-
- # Check if the builder matches
- if not self.current_user == self.job.builder:
- raise tornado.web.HTTPError(403, "Job %s belongs to %s, not %s" % \
- (self.job, self.job.builder, self.current_user))
-
- logging.debug("Connection opened for %s by %s" % (self.job, self.current_user))
-
- async def on_message(self, message):
- # Decode JSON message
- try:
- message = json.loads(message)
- except json.DecodeError as e:
- logging.error("Could not decode JSON message", exc_info=True)
- return
-
- # Log message
- logging.debug("Received message:")
- logging.debug("%s" % json.dumps(message, indent=4))
-
- # Get message type
- t = message.get("message")
-
- # Handle status messages
- if t == "status":
- pass
-
- # Handle log messages
- elif t == "log":
- pass
-
- # Unknown message
- else:
- logging.warning("Received a message of an unknown type: %s" % t)
-
-
-class FinishedHandler(BaseHandler):
- """
- Called after the builder has finished the job
- """
- # Don't allow users to authenticate
- allow_users = False
-
- @tornado.web.authenticated
- async def post(self, uuid):
- job = self.backend.jobs.get_by_uuid(uuid)
- if not job:
- raise tornado.web.HTTPError(404, "Could not find job %s" % uuid)
-
- # Has the job been successful?
- success = self.get_argument_bool("success")
-
- # Fetch the log
- logfile = self.get_argument_upload("log")
-
- # Fetch the packages
- packages = self.get_argument_uploads("packages")
-
- with self.db.transaction():
- # Mark the job as finished
- await job.finished(success=success,
- logfile=logfile, packages=packages)
-
- # Try to dispatch the next job
- await queue.dispatch_jobs(self.backend)
+++ /dev/null
-#!/usr/bin/python3
-###############################################################################
-# #
-# Pakfire - The IPFire package management system #
-# Copyright (C) 2011 Pakfire development team #
-# #
-# This program is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published by #
-# the Free Software Foundation, either version 3 of the License, or #
-# (at your option) any later version. #
-# #
-# This program is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with this program. If not, see <http://www.gnu.org/licenses/>. #
-# #
-###############################################################################
-
-import logging
-import tornado.websocket
-
-from .handlers import BackendMixin
-
-# A list of all builders that have a connection
-connections = []
-
-async def dispatch_jobs(backend):
- """
- Will be called regularly and will dispatch any pending jobs to any
- available builders
- """
- logging.debug("Dispatching jobs...")
-
- # Exit if there are no builders connected
- if not connections:
- logging.debug(" No connections open")
- return
-
- builders = {}
-
- # Map all connections by builder
- for connection in connections:
- builders[connection.builder] = connection
-
- # Process all builders and assign jobs
- # We prioritize builders with fewer jobs
- for builder in sorted(builders, key=lambda b: len(b.jobs)):
- logging.debug(" Processing builder %s" % builder)
-
- # Find the connection
- connection = builders[builder]
-
- with backend.db.transaction():
- if not builder.is_ready():
- logging.debug(" Builder %s is not ready" % builder)
- continue
-
- # We are ready for a new job
- job = backend.jobqueue.pop(builder)
- if job:
- connection.assign_job(job)
- continue
-
- logging.debug(" No jobs processable for %s" % builder)
-
- # If there is no job for the builder, we might as well shut it down
- await builder.stop()
-
-class QueueHandler(BackendMixin, tornado.websocket.WebSocketHandler):
- """
- Builders connect to this handler which will add them to a list of connections.
-
- For all connections, we regularly check if we have any new build jobs, and if so,
- we will send them the job.
- """
-
- # Don't allow users to authenticate
- allow_users = False
-
- @property
- def builder(self):
- return self.current_user
-
- @tornado.web.authenticated
- async def open(self):
- logging.debug("Connection opened by %s" % self.builder)
-
- # Find any previous connections of this builder and close them
- for connection in connections:
- if not connection.builder == self.builder:
- continue
-
- logging.warning("Closing connection to builder %s because it is being replaced" \
- % self.builder)
-
- # Close the previous connection
- connection.close(code=1000, reason="Replaced by a new connection")
-
- # Add this connection to the list
- connections.append(self)
-
- # Dispatch any jobs immediately
- await dispatch_jobs(self.backend)
-
- def on_close(self):
- logging.debug("Connection to %s closed" % self.builder)
-
- # Remove the connection
- try:
- connections.remove(self)
- except IndexError:
- pass
-
- def assign_job(self, job):
- logging.debug("Sending job %s to %s" % (job, self))
-
- # Assign this job
- with self.db.transaction():
- job.assign(builder=self.builder)
-
- self.write_message({
- "message" : "job",
-
- # Add job information
- "id" : job.uuid,
- "name" : "%s" % job,
- "arch" : job.arch,
-
- # Is this a test job?
- "test" : job.test,
-
- # Send the pakfire configuration without using any mirrors
- "conf" : "%s" % job.pakfire(mirrored=False),
-
- # URL to the package
- "pkg" : job.pkg.download_url,
- })
+++ /dev/null
-#!/usr/bin/python3
-
-import asyncio
-import tornado.options
-
-import pakfire.buildservice.hub
-
-tornado.options.define("debug", type=bool, default=False, help="Enable debug mode")
-tornado.options.define("port", type=int, default=8000, help="Port to listen on")
-
-async def main():
- tornado.options.parse_command_line()
-
- # Initialise application
- app = pakfire.buildservice.hub.Application(debug=tornado.options.options.debug)
- app.listen(
- tornado.options.options.port,
- xheaders=True,
- max_body_size=1073741824,
- )
-
- # Wait for forever
- await asyncio.Event().wait()
-
-if __name__ == "__main__":
- asyncio.run(main())
+++ /dev/null
-[Unit]
-Description=Pakfire Hub
-After=network.target
-
-[Service]
-ExecStart=@bindir@/pakfire-hub --port=8001
-User=_pakfire
-
-[Install]
-WantedBy=multi-user.target
from . import packages
from . import repos
from . import search
+from . import uploads
from . import users
from .handlers import *
(r"/builds/([\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{12})/unwatch", builds.UnwatchHandler),
(r"/build/([\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{12})/comment", builds.BuildDetailCommentHandler),
+ (r"/api/v1/builds", builds.APIv1IndexHandler),
+
+ # Queue
(r"/queue", jobs.QueueHandler),
+ (r"/api/v1/jobs/queue", jobs.APIv1QueueHandler),
# Jobs
(r"/jobs/([\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{12})/abort", jobs.AbortHandler),
(r"/jobs/([\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{12})/log", jobs.LogHandler),
(r"/job/([\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{12})/buildroot", jobs.JobBuildrootHandler),
+ (r"/api/v1/jobs/([\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{12})",
+ jobs.APIv1DetailHandler),
# Builders
(r"/builders", builders.BuilderListHandler),
(r"/builders/([A-Za-z0-9\-\.]+)/delete", builders.BuilderDeleteHandler),
(r"/builders/([A-Za-z0-9\-\.]+)/edit", builders.BuilderEditHandler),
(r"/builders/([A-Za-z0-9\-\.]+)", builders.BuilderDetailHandler),
+ (r"/api/v1/builders/stats", builders.APIv1StatsHandler),
# Distributions
(r"/distros", distributions.IndexHandler),
# Log
(r"/log", handlers.LogHandler),
+
+ # Uploads
+ (r"/api/v1/uploads", uploads.APIv1IndexHandler),
+ (r"/api/v1/uploads/([\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{12})",
+ uploads.APIv1DetailHandler),
], default_handler_class=errors.Error404Handler, **settings)
# Launch backend
#!/usr/bin/python3
-import base64
-import kerberos
import logging
-import os
import tornado.web
-import tornado.websocket
from . import base
# Setup logging
-log = logging.getLogger("pakfire.buildservice.auth")
+log = logging.getLogger("pakfire.buildservice.web.auth")
-class KerberosAuthMixin(object):
- """
- A mixin that handles Kerberos authentication
- """
- @property
- def kerberos_realm(self):
- return "IPFIRE.ORG"
-
- @property
- def kerberos_service(self):
- return self.settings.get("kerberos_service", "HTTP")
-
- def authenticate_redirect(self):
- """
- Called when the application needs the user to authenticate.
-
- We will send a response with status code 401 and set the
- WWW-Authenticate header to ask the client to either initiate
- some Kerberos authentication, or to perform HTTP Basic authentication.
- """
- # Ask the client to authenticate using Kerberos
- self.add_header("WWW-Authenticate", "Negotiate")
-
- # Ask the client to authenticate using HTTP Basic Auth
- self.add_header("WWW-Authenticate", "Basic realm=\"%s\"" % self.kerberos_realm)
-
- # Set status to 401
- self.set_status(401)
-
- def get_authenticated_user(self):
- auth_header = self.request.headers.get("Authorization", None)
-
- # No authentication header
- if not auth_header:
- return
-
- # Perform GSS API Negotiation
- if auth_header.startswith("Negotiate"):
- return self._auth_negotiate(auth_header)
-
- # Perform Basic Authentication
- elif auth_header.startswith("Basic "):
- return self._auth_basic(auth_header)
-
- # Fail on anything else
- else:
- raise tornado.web.HTTPError(400, "Unexpected Authentication attempt: %s" % auth_header)
-
- def _auth_negotiate(self, auth_header):
- os.environ["KRB5_KTNAME"] = self.backend.settings.get("krb5-keytab")
-
- auth_value = auth_header.removeprefix("Negotiate ")
-
- try:
- # Initialise the server session
- result, context = kerberos.authGSSServerInit(self.kerberos_service)
-
- if not result == kerberos.AUTH_GSS_COMPLETE:
- raise tornado.web.HTTPError(500, "Kerberos Initialization failed: %s" % result)
-
- # Check the received authentication header
- result = kerberos.authGSSServerStep(context, auth_value)
-
- # If this was not successful, we will fall back to Basic authentication
- if not result == kerberos.AUTH_GSS_COMPLETE:
- return self._auth_basic(auth_header)
-
- if not isinstance(self, tornado.websocket.WebSocketHandler):
- # Fetch the server response
- response = kerberos.authGSSServerResponse(context)
-
- # Send the server response
- self.set_header("WWW-Authenticate", "Negotiate %s" % response)
-
- # Return the user who just authenticated
- user = kerberos.authGSSServerUserName(context)
-
- except kerberos.GSSError as e:
- log.error("Kerberos Authentication Error: %s" % e)
-
- raise tornado.web.HTTPError(500, "Could not initialize the Kerberos context")
-
- finally:
- # Cleanup
- kerberos.authGSSServerClean(context)
-
- log.debug("Successfully authenticated %s" % user)
-
- return user
-
- def _auth_basic(self, auth_header):
- os.environ["KRB5_KTNAME"] = self.backend.settings.get("krb5-keytab")
-
- # Remove "Basic "
- auth_header = auth_header.removeprefix("Basic ")
-
- try:
- # Decode base64
- auth_header = base64.b64decode(auth_header).decode()
-
- username, password = auth_header.split(":", 1)
- except:
- raise tornado.web.HTTPError(400, "Authorization data was malformed")
-
- # Check the credentials against the Kerberos database
- try:
- kerberos.checkPassword(username, password,
- "%s/pakfire.ipfire.org" % self.kerberos_service, self.kerberos_realm)
-
- # Catch any authentication errors
- except kerberos.BasicAuthError as e:
- log.error("Could not authenticate %s: %s" % (username, e))
- return
-
- # Create user principal name
- user = "%s@%s" % (username, self.kerberos_realm)
-
- log.debug("Successfully authenticated %s" % user)
-
- return user
-
-
-class LoginHandler(KerberosAuthMixin, base.BaseHandler):
+class LoginHandler(base.KerberosAuthMixin, base.BaseHandler):
def get(self):
username = self.get_authenticated_user()
if not username:
#!/usr/bin/python
+import base64
import http.client
+import json
+import kerberos
+import logging
+import os
import time
import tornado.locale
import tornado.web
+import tornado.websocket
import traceback
from .. import __version__
from .. import misc
+from .. import users
from ..decorators import *
+# Setup logging
+log = logging.getLogger("pakfire.buildservice.web.base")
+
+class KerberosAuthMixin(object):
+ """
+ A mixin that handles Kerberos authentication
+ """
+ @property
+ def kerberos_realm(self):
+ return "IPFIRE.ORG"
+
+ @property
+ def kerberos_service(self):
+ return self.settings.get("kerberos_service", "HTTP")
+
+ def authenticate_redirect(self):
+ """
+ Called when the application needs the user to authenticate.
+
+ We will send a response with status code 401 and set the
+ WWW-Authenticate header to ask the client to either initiate
+ some Kerberos authentication, or to perform HTTP Basic authentication.
+ """
+ # Ask the client to authenticate using Kerberos
+ self.add_header("WWW-Authenticate", "Negotiate")
+
+ # Ask the client to authenticate using HTTP Basic Auth
+ self.add_header("WWW-Authenticate", "Basic realm=\"%s\"" % self.kerberos_realm)
+
+ # Set status to 401
+ self.set_status(401)
+
+ def get_authenticated_user(self):
+ auth_header = self.request.headers.get("Authorization", None)
+
+ # No authentication header
+ if not auth_header:
+ return
+
+ # Perform GSS API Negotiation
+ if auth_header.startswith("Negotiate"):
+ return self._auth_negotiate(auth_header)
+
+ # Perform Basic Authentication
+ elif auth_header.startswith("Basic "):
+ return self._auth_basic(auth_header)
+
+ # Fail on anything else
+ else:
+ raise tornado.web.HTTPError(400, "Unexpected Authentication attempt: %s" % auth_header)
+
+ def _auth_negotiate(self, auth_header):
+ os.environ["KRB5_KTNAME"] = self.backend.settings.get("krb5-keytab")
+
+ auth_value = auth_header.removeprefix("Negotiate ")
+
+ try:
+ # Initialise the server session
+ result, context = kerberos.authGSSServerInit(self.kerberos_service)
+
+ if not result == kerberos.AUTH_GSS_COMPLETE:
+ raise tornado.web.HTTPError(500, "Kerberos Initialization failed: %s" % result)
+
+ # Check the received authentication header
+ result = kerberos.authGSSServerStep(context, auth_value)
+
+ # If this was not successful, we will fall back to Basic authentication
+ if not result == kerberos.AUTH_GSS_COMPLETE:
+ return self._auth_basic(auth_header)
+
+ if not isinstance(self, tornado.websocket.WebSocketHandler):
+ # Fetch the server response
+ response = kerberos.authGSSServerResponse(context)
+
+ # Send the server response
+ self.set_header("WWW-Authenticate", "Negotiate %s" % response)
+
+ # Return the user who just authenticated
+ user = kerberos.authGSSServerUserName(context)
+
+ except kerberos.GSSError as e:
+ log.error("Kerberos Authentication Error: %s" % e)
+
+ raise tornado.web.HTTPError(500, "Could not initialize the Kerberos context")
+
+ finally:
+ # Cleanup
+ kerberos.authGSSServerClean(context)
+
+ log.debug("Successfully authenticated %s" % user)
+
+ return user
+
+ def _auth_basic(self, auth_header):
+ os.environ["KRB5_KTNAME"] = self.backend.settings.get("krb5-keytab")
+
+ # Remove "Basic "
+ auth_header = auth_header.removeprefix("Basic ")
+
+ try:
+ # Decode base64
+ auth_header = base64.b64decode(auth_header).decode()
+
+ username, password = auth_header.split(":", 1)
+ except:
+ raise tornado.web.HTTPError(400, "Authorization data was malformed")
+
+ # Check the credentials against the Kerberos database
+ try:
+ kerberos.checkPassword(username, password,
+ "%s/pakfire.ipfire.org" % self.kerberos_service, self.kerberos_realm)
+
+ # Catch any authentication errors
+ except kerberos.BasicAuthError as e:
+ log.error("Could not authenticate %s: %s" % (username, e))
+ return
+
+ # Create user principal name
+ user = "%s@%s" % (username, self.kerberos_realm)
+
+ log.debug("Successfully authenticated %s" % user)
+
+ return user
+
+
class BaseHandler(tornado.web.RequestHandler):
@property
def backend(self):
# Collect more information about the exception if possible.
if exc_info:
- if self.current_user and self.current_user.is_admin():
- _traceback += traceback.format_exception(*exc_info)
+ if self.current_user and isinstance(self.current_user, users.User):
+ if self.current_user.is_admin():
+ _traceback += traceback.format_exception(*exc_info)
self.render("errors/error.html",
code=code, message=message, traceback="".join(_traceback), **kwargs)
if slug:
return self.backend.distros.get_by_slug(slug)
+ def get_argument_upload(self, *args, **kwargs):
+ """
+ Returns an upload
+ """
+ uuid = self.get_argument(*args, **kwargs)
+
+ if uuid:
+ return self.backend.uploads.get_by_uuid(uuid)
+
+ def get_argument_uploads(self, *args, **kwargs):
+ """
+ Returns a list of uploads
+ """
+ uuids = self.get_arguments(*args, **kwargs)
+
+ # Return all uploads
+ return [self.backend.uploads.get_by_uuid(uuid) for uuid in uuids]
+
def get_argument_user(self, *args, **kwargs):
name = self.get_argument(*args, **kwargs)
if name:
return self.backend.users.get_by_name(name)
+
+# XXX TODO
+BackendMixin = BaseHandler
+
+class APIMixin(KerberosAuthMixin, BackendMixin):
+ # Generally do not permit users to authenticate against the API
+ allow_users = False
+
+ # Do not perform any XSRF cookie validation on API calls
+ def check_xsrf_cookie(self):
+ pass
+
+ def get_current_user(self):
+ """
+ Authenticates a user or builder
+ """
+ # Fetch the Kerberos ticket
+ principal = self.get_authenticated_user()
+
+ # Return nothing if we did not receive any credentials
+ if not principal:
+ return
+
+ logging.debug("Searching for principal %s..." % principal)
+
+ # Strip the realm
+ principal, delimiter, realm = principal.partition("@")
+
+ # Return any builders
+ if principal.startswith("host/"):
+ hostname = principal.removeprefix("host/")
+
+ return self.backend.builders.get_by_name(hostname)
+
+ # End here if users are not allowed to authenticate
+ if not self.allow_users:
+ return
+
+ # Return users
+ return self.backend.users.get_by_name(principal)
+
+ def get_user_locale(self):
+ return self.get_browser_locale()
+
+ def write_error(self, code, **kwargs):
+ # Send a JSON-encoded error message
+ self.finish({
+ "error" : True,
+ # XXX add error string
+ })
+
+ def _decode_json_message(self, message):
+ # Decode JSON message
+ try:
+ message = json.loads(message)
+
+ except json.DecodeError as e:
+ log.error("Could not decode JSON message", exc_info=True)
+ raise e
+
+ # Log message
+ log.debug("Received message:")
+ log.debug("%s" % json.dumps(message, indent=4))
+
+ return message
from . import base
+class APIv1StatsHandler(base.APIMixin, tornado.websocket.WebSocketHandler):
+ @tornado.web.authenticated
+ def __open(self):
+ args = {
+ # CPU info
+ "cpu_model" : self.get_argument("cpu_model", None),
+ "cpu_count" : self.get_argument("cpu_count", None),
+ "cpu_arch" : self.get_argument("cpu_arch", None),
+
+ # Pakfire
+ "pakfire_version" : self.get_argument("pakfire_version", None),
+
+ # OS
+ "os_name" : self.get_argument("os_name", None),
+ }
+
+ with self.db.transaction():
+ self.builder.update_info(**args)
+
+ def on_message(self, message):
+ # Decode message
+ message = self._decode_json_message(message)
+
+ with self.db.transaction():
+ self.current_user.log_stats(**message)
+
+
class BuilderListHandler(base.BaseHandler):
def get(self):
self.render("builders/list.html", builders=self.backend.builders)
import tornado.web
+from ..errors import NoSuchDistroError
+
from . import base
from . import ui_modules
-class BuildBaseHandler(base.BaseHandler):
- def get_build(self, uuid):
- build = self.backend.builds.get_by_uuid(uuid)
- if not build:
- raise tornado.web.HTTPError(404, "No such build: %s" % uuid)
+class APIv1IndexHandler(base.APIMixin, base.BaseHandler):
+ # Allow users to create builds
+ allow_users = True
+
+ @tornado.web.authenticated
+ async def post(self):
+ # Fetch the upload
+ upload = self.get_argument_upload("upload_id")
+ if not upload:
+ raise tornado.web.HTTPError(404, "Could not find upload")
+
+ # Check permissions of the upload
+ if not upload.has_perm(self.current_user):
+ raise tornado.web.HTTPError(403, "No permission for using upload %s" % upload)
+
+ # Fetch the repository
+ repo_name = self.get_argument("repo", None)
+
+ with self.db.transaction():
+ # Import the package
+ try:
+ package = await self.backend.packages.create(upload)
+
+ # If the distribution that is coded into the package could not be found,
+ # we will send that error to the user...
+ except NoSuchDistroError as e:
+ raise tornado.web.HTTPError(404, "Could not find distribution: %s" % e)
+
+ # Find the repository
+ repo = self.current_user.get_repo(package.distro, repo_name)
+ if not repo:
+ raise tornado.web.HTTPError(404, "Could not find repository")
+
+ # Create a new build
+ build = await self.backend.builds.create(repo, package, owner=self.current_user)
+
+ # Delete the upload
+ await upload.delete()
+
+ # Send some data about the build
+ self.finish({
+ "uuid" : build.uuid,
+ "name" : "%s" % build,
+ })
+
+ # Run dependency check on all jobs
+ await self.backend.jobs.depcheck(build.jobs)
- return build
+ # Try to dispatch jobs
+ await self.backend.jobqueue.dispatch_jobs()
class IndexHandler(base.BaseHandler):
-#!/usr/bin/python
+#!/usr/bin/python3
+import logging
import tornado.web
+import tornado.websocket
from . import base
from . import ui_modules
+# Setup logging
+log = logging.getLogger("pakfire.buildservice.web.jobs")
+
+class APIv1QueueHandler(base.APIMixin, tornado.websocket.WebSocketHandler):
+ """
+ Builders connect to this handler which will add them to a list of connections.
+
+ For all connections, we regularly check if we have any new build jobs, and if so,
+ we will send them the job.
+ """
+
+ # Don't allow users to authenticate
+ allow_users = False
+
+ @property
+ def builder(self):
+ return self.current_user
+
+ @tornado.web.authenticated
+ async def open(self):
+ # Register a new connection
+ await self.backend.jobqueue.open(builder=self.builder, connection=self)
+
+ def on_close(self):
+ # Close the connection
+ self.backend.jobqueue.close(builder=self.builder, connection=self)
+
+ def assign_job(self, job):
+ log.debug("Sending job %s to %s" % (job, self))
+
+ # Assign this job
+ with self.db.transaction():
+ job.assign(builder=self.builder)
+
+ self.write_message({
+ "message" : "job",
+
+ # Add job information
+ "id" : job.uuid,
+ "name" : "%s" % job,
+ "arch" : job.arch,
+
+ # Is this a test job?
+ "test" : job.test,
+
+ # Send the pakfire configuration without using any mirrors
+ "conf" : "%s" % job.pakfire(mirrored=False),
+
+ # URL to the package
+ "pkg" : job.pkg.download_url,
+ })
+
+
+class APIv1DetailHandler(base.APIMixin, tornado.websocket.WebSocketHandler):
+ """
+ Builders connect to this handler when they are running a build.
+
+ We can pass information about this build around in real time.
+ """
+ # Don't allow users to authenticate
+ allow_users = False
+
+ @tornado.web.authenticated
+ def open(self, job_id):
+ self.job = self.backend.jobs.get_by_uuid(job_id)
+ if not self.job:
+ raise tornado.web.HTTPError(404, "Could not find job %s" % job_id)
+
+ # Check if the builder matches
+ if not self.current_user == self.job.builder:
+ raise tornado.web.HTTPError(403, "Job %s belongs to %s, not %s" % \
+ (self.job, self.job.builder, self.current_user))
+
+ log.debug("Connection opened for %s by %s" % (self.job, self.current_user))
+
+ async def on_message(self, message):
+ message = self._decode_json_message(message)
+
+ # Get message type
+ t = message.get("message")
+
+ # Handle status messages
+ if t == "status":
+ pass
+
+ # Handle log messages
+ elif t == "log":
+ pass
+
+ # Handle finished message
+ elif t == "finished":
+ await self._handle_finished(**message)
+
+ # Unknown message
+ else:
+ log.warning("Received a message of an unknown type: %s" % t)
+
+ async def _handle_finished(self, success=False, logfile=None, packages=[], **kwargs):
+ """
+ Called when a job has finished - whether successfully or not
+ """
+ # Fetch the log
+ if logfile:
+ logfile = self.backend.uploads.get_by_uuid(logfile)
+
+ # Fetch the packages
+ if packages:
+ packages = [self.backend.uploads.get_by_uuid(p) for p in packages]
+
+ # Mark the job as finished
+ with self.db.transaction():
+ await self.job.finished(success=success, logfile=logfile, packages=packages)
+
+ # Try to dispatch the next job
+ await self.backend.jobqueue.dispatch_jobs()
+
+
class QueueHandler(base.BaseHandler):
def get(self):
self.render("queue.html", queue=self.backend.jobqueue)
import io
import tornado.web
-from .handlers import BaseHandler
+from . import base
from .. import users
@tornado.web.stream_request_body
-class IndexHandler(BaseHandler):
+class APIv1IndexHandler(base.APIMixin, tornado.web.RequestHandler):
+ # Allow users to perform uploads
+ allow_users = True
+
def initialize(self):
# Buffer to cache the uploaded content
self.buffer = io.BytesIO()
upload = self.backend.uploads.create(
filename,
size=size,
- builder=self.builder,
- user=self.user,
+ uploader=self.current_user,
)
except users.QuotaExceededError as e:
"expires_at" : upload.expires_at.isoformat(),
})
+
+class APIv1DetailHandler(base.APIMixin, tornado.web.RequestHandler):
+ # Allow users to perform uploads
+ allow_users = True
+
@tornado.web.authenticated
- async def delete(self):
+ async def delete(self, uuid):
"""
Deletes an upload with a certain UUID
"""
- # Fetch the UUID
- uuid = self.get_argument("id")
-
# Fetch the upload
upload = self.backend.uploads.get_by_uuid(uuid)
if not upload:
projects / pbs.git / commitdiff
