From: Michael Tremer Date: Tue, 17 Oct 2017 16:03:23 +0000 (+0100) Subject: Add command to dist packages X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=13b9276effe34062d2e061e298861b93b73d7523;p=people%2Fjschlag%2Fpbs.git Add command to dist packages Signed-off-by: Michael Tremer --- diff --git a/Makefile.am b/Makefile.am index 1364572..4866f73 100644 --- a/Makefile.am +++ b/Makefile.am @@ -67,7 +67,6 @@ dist_doc_DATA = \ dist_bin_SCRIPTS = \ src/scripts/pakfire-build-service \ src/scripts/pakfire-hub \ - src/scripts/pakfire-manager \ src/scripts/pakfire-web dist_configs_DATA = \ @@ -121,13 +120,6 @@ hub_PYTHON = \ hubdir = $(buildservicedir)/hub -manager_PYTHON = \ - src/manager/__init__.py \ - src/manager/base.py \ - src/manager/builds.py - -managerdir = $(buildservicedir)/manager - web_PYTHON = \ src/web/__init__.py \ src/web/handlers.py \ diff --git a/po/POTFILES.in b/po/POTFILES.in index 24dae61..f652982 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -161,10 +161,6 @@ src/templates/user-profile-passwd.html src/templates/user-profile-passwd-ok.html src/hub/handlers.py src/hub/__init__.py -src/manager/base.py -src/manager/builds.py -src/manager/__init__.py -src/manager/repositories.py src/web/handlers_api.py src/web/handlers_auth.py src/web/handlers_base.py diff --git a/src/buildservice/jobqueue.py b/src/buildservice/jobqueue.py index f803271..5259ca1 100644 --- a/src/buildservice/jobqueue.py +++ b/src/buildservice/jobqueue.py @@ -77,3 +77,14 @@ class JobQueue(base.Object): if job.arch == arch: job.schedule("test") break + + def check_build_dependencies(self): + jobs = self.backend.jobs._get_jobs("SELECT * FROM jobs \ + WHERE state = 'new' OR \ + (state = 'dependency_error' AND time_finished < NOW() - '5 minutes'::interval) \ + ORDER BY time_finished LIMIT 50") + + for job in jobs: + with self.db.transaction(): + # Resolve the dependencies + job.resolvdep() \ No newline at end of file diff --git a/src/buildservice/sources.py b/src/buildservice/sources.py index e028478..b65504b 100644 --- a/src/buildservice/sources.py +++ b/src/buildservice/sources.py @@ -3,9 +3,14 @@ import datetime import logging import os +import pakfire +import pakfire.config +import shutil import subprocess +import tempfile from . import base +from . import builds from . import database from . import git @@ -91,6 +96,97 @@ class Sources(base.Object): # Import all new revisions. repo.import_revisions() + def dist(self): + self._init_repos() + + for commit in self.get_pending_commits(): + commit.state = "running" + + logging.debug("Processing commit %s: %s" % (commit.revision, commit.subject)) + + # Get the repository of this commit. + repo = git.Repo(self.pakfire, commit.source) + + # Make sure, it is checked out. + if not repo.cloned: + repo.clone() + + # Navigate to the right revision. + repo.checkout(commit.revision) + + # Get all changed makefiles. + deleted_files = [] + updated_files = [] + + for file in repo.changed_files(commit.revision): + # Don't care about files that are not a makefile. + if not file.endswith(".%s" % MAKEFILE_EXTENSION): + continue + + if os.path.exists(file): + updated_files.append(file) + else: + deleted_files.append(file) + + if updated_files: + # Create a temporary directory where to put all the files + # that are generated here. + pkg_dir = tempfile.mkdtemp() + + try: + config = pakfire.config.Config(["general.conf",]) + config.parse(source.distro.get_config()) + + p = pakfire.PakfireServer(config=config) + + pkgs = [] + for file in updated_files: + try: + pkg_file = p.dist(file, pkg_dir) + pkgs.append(pkg_file) + except: + raise + + # Import all packages in one swoop. + for pkg in pkgs: + # Import the package file and create a build out of it. + builds.import_from_package(_pakfire, pkg, + distro=source.distro, commit=commit, type="release") + + except: + if commit: + commit.state = "failed" + + raise + + finally: + if os.path.exists(pkg_dir): + shutil.rmtree(pkg_dir) + + for file in deleted_files: + # Determine the name of the package. + name = os.path.basename(file) + name = name[:len(MAKEFILE_EXTENSION) + 1] + + source.distro.delete_package(name) + + if commit: + commit.state = "finished" + + def _init_repos(self): + """ + Initialize all repositories. + """ + for source in self.get_all(): + # Skip those which already have a revision. + if source.revision: + continue + + # Initialize the repository or and clone it if necessary. + repo = git.Repo(self.pakfire, source) + if not repo.cloned: + repo.clone() + class Commit(base.DataObject): table = "sources_commits" diff --git a/src/manager/__init__.py b/src/manager/__init__.py deleted file mode 100644 index 69eb8fa..0000000 --- a/src/manager/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/python - -from .builds import CheckBuildDependenciesEvent -from .builds import DistEvent diff --git a/src/manager/base.py b/src/manager/base.py deleted file mode 100644 index 35ad9d5..0000000 --- a/src/manager/base.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/python - -from .. import scheduler - -class Event(scheduler.Event): - def __init__(self, pakfire, *args, **kwargs): - scheduler.Event.__init__(self, *args, **kwargs) - - self.pakfire = pakfire - - @property - def db(self): - return self.pakfire.db diff --git a/src/manager/builds.py b/src/manager/builds.py deleted file mode 100644 index 4ae9bcf..0000000 --- a/src/manager/builds.py +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/python - -import datetime -import logging -import pakfire -import pakfire.config -import shutil -import tempfile - -from .. import builds -from .. import git - -from . import base - -from ..constants import * - -class CheckBuildDependenciesEvent(base.Event): - # Process them as quickly as possible, but there may be more important events. - priority = 3 - - @property - def interval(self): - return self.pakfire.settings.get_int("dependency_checker_interval", 30) - - def run(self): - query = self.db.query("SELECT id FROM jobs \ - WHERE state = 'new' OR \ - (state = 'dependency_error' AND \ - time_finished < NOW() - '5 minutes'::interval) \ - ORDER BY time_finished LIMIT 50") - - for row in query: - e = CheckBuildDependencyEvent(self.pakfire, row.id) - self.scheduler.add_event(e) - - -class CheckBuildDependencyEvent(base.Event): - # Process them as quickly as possible, but there may be more important events. - priority = 3 - - def run(self, job_id): - self.run_subprocess(self._run, job_id) - - @staticmethod - def _run(_pakfire, job_id): - # Get the build job we are working on. - job = _pakfire.jobs.get_by_id(job_id) - if not job: - logging.debug("Job %s does not exist." % job_id) - return - - # Check if the job status has changed in the meanwhile. - if not job.state in ("new", "dependency_error", "failed"): - logging.warning("Job status has already changed: %s - %s" % (job.name, job.state)) - return - - # Resolve the dependencies. - job.resolvdep() - - -class DistEvent(base.Event): - interval = 60 - - first_run = True - - def run(self): - if self.first_run: - self.first_run = False - - self.process = self.init_repos() - - for commit in self.pakfire.sources.get_pending_commits(): - commit.state = "running" - - logging.debug("Processing commit %s: %s" % (commit.revision, commit.subject)) - - # Get the repository of this commit. - repo = git.Repo(self.pakfire, commit.source) - - # Make sure, it is checked out. - if not repo.cloned: - repo.clone() - - # Navigate to the right revision. - repo.checkout(commit.revision) - - # Get all changed makefiles. - deleted_files = [] - updated_files = [] - - for file in repo.changed_files(commit.revision): - # Don't care about files that are not a makefile. - if not file.endswith(".%s" % MAKEFILE_EXTENSION): - continue - - if os.path.exists(file): - updated_files.append(file) - else: - deleted_files.append(file) - - e = DistFileEvent(self.pakfire, None, commit.id, updated_files, deleted_files) - self.scheduler.add_event(e) - - def init_repos(self): - """ - Initialize all repositories. - """ - for source in self.pakfire.sources.get_all(): - # Skip those which already have a revision. - if source.revision: - continue - - # Initialize the repository or and clone it if necessary. - repo = git.Repo(self.pakfire, source) - if not repo.cloned: - repo.clone() - - # Get a list of all files in the repository. - files = repo.get_all_files() - - for file in [f for f in files if file.endswith(".%s" % MAKEFILE_EXTENSION)]: - e = DistFileEvent(self.pakfire, source.id, None, [file,], []) - self.scheduler.add_event(e) - - -class DistFileEvent(base.Event): - def run(self, *args): - self.run_subprocess(self._run, *args) - - @staticmethod - def _run(_pakfire, source_id, commit_id, updated_files, deleted_files): - commit = None - source = None - - if commit_id: - commit = _pakfire.sources.get_commit_by_id(commit_id) - assert commit - - source = commit.source - - if source_id and not source: - source = _pakfire.sources.get_by_id(source_id) - - assert source - - if updated_files: - # Create a temporary directory where to put all the files - # that are generated here. - pkg_dir = tempfile.mkdtemp() - - try: - config = pakfire.config.Config(["general.conf",]) - config.parse(source.distro.get_config()) - - p = pakfire.PakfireServer(config=config) - - pkgs = [] - for file in updated_files: - try: - pkg_file = p.dist(file, pkg_dir) - pkgs.append(pkg_file) - except: - raise - - # Import all packages in one swoop. - for pkg in pkgs: - # Import the package file and create a build out of it. - builds.import_from_package(_pakfire, pkg, - distro=source.distro, commit=commit, type="release") - - except: - if commit: - commit.state = "failed" - - raise - - finally: - if os.path.exists(pkg_dir): - shutil.rmtree(pkg_dir) - - for file in deleted_files: - # Determine the name of the package. - name = os.path.basename(file) - name = name[:len(MAKEFILE_EXTENSION) + 1] - - source.distro.delete_package(name) - - if commit: - commit.state = "finished" diff --git a/src/scripts/pakfire-build-service b/src/scripts/pakfire-build-service index 2ef005b..c757a74 100644 --- a/src/scripts/pakfire-build-service +++ b/src/scripts/pakfire-build-service @@ -14,6 +14,9 @@ class Cli(object): self.backend = pakfire.buildservice.Backend(*args, **kwargs) self._commands = { + # Check build dependencies + "check-build-dependencies" : self.backend.jobqueue.check_build_dependencies, + # Run mirror check "check-mirrors" : self.backend.mirrors.check, @@ -29,6 +32,9 @@ class Cli(object): # Create test jobs "create-test-jobs" : self.backend.jobqueue.create_test_jobs, + # Dist + "dist" : self.backend.sources.dist, + # List repository "list-repository" : self._list_repository, diff --git a/src/scripts/pakfire-manager b/src/scripts/pakfire-manager deleted file mode 100644 index a758c00..0000000 --- a/src/scripts/pakfire-manager +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/python - -import pakfire.buildservice -import pakfire.buildservice.manager - -# Use tornado's logging options. -import tornado.options -tornado.options.parse_command_line() - -# main - -# Create Scheduler instance. -s = pakfire.buildservice.scheduler.Scheduler() - -# Create Pakfire instance. -p = pakfire.buildservice.Pakfire() - -events = ( - pakfire.buildservice.manager.CheckBuildDependenciesEvent, - pakfire.buildservice.manager.DistEvent, -) - -# Add all events to the scheduler. -for e in events: - i = e(p) - s.add_event(i) - -# Run the scheduler. -s.run()