From a52f536c39c42366b60763e9c5736556271685ab Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Fri, 8 Apr 2011 19:17:07 +0200 Subject: [PATCH] Add support to fetch packages from the server and build them. --- pakfire/__init__.py | 23 +++++++++++++--- pakfire/cli.py | 10 +++++++ pakfire/constants.py | 1 + pakfire/server/master.py | 30 ++++++++++++++------ pakfire/server/slave.py | 59 ++++++++++++++++++++++++++++++++++++++-- 5 files changed, 108 insertions(+), 15 deletions(-) diff --git a/pakfire/__init__.py b/pakfire/__init__.py index 79f910078..b4cfa15f6 100644 --- a/pakfire/__init__.py +++ b/pakfire/__init__.py @@ -14,6 +14,7 @@ import packages import plugins import repository import transaction +import util from constants import * from errors import BuildError, PakfireError @@ -28,15 +29,22 @@ class Pakfire(object): # Check if we are operating as the root user. self.check_root_user() + # Generate a random value. + rnd = random.sample(string.lowercase + string.digits, 12) + rnd = "".join(rnd) + # The path where we are operating in self.path = path + self.tempdir = os.path.join(LOCAL_TMP_PATH, rnd) + + if not os.path.exists(self.tempdir): + os.makedirs(self.tempdir) # Save if we are in the builder mode self.builder = builder if self.builder: - rnd = random.sample(string.lowercase + string.digits, 12) - self.path = os.path.join(BUILD_ROOT, "".join(rnd)) + self.path = os.path.join(BUILD_ROOT, rnd) self.debug = False @@ -75,6 +83,13 @@ class Pakfire(object): # always work with valid data. self.repos.update() + def __del__(self): + util.rm(self.tempdir) + + @property + def supported_arches(self): + return self.distro.supported_arches + def check_root_user(self): if not os.getuid() == 0 or not os.getgid() == 0: raise Exception, "You must run pakfire as the root user." @@ -102,7 +117,7 @@ class Pakfire(object): raise BuildError, arch - def build(self, pkg, arch=None, resultdirs=None): + def build(self, pkg, arch=None, resultdirs=None, **kwargs): self.check_build_mode() self.check_host_arch(arch) @@ -112,7 +127,7 @@ class Pakfire(object): # Always include local repository resultdirs.append(self.repos.local_build.path) - b = builder.Builder(pakfire=self, pkg=pkg) + b = builder.Builder(pakfire=self, pkg=pkg, **kwargs) try: b.prepare() diff --git a/pakfire/cli.py b/pakfire/cli.py index 03789f7f3..ac32dc375 100644 --- a/pakfire/cli.py +++ b/pakfire/cli.py @@ -485,6 +485,7 @@ class CliSlave(Cli): # Add sub-commands. self.sub_commands = self.parser.add_subparsers() + self.parse_command_build() self.parse_command_keepalive() # Finally parse all arguments from the command line and save them. @@ -499,9 +500,16 @@ class CliSlave(Cli): self.slave = server.slave.Slave(self.pakfire) self.action2func = { + "build" : self.handle_build, "keepalive" : self.handle_keepalive, } + def parse_command_build(self): + # Implement the "build" command. + sub_keepalive = self.sub_commands.add_parser("build", + help=_("Request a build job from the server.")) + sub_keepalive.add_argument("action", action="store_const", const="build") + def parse_command_keepalive(self): # Implement the "keepalive" command. sub_keepalive = self.sub_commands.add_parser("keepalive", @@ -512,3 +520,5 @@ class CliSlave(Cli): def handle_keepalive(self): self.slave.keepalive() + def handle_build(self): + self.slave.build_job() diff --git a/pakfire/constants.py b/pakfire/constants.py index b0e36d551..fc9464529 100644 --- a/pakfire/constants.py +++ b/pakfire/constants.py @@ -14,6 +14,7 @@ CCACHE_CACHE_DIR = os.path.join(CACHE_DIR, "ccache") REPO_CACHE_DIR = os.path.join(CACHE_DIR, "repos") LOCAL_BUILD_REPO_PATH = "/var/lib/pakfire/local" +LOCAL_TMP_PATH = "/var/tmp/pakfire" PACKAGES_DB_DIR = "var/lib/pakfire" PACKAGES_DB = os.path.join(PACKAGES_DB_DIR, "packages.db") diff --git a/pakfire/server/master.py b/pakfire/server/master.py index 7481a27d0..e0e2ad1ba 100644 --- a/pakfire/server/master.py +++ b/pakfire/server/master.py @@ -37,9 +37,20 @@ class Source(object): if not revision: revision = self.revision - revs = self._git("rev-list %s..origin/%s --no-merges" % (revision, self.branch)) + command = "rev-list %s..origin/%s" % (revision, self.branch) - return reversed(revs.splitlines()) + # Get all normal commits. + commits = self._git("%s --no-merges" % command) + commits = commits.splitlines() + + revisions = [] + for commit in self._git(command).splitlines(): + # Check if commit is a normal commit or merge commit. + merge = not commit in commits + + revisions.append((commit, merge)) + + return reversed(revisions) def _git_changed_files(self, revision1, revision2=""): files = self._git("diff --name-only %s %s" % (revision1, revision2)) @@ -49,17 +60,18 @@ class Source(object): def _git_checkout_revision(self, revision): self._git("checkout %s" % revision) - def update_revision(self, revision): - self._git_checkout_revision(revision) + def update_revision(self, (revision, merge)): + if not merge: + self._git_checkout_revision(revision) - # Get list of all changes files between the current revision and - # the previous one. - files = self._git_changed_files("HEAD^", "HEAD") + # Get list of all changes files between the current revision and + # the previous one. + files = self._git_changed_files("HEAD^", "HEAD") - self.update_files([f for f in files if f.endswith(".%s" % MAKEFILE_EXTENSION)]) + self.update_files([f for f in files if f.endswith(".%s" % MAKEFILE_EXTENSION)]) # Send update to the server. - self.master.update_revision(self, revision) + self.master.update_revision(self, revision) def update_files(self, files): rnd = random.randint(0, 1024**2) diff --git a/pakfire/server/slave.py b/pakfire/server/slave.py index 5692f3bed..31f760256 100644 --- a/pakfire/server/slave.py +++ b/pakfire/server/slave.py @@ -5,6 +5,11 @@ import os import socket import xmlrpclib +import pakfire.downloader +import pakfire.packages + +from pakfire.errors import * + class Slave(object): def __init__(self, pakfire): self.pakfire = pakfire @@ -15,14 +20,64 @@ class Slave(object): self.conn = xmlrpclib.Server(server) + @property + def hostname(self): + """ + Return the host's name. + """ + return socket.gethostname() + def keepalive(self): """ Send the server a keep-alive to say that we are still there. """ - hostname = socket.gethostname() + hostname = self.hostname l1, l5, l15 = os.getloadavg() logging.info("Sending the server a keepalive: %s" % hostname) - self.conn.keepalive(hostname, l5) + # Get all supported architectures and send them to the server. + arches = [a for a in self.pakfire.supported_arches] + arches.sort() + + self.conn.keepalive(hostname, l5, arches) + + def update_build_status(self, build_id, status, message=""): + self.conn.update_build_state(build_id, status, message) + + def build_job(self): + build = self.conn.build_job(self.hostname) + + # If the server has got no job for us, we end right here. + if not build: + return + + build_id = build["id"] + filename = build["source"] + + # Get a package grabber and add mirror download capabilities to it. + grabber = pakfire.downloader.PackageDownloader() + + # Temporary path to store the source. + tempfile = os.path.join(self.pakfire.tempdir, os.path.basename(filename)) + + # Download the source. + grabber.urlgrab(filename, filename=tempfile) + + # Read the package file. + pkg = pakfire.packages.SourcePackage(self.pakfire, + self.pakfire.repos.dummy, tempfile) + + try: + self.update_build_status(build_id, "running") + + self.pakfire.build(pkg, build_id=build_id) + + except DependencyError, e: + self.update_build_status(build_id, "dependency_error", e) + + except: + self.update_build_status(build_id, "failed") + + self.update_build_status(build_id, "finished") -- 2.39.5