From: Michael Tremer Date: Sat, 23 Apr 2011 10:38:41 +0000 (+0200) Subject: Move building source packages from master to slaves. X-Git-Tag: 0.9.3~46^2~1 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=fc7ed910831f9aa82c0b819390fa86f9adae0187;p=pakfire.git Move building source packages from master to slaves. --- diff --git a/pakfire/server/base.py b/pakfire/server/base.py index 2df0e72b1..9490dce18 100644 --- a/pakfire/server/base.py +++ b/pakfire/server/base.py @@ -85,3 +85,27 @@ class MasterSlave(object): def upload_log_file(self): pass # XXX TO BE DONE + + def package_add(self, source, pkg): + logging.info("Adding package: %s" % pkg.friendly_name) + + # Collect data that is sent to the database... + info = { + "name" : pkg.name, + "epoch" : pkg.epoch, + "version" : pkg.version, + "release" : pkg.release, + "groups" : " ".join(pkg.groups), + "maintainer" : pkg.maintainer, + "license" : pkg.license, + "url" : pkg.url, + "summary" : pkg.summary, + "description" : pkg.description, + "supported_arches" : pkg.supported_arches, + "source_id" : source.id, + } + + return self.conn.package_add(info) + + def package_remove(self, source, pkg): + logging.info("Package '%s' has been removed." % pkg) diff --git a/pakfire/server/master.py b/pakfire/server/master.py index c465ed9da..faa8829db 100644 --- a/pakfire/server/master.py +++ b/pakfire/server/master.py @@ -1,6 +1,7 @@ #!/usr/bin/python import logging +import hashlib import os import random import shutil @@ -19,21 +20,46 @@ from pakfire.constants import * from base import MasterSlave class Source(object): - def __init__(self, master, id, name, path, targetpath, revision, branch): + def __init__(self, master, id, name, url, path, targetpath, revision, branch): self.master = master self.id = id self.name = name - self.path = path + self.url = url self.targetpath = targetpath self.revision = revision self.branch = branch + # If the repository is not yet checked out, we create a local clone + # from it to work with it. + if not os.path.exists(self.path): + dirname = os.path.dirname(self.path) + basename = os.path.basename(self.path) + + if not os.path.exists(dirname): + os.makedirs(dirname) + + self._git("clone %s %s" % (self.url, basename), path=dirname) + + else: + # Always refresh the repository to have the recent commits. + self._git("fetch") + + @property + def path(self): + h = hashlib.sha1(self.url) + + # XXX path is to be changed + return "/var/cache/pakfire/sources/%s" % h.hexdigest() + @property def pakfire(self): return self.master.pakfire - def _git(self, cmd): - cmd = "cd %s; git %s" % (self.path, cmd) + def _git(self, cmd, path=None): + if not path: + path = self.path + + cmd = "cd %s && git %s" % (path, cmd) logging.debug("Running command: %s" % cmd) @@ -78,7 +104,7 @@ class Source(object): **pakfire_args) # Send update to the server. - self.master.update_revision(self, revision) + #self.master.update_revision(self, revision) def update_files(self, files, **pakfire_args): rnd = random.randint(0, 1024**2) @@ -98,8 +124,7 @@ class Source(object): return # XXX This totally ignores the local configuration. - for pkg in pkgs: - pakfire.api.dist(pkg, resultdirs=[tmpdir,], **pakfire_args) + pakfire.api.dist(pkgs, resultdirs=[tmpdir,], **pakfire_args) # Create a kind of dummy repository to link the packages against it. repo = repository.LocalSourceRepository(self.pakfire, @@ -118,9 +143,6 @@ class Source(object): util.rm(tmpdir) def update(self): - # Update files from server. - self._git("fetch") - # If there has been no data, yet we need to import all packages # that are currently checked out. if not self.revision: @@ -168,42 +190,5 @@ class Master(MasterSlave): source.update() - def build(self): - build = self.conn.build_job(self.hostname) - - if not build: - return - - print build - - source = Source(self, **build["source"]) - - source.update_revision((build["revision"], False), build_id=build["id"]) - def update_revision(self, source, revision): self.conn.sources_update_revision(source.id, revision) - - def package_add(self, source, pkg): - logging.info("Adding package: %s" % pkg.friendly_name) - - # Collect data that is sent to the database... - info = { - "name" : pkg.name, - "epoch" : pkg.epoch, - "version" : pkg.version, - "release" : pkg.release, - "groups" : " ".join(pkg.groups), - "maintainer" : pkg.maintainer, - "license" : pkg.license, - "url" : pkg.url, - "summary" : pkg.summary, - "description" : pkg.description, - "supported_arches" : pkg.supported_arches, - "source_id" : source.id, - } - - return self.conn.package_add(info) - - def package_remove(self, source, pkg): - logging.info("Package '%s' has been removed." % pkg) - diff --git a/pakfire/server/slave.py b/pakfire/server/slave.py index 32f103e59..8fde679e0 100644 --- a/pakfire/server/slave.py +++ b/pakfire/server/slave.py @@ -15,6 +15,7 @@ import pakfire.util from pakfire.constants import * from base import MasterSlave +from master import Source class Slave(MasterSlave): def __init__(self, **pakfire_args): @@ -53,7 +54,34 @@ class Slave(MasterSlave): print build - build_id = build["id"] + job_types = { + "binary" : self.build_binary_job, + "source" : self.build_source_job, + } + + build_id = build["id"] + build_type = build["type"] + + try: + func = job_types[build_type] + except KeyError: + raise Exception, "Build type not supported: %s" % type + + # Call the function that processes the build and try to catch general + # exceptions and report them to the server. + # If everything goes okay, we tell this the server, too. + try: + func(build_id, build) + + except Exception, e: + message = "%s: %s" % (e.__class__.__name__, e) + self.update_build_status(build_id, "failed", message) + raise + + else: + self.update_build_status(build_id, "finished") + + def build_binary_job(self, build_id, build): filename = build["name"] download = build["download"] hash1 = build["hash1"] @@ -98,14 +126,14 @@ class Slave(MasterSlave): message = "%s: %s" % (e.__class__.__name__, e) self.update_build_status(build_id, "dependency_error", message) - except Exception, e: - message = "%s: %s" % (e.__class__.__name__, e) - self.update_build_status(build_id, "failed", message) - raise - - else: - self.update_build_status(build_id, "finished") - finally: # Cleanup the files we created. pakfire.util.rm(tmpdir) + + def build_source_job(self, build_id, build): + # Update the build status on the server. + self.update_build_status(build_id, "running") + + source = Source(self, **build["source"]) + + source.update_revision((build["revision"], False), build_id=build_id) diff --git a/po/pakfire.pot b/po/pakfire.pot index 049ef31ba..9763b61bb 100644 --- a/po/pakfire.pot +++ b/po/pakfire.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2011-04-22 23:04+0200\n" +"POT-Creation-Date: 2011-04-23 01:38+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -21,7 +21,7 @@ msgstr "" msgid "Is this okay?" msgstr "" -#: ../pakfire/builder.py:198 +#: ../pakfire/builder.py:208 #, python-format msgid "Extracting: %s (source)" msgstr ""