]> git.ipfire.org Git - pakfire.git/commitdiff
Move building source packages from master to slaves.
authorMichael Tremer <michael.tremer@ipfire.org>
Sat, 23 Apr 2011 10:38:41 +0000 (12:38 +0200)
committerMichael Tremer <michael.tremer@ipfire.org>
Sat, 23 Apr 2011 10:38:41 +0000 (12:38 +0200)
pakfire/server/base.py
pakfire/server/master.py
pakfire/server/slave.py
po/pakfire.pot

index 2df0e72b14890a85b16e5ac93cf6f034216204a9..9490dce18c5e8442c61026d1b007466f049f2aa8 100644 (file)
@@ -85,3 +85,27 @@ class MasterSlave(object):
 
        def upload_log_file(self):
                pass # XXX TO BE DONE
+
+       def package_add(self, source, pkg):
+               logging.info("Adding package: %s" % pkg.friendly_name)
+
+               # Collect data that is sent to the database...
+               info = {
+                       "name"             : pkg.name,
+                       "epoch"            : pkg.epoch,
+                       "version"          : pkg.version,
+                       "release"          : pkg.release,
+                       "groups"           : " ".join(pkg.groups),
+                       "maintainer"       : pkg.maintainer,
+                       "license"          : pkg.license,
+                       "url"              : pkg.url,
+                       "summary"          : pkg.summary,
+                       "description"      : pkg.description,
+                       "supported_arches" : pkg.supported_arches,
+                       "source_id"        : source.id,
+               }
+
+               return self.conn.package_add(info)
+
+       def package_remove(self, source, pkg):
+               logging.info("Package '%s' has been removed." % pkg)
index c465ed9daef29bcaccf33b8bfd7f1e1db63c9b6b..faa8829dbebf46fc777080bb2b4c893335cc404f 100644 (file)
@@ -1,6 +1,7 @@
 #!/usr/bin/python
 
 import logging
+import hashlib
 import os
 import random
 import shutil
@@ -19,21 +20,46 @@ from pakfire.constants import *
 from base import MasterSlave
 
 class Source(object):
-       def __init__(self, master, id, name, path, targetpath, revision, branch):
+       def __init__(self, master, id, name, url, path, targetpath, revision, branch):
                self.master = master
                self.id = id
                self.name = name
-               self.path = path
+               self.url = url
                self.targetpath = targetpath
                self.revision = revision
                self.branch = branch
 
+               # If the repository is not yet checked out, we create a local clone
+               # from it to work with it.
+               if not os.path.exists(self.path):
+                       dirname = os.path.dirname(self.path)
+                       basename = os.path.basename(self.path)
+
+                       if not os.path.exists(dirname):
+                               os.makedirs(dirname)
+
+                       self._git("clone %s %s" % (self.url, basename), path=dirname)
+
+               else:
+                       # Always refresh the repository to have the recent commits.
+                       self._git("fetch")
+
+       @property
+       def path(self):
+               h = hashlib.sha1(self.url)
+
+               # XXX path is to be changed
+               return "/var/cache/pakfire/sources/%s" % h.hexdigest()
+
        @property
        def pakfire(self):
                return self.master.pakfire
 
-       def _git(self, cmd):
-               cmd = "cd %s; git %s" % (self.path, cmd)
+       def _git(self, cmd, path=None):
+               if not path:
+                       path = self.path
+
+               cmd = "cd %s && git %s" % (path, cmd)
 
                logging.debug("Running command: %s" % cmd)
 
@@ -78,7 +104,7 @@ class Source(object):
                                **pakfire_args)
 
                # Send update to the server.
-               self.master.update_revision(self, revision)
+               #self.master.update_revision(self, revision)
 
        def update_files(self, files, **pakfire_args):
                rnd = random.randint(0, 1024**2)
@@ -98,8 +124,7 @@ class Source(object):
                        return
 
                # XXX This totally ignores the local configuration.
-               for pkg in pkgs:
-                       pakfire.api.dist(pkg, resultdirs=[tmpdir,], **pakfire_args)
+               pakfire.api.dist(pkgs, resultdirs=[tmpdir,], **pakfire_args)
 
                # Create a kind of dummy repository to link the packages against it.
                repo = repository.LocalSourceRepository(self.pakfire,
@@ -118,9 +143,6 @@ class Source(object):
                util.rm(tmpdir)
 
        def update(self):
-               # Update files from server.
-               self._git("fetch")
-
                # If there has been no data, yet we need to import all packages
                # that are currently checked out.
                if not self.revision:
@@ -168,42 +190,5 @@ class Master(MasterSlave):
 
                        source.update()
 
-       def build(self):
-               build = self.conn.build_job(self.hostname)
-
-               if not build:
-                       return
-
-               print build
-
-               source = Source(self, **build["source"])
-
-               source.update_revision((build["revision"], False), build_id=build["id"])
-
        def update_revision(self, source, revision):
                self.conn.sources_update_revision(source.id, revision)
-
-       def package_add(self, source, pkg):
-               logging.info("Adding package: %s" % pkg.friendly_name)
-
-               # Collect data that is sent to the database...
-               info = {
-                       "name"             : pkg.name,
-                       "epoch"            : pkg.epoch,
-                       "version"          : pkg.version,
-                       "release"          : pkg.release,
-                       "groups"           : " ".join(pkg.groups),
-                       "maintainer"       : pkg.maintainer,
-                       "license"          : pkg.license,
-                       "url"              : pkg.url,
-                       "summary"          : pkg.summary,
-                       "description"      : pkg.description,
-                       "supported_arches" : pkg.supported_arches,
-                       "source_id"        : source.id,
-               }
-
-               return self.conn.package_add(info)
-
-       def package_remove(self, source, pkg):
-               logging.info("Package '%s' has been removed." % pkg)
-
index 32f103e59f8e06699ff76002f59612f2999d3e56..8fde679e09a0fdc3a22132b9af0e0f0ddaa82513 100644 (file)
@@ -15,6 +15,7 @@ import pakfire.util
 from pakfire.constants import *
 
 from base import MasterSlave
+from master import Source
 
 class Slave(MasterSlave):
        def __init__(self, **pakfire_args):
@@ -53,7 +54,34 @@ class Slave(MasterSlave):
 
                print build
 
-               build_id = build["id"]
+               job_types = {
+                       "binary" : self.build_binary_job,
+                       "source" : self.build_source_job,
+               }
+
+               build_id   = build["id"]
+               build_type = build["type"]
+
+               try:
+                       func = job_types[build_type]
+               except KeyError:
+                       raise Exception, "Build type not supported: %s" % type
+
+               # Call the function that processes the build and try to catch general
+               # exceptions and report them to the server.
+               # If everything goes okay, we tell this the server, too.
+               try:
+                       func(build_id, build)
+
+               except Exception, e:
+                       message = "%s: %s" % (e.__class__.__name__, e)
+                       self.update_build_status(build_id, "failed", message)
+                       raise
+
+               else:
+                       self.update_build_status(build_id, "finished")
+
+       def build_binary_job(self, build_id, build):
                filename = build["name"]
                download = build["download"]
                hash1    = build["hash1"]
@@ -98,14 +126,14 @@ class Slave(MasterSlave):
                        message = "%s: %s" % (e.__class__.__name__, e)
                        self.update_build_status(build_id, "dependency_error", message)
 
-               except Exception, e:
-                       message = "%s: %s" % (e.__class__.__name__, e)
-                       self.update_build_status(build_id, "failed", message)
-                       raise
-
-               else:
-                       self.update_build_status(build_id, "finished")
-
                finally:
                        # Cleanup the files we created.
                        pakfire.util.rm(tmpdir)
+
+       def build_source_job(self, build_id, build):
+               # Update the build status on the server.
+               self.update_build_status(build_id, "running")
+
+               source = Source(self, **build["source"])
+
+               source.update_revision((build["revision"], False), build_id=build_id)
index 049ef31ba4df6b4e93e49103052a23178e129773..9763b61bb51ae0917e084ddf71e45f9bf5677fc7 100644 (file)
@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2011-04-22 23:04+0200\n"
+"POT-Creation-Date: 2011-04-23 01:38+0200\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -21,7 +21,7 @@ msgstr ""
 msgid "Is this okay?"
 msgstr ""
 
-#: ../pakfire/builder.py:198
+#: ../pakfire/builder.py:208
 #, python-format
 msgid "Extracting: %s (source)"
 msgstr ""