From: Michael Tremer Date: Wed, 13 Apr 2011 20:18:37 +0000 (+0200) Subject: Update code to upload packages to the master server. X-Git-Tag: 0.9.3~47^2~3 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=1f888d5bc648332210b663bc3696b8dbc9ab751c;p=pakfire.git Update code to upload packages to the master server. --- diff --git a/pakfire/packages/source.py b/pakfire/packages/source.py index 9bea3b13c..e606f4748 100644 --- a/pakfire/packages/source.py +++ b/pakfire/packages/source.py @@ -14,3 +14,7 @@ class SourcePackage(FilePackage): """ return self.metadata.get("PKG_REQUIRES", "").split() + @property + def provides(self): + # XXX just a dummy + return [] diff --git a/pakfire/server/base.py b/pakfire/server/base.py new file mode 100644 index 000000000..958794904 --- /dev/null +++ b/pakfire/server/base.py @@ -0,0 +1,59 @@ +#!/usr/bin/python + +import logging +import os +import socket +import xmlrpclib + +import pakfire.packages + +class MasterSlave(object): + @property + def hostname(self): + """ + Return the host's name. + """ + return socket.gethostname() + + def upload_package_file(self, source_id, pkg_id, pkg): + logging.info("Adding package file: %s" % pkg.filename) + + # Read-in the package payload. + f = open(pkg.filename, "rb") + payload = xmlrpclib.Binary(f.read()) + f.close() + + info = { + "filename" : os.path.basename(pkg.filename), + "source_id" : source_id, + "type" : pkg.type, + "arch" : pkg.arch, + "summary" : pkg.summary, + "description" : pkg.description, + "requires" : " ".join(pkg.requires), + "provides" : "", + "obsoletes" : "", + "conflicts" : "", + "url" : pkg.url, + "license" : pkg.license, + "maintainer" : pkg.maintainer, + "size" : pkg.size, + "hash1" : pkg.hash1, + "build_host" : pkg.build_host, + "build_id" : pkg.build_id, + "build_time" : pkg.build_time, + "uuid" : pkg.uuid, + "payload" : payload, + } + + if isinstance(pkg, pakfire.packages.BinaryPackage): + info.update({ + "provides" : " ".join(pkg.provides), + "obsoletes" : " ".join(pkg.obsoletes), + "conflicts" : " ".join(pkg.conflicts), + }) + + return self.conn.upload_package_file(pkg_id, info) + + def upload_log_file(self): + pass # XXX TO BE DONE diff --git a/pakfire/server/master.py b/pakfire/server/master.py index a03fbe51b..9b911f33f 100644 --- a/pakfire/server/master.py +++ b/pakfire/server/master.py @@ -16,6 +16,8 @@ import pakfire.repository as repository import pakfire.util as util from pakfire.constants import * +from base import MasterSlave + class Source(object): def __init__(self, master, id, name, path, targetpath, revision, branch): self.master = master @@ -106,30 +108,11 @@ class Source(object): for pkg in repo.get_all(): logging.debug("Processing package: %s" % pkg) - pkg_path = "%(name)s/%(epoch)s-%(version)s-%(release)s/%(arch)s" % pkg.info - - file = os.path.join(self.targetpath, pkg_path, os.path.basename(pkg.filename)) - dir = os.path.dirname(file) - - print file - - if os.path.exists(file): - logging.warning("Package does already exist: %s" % file) - - else: - if not os.path.exists(dir): - os.makedirs(dir) - - # Copy the source file to the designated data pool. - shutil.copy2(pkg.filename, file) - # Register package in database and get an ID. pkg_id = self.master.package_add(self, pkg) - # Re-read the package metadata (mainly update filenames). - pkg = packages.SourcePackage(self.pakfire, repo, file) - - self.master.package_file_add(self, pkg_id, pkg) + # Upload the package. + self.master.upload_package_file(self.id, pkg_id, pkg) util.rm(tmpdir) @@ -157,7 +140,7 @@ class Source(object): self.update_files(_files) -class Master(object): +class Master(MasterSlave): def __init__(self, **pakfire_args): self.pakfire = pakfire.base.Pakfire(**pakfire_args) @@ -199,40 +182,6 @@ class Master(object): return self.conn.package_add(info) - def package_file_add(self, source, pkg_id, pkg): - logging.info("Adding package file: %s" % pkg.filename) - - info = { - "path" : pkg.filename[len(source.path) + 1:], - "source_id" : source.id, - "type" : pkg.type, - "arch" : pkg.arch, - "summary" : pkg.summary, - "description" : pkg.description, - "requires" : " ".join(pkg.requires), - "provides" : "", - "obsoletes" : "", - "conflicts" : "", - "url" : pkg.url, - "license" : pkg.license, - "maintainer" : pkg.maintainer, - "size" : pkg.size, - "hash1" : pkg.hash1, - "build_host" : pkg.build_host, - "build_id" : pkg.build_id, - "build_time" : pkg.build_time, - "uuid" : pkg.uuid, - } - - if isinstance(pkg, packages.BinaryPackage): - info.update({ - "provides" : " ".join(pkg.provides), - "obsoletes" : " ".join(pkg.obsoletes), - "conflicts" : " ".join(pkg.conflicts), - }) - - return self.conn.package_file_add(pkg_id, info) - def package_remove(self, source, pkg): logging.info("Package '%s' has been removed." % pkg) diff --git a/pakfire/server/slave.py b/pakfire/server/slave.py index 4a4c00090..d9dc328f2 100644 --- a/pakfire/server/slave.py +++ b/pakfire/server/slave.py @@ -5,14 +5,17 @@ import os import socket import xmlrpclib -import pakfire +import pakfire.api import pakfire.base import pakfire.downloader import pakfire.packages +import pakfire.util -from pakfire.errors import * +from pakfire.constants import * -class Slave(object): +from base import MasterSlave + +class Slave(MasterSlave): def __init__(self, **pakfire_args): self.pakfire = pakfire.base.Pakfire(**pakfire_args) @@ -22,13 +25,6 @@ class Slave(object): self.conn = xmlrpclib.Server(server) - @property - def hostname(self): - """ - Return the host's name. - """ - return socket.gethostname() - def keepalive(self): """ Send the server a keep-alive to say that we are still there. @@ -55,29 +51,56 @@ class Slave(object): return build_id = build["id"] - filename = build["source"] - - # Get a package grabber and add mirror download capabilities to it. - grabber = pakfire.downloader.PackageDownloader() - - # Temporary path to store the source. - tempfile = os.path.join("/var/tmp", os.path.basename(filename)) + filename = build["name"] + data = build["data"].data + hash1 = build["hash1"] - # Download the source. - pkg = grabber.urlgrab(filename, filename=tempfile) + # XXX need to find a better temp dir. + tempfile = os.path.join("/var/tmp", filename) + resultdir = os.path.join("/var/tmp", build_id) try: + # Check if the download checksum matches. + if pakfire.util.calc_hash1(data=data) == hash1: + print "Checksum matches: %s" % hash1 + else: + raise DownloadError, "Download was corrupted" + + # Save the data to a temporary directory. + f = open(tempfile, "wb") + f.write(data) + f.close() + + # Update the build status on the server. self.update_build_status(build_id, "running") - pakfire.build(pkg, build_id=build_id) + # Run the build. + pakfire.api.build(tempfile, build_id=build_id, + resultdirs=[resultdir,]) + + self.update_build_status(build_id, "uploading") + + for dir, subdirs, files in os.walk(resultdir): + for file in files: + file = os.path.join(dir, file) + + pkg = pakfire.packages.open(self.pakfire, None, file) + + self.upload_package_file(build["source_id"], build["pkg_id"], pkg) except DependencyError, e: - self.update_build_status(build_id, "dependency_error", e) + message = "%s: %s" % (e.__class__.__name__, e) + self.update_build_status(build_id, "dependency_error", message) - except: - self.update_build_status(build_id, "failed") + except Exception, e: + raise + message = "%s: %s" % (e.__class__.__name__, e) + self.update_build_status(build_id, "failed", message) raise else: self.update_build_status(build_id, "finished") + finally: + #pakfire.util.rm(tempfile) + pass diff --git a/po/pakfire.pot b/po/pakfire.pot index 346abf093..176e4d3f0 100644 --- a/po/pakfire.pot +++ b/po/pakfire.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2011-04-09 22:27+0200\n" +"POT-Creation-Date: 2011-04-13 21:57+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -21,7 +21,7 @@ msgstr "" msgid "Is this okay?" msgstr "" -#: ../pakfire/builder.py:189 +#: ../pakfire/builder.py:198 #, python-format msgid "Extracting: %s (source)" msgstr ""