From 8276111da19c9754ce94d9bc60051befb38b43c0 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Sun, 24 Jul 2011 13:32:45 +0200 Subject: [PATCH] Fix composing of source packages on the server. --- pakfire/api.py | 4 ++-- pakfire/base.py | 7 ++++++- pakfire/cli.py | 16 ++++++++++++++-- pakfire/packages/__init__.py | 13 +++++++++---- pakfire/packages/solv.py | 8 +++++--- pakfire/packages/source.py | 4 ++++ pakfire/repository/base.py | 10 ++++++++++ pakfire/repository/index.py | 21 +++++++++++++++++---- pakfire/repository/local.py | 10 +++++++++- pakfire/server.py | 34 +++++++++++++++++++++++++--------- src/_pakfiremodule.c | 1 + src/repo.c | 27 +++++++++++++++++++++++++++ src/repo.h | 1 + 13 files changed, 130 insertions(+), 26 deletions(-) diff --git a/pakfire/api.py b/pakfire/api.py index d61b35deb..88698cf78 100644 --- a/pakfire/api.py +++ b/pakfire/api.py @@ -69,10 +69,10 @@ def requires(patterns, **pakfire_args): return pakfire.requires(requires) -def repo_create(path, input_paths, **pakfire_args): +def repo_create(path, input_paths, type="binary", **pakfire_args): pakfire = Pakfire(**pakfire_args) - return pakfire.repo_create(path, input_paths) + return pakfire.repo_create(path, input_paths, type=type) def repo_list(**pakfire_args): pakfire = Pakfire(**pakfire_args) diff --git a/pakfire/base.py b/pakfire/base.py index f0705bdae..97c4f29e7 100644 --- a/pakfire/base.py +++ b/pakfire/base.py @@ -377,12 +377,15 @@ class Pakfire(object): return pkgs - def repo_create(self, path, input_paths): + def repo_create(self, path, input_paths, type="binary"): + assert type in ("binary", "source",) + repo = repository.RepositoryDir( self, name="new", description="New repository.", path=path, + type=type, ) for input_path in input_paths: @@ -390,6 +393,8 @@ class Pakfire(object): repo.save() + return repo + def repo_list(self): return [r for r in self.repos] diff --git a/pakfire/cli.py b/pakfire/cli.py index 49c60701a..62895a4c8 100644 --- a/pakfire/cli.py +++ b/pakfire/cli.py @@ -503,6 +503,7 @@ class CliServer(Cli): self.parse_command_build() self.parse_command_keepalive() + self.parse_command_repoupdate() # Finally parse all arguments from the command line and save them. self.args = self.parser.parse_args() @@ -510,8 +511,9 @@ class CliServer(Cli): self.server = server.Server() self.action2func = { - "build" : self.handle_build, - "keepalive" : self.handle_keepalive, + "build" : self.handle_build, + "keepalive" : self.handle_keepalive, + "repoupdate" : self.handle_repoupdate, } @property @@ -533,8 +535,18 @@ class CliServer(Cli): sub_keepalive.add_argument("action", action="store_const", const="keepalive") + def parse_command_repoupdate(self): + # Implement the "repoupdate" command. + sub_repoupdate = self.sub_commands.add_parser("repoupdate", + help=_("Update all repositories.")) + sub_repoupdate.add_argument("action", action="store_const", + const="repoupdate") + def handle_keepalive(self): self.server.update_info() def handle_build(self): self.server.build_job() + + def handle_repoupdate(self): + self.server.update_repositories() diff --git a/pakfire/packages/__init__.py b/pakfire/packages/__init__.py index e8959f607..f1bc47b70 100644 --- a/pakfire/packages/__init__.py +++ b/pakfire/packages/__init__.py @@ -1,5 +1,7 @@ #!/usr/bin/python +import tarfile + from binary import BinaryPackage from file import InnerTarFile from installed import DatabasePackage, InstalledPackage @@ -21,10 +23,13 @@ def open(pakfire, repo, filename): not. """ # XXX We should make this check much better... - if filename.endswith(".src.%s" % PACKAGE_EXTENSION): - return SourcePackage(pakfire, repo, filename) + + # Simply check if the given file is a tarfile. + if tarfile.is_tarfile(filename): + if filename.endswith(".src.%s" % PACKAGE_EXTENSION): + return SourcePackage(pakfire, repo, filename) + + return BinaryPackage(pakfire, repo, filename) elif filename.endswith(".%s" % MAKEFILE_EXTENSION): return Makefile(pakfire, filename) - - return BinaryPackage(pakfire, repo, filename) diff --git a/pakfire/packages/solv.py b/pakfire/packages/solv.py index 957094acb..2742a865c 100644 --- a/pakfire/packages/solv.py +++ b/pakfire/packages/solv.py @@ -7,8 +7,8 @@ import base import binary class SolvPackage(base.Package): - def __init__(self, pakfire, solvable): - base.Package.__init__(self, pakfire) + def __init__(self, pakfire, solvable, repo=None): + base.Package.__init__(self, pakfire, repo) # Save solvable object self.solvable = solvable @@ -63,8 +63,10 @@ class SolvPackage(base.Package): @property def repo(self): - repo_name = self.solvable.get_repo_name() + if self._repo: + return self._repo + repo_name = self.solvable.get_repo_name() return self.pakfire.repos.get_repo(repo_name) @property diff --git a/pakfire/packages/source.py b/pakfire/packages/source.py index 67a0b48e7..3947610a5 100644 --- a/pakfire/packages/source.py +++ b/pakfire/packages/source.py @@ -13,3 +13,7 @@ class SourcePackage(FilePackage): Return the requirements for the build. """ return self.metadata.get("PKG_REQUIRES", "").split() + + @property + def conflicts(self): + return self.metadata.get("PKG_CONFLICTS", "").split() diff --git a/pakfire/repository/base.py b/pakfire/repository/base.py index 6b0b7aa04..276150bf1 100644 --- a/pakfire/repository/base.py +++ b/pakfire/repository/base.py @@ -6,6 +6,7 @@ import logging import re import cache +import pakfire.packages as packages import pakfire.satsolver as satsolver class RepositoryFactory(object): @@ -36,6 +37,15 @@ class RepositoryFactory(object): def __len__(self): return self.solver_repo.size() + def __iter__(self): + pkgs = [] + + for solv in self.solver_repo.get_all(): + pkg = packages.SolvPackage(self.pakfire, solv, self) + pkgs.append(pkg) + + return iter(pkgs) + @property def pool(self): return self.pakfire.pool diff --git a/pakfire/repository/index.py b/pakfire/repository/index.py index 3fc2cb2ab..9df713bd8 100644 --- a/pakfire/repository/index.py +++ b/pakfire/repository/index.py @@ -250,6 +250,16 @@ class IndexSolv(Index): class IndexDir(Index): + def init(self): + self.pkg_type = None + + if self.repo.type == "binary": + self.pkg_type = packages.BinaryPackage + elif self.repo.type == "source": + self.pkg_type = packages.SourcePackage + + assert self.pkg_type + def check(self): pass # XXX to be done @@ -302,15 +312,18 @@ class IndexDir(Index): package = packages.open(self.pakfire, self.repo, file) - if isinstance(package, packages.BinaryPackage): - if not package.arch in (self.repo.arch, "noarch"): + # Find all packages with the given type and skip those of + # the other type. + if isinstance(package, self.pkg_type): + # Check for binary packages if the architecture matches. + if isinstance(package, packages.BinaryPackage) and \ + not package.arch in (self.repo.arch, "noarch"): logging.warning("Skipped package with wrong architecture: %s (%s)" \ % (package.filename, package.arch)) - print package.type continue # Skip all source packages. - elif isinstance(package, packages.SourcePackage): + else: continue self.add_package(package) diff --git a/pakfire/repository/local.py b/pakfire/repository/local.py index 00d3f84b5..72445a693 100644 --- a/pakfire/repository/local.py +++ b/pakfire/repository/local.py @@ -15,15 +15,23 @@ import pakfire.util as util from pakfire.constants import * class RepositoryDir(base.RepositoryFactory): - def __init__(self, pakfire, name, description, path): + def __init__(self, pakfire, name, description, path, type="binary"): base.RepositoryFactory.__init__(self, pakfire, name, description) # Path to files. self.path = path + # Save type. + assert type in ("binary", "source",) + self.type = type + # Create index self.index = index.IndexDir(self.pakfire, self) + def remove(self): + self.index.clear() + util.rm(self.path) + @property def priority(self): """ diff --git a/pakfire/server.py b/pakfire/server.py index 5a98f445c..79c0c3382 100644 --- a/pakfire/server.py +++ b/pakfire/server.py @@ -2,6 +2,7 @@ import hashlib import logging +import os import random import socket import subprocess @@ -116,13 +117,14 @@ class Source(object): pakfire.api.dist(pkgs, resultdirs=[tmpdir,], **pakfire_args) # Create a kind of dummy repository to link the packages against it. - repo = pakfire.repository.LocalSourceRepository(self.pakfire, - "source-%s" % rnd, "Source packages", tmpdir, idx="directory") - repo.update(force=True) + if pakfire_args.has_key("build_id"): + del pakfire_args["build_id"] + pakfire_args["mode"] = "server" - return repo + repo = pakfire.api.repo_create("source-%s" % rnd, [tmpdir,], type="source", + **pakfire_args) - # XXX don't forget to remove the repository. + return repo def update_all(self): _files = [] @@ -395,8 +397,22 @@ class Server(object): repo = source.update_revision(build["revision"], build_id=build_id) - # Upload all files in the repository. - for pkg in repo.get_all(): - self.upload_file(pkg.filename, build_id) + try: + # Upload all files in the repository. + for pkg in repo: + path = os.path.join(pkg.repo.path, pkg.filename) + self.upload_file(path, build_id) + finally: + repo.remove() + + def update_repositories(self, limit=2): + repos = self.conn.get_repos(limit) + + for repo in repos: + files = self.conn.get_repo_packages(repo["id"]) + + for arch in repo["arches"]: + path = "/pakfire/repositories/%s/%s/%s" % \ + (repo["distro"]["sname"], repo["name"], arch) - repo.remove() + pakfire.api.repo_create(path, files) diff --git a/src/_pakfiremodule.c b/src/_pakfiremodule.c index 0c3b1fe02..959ccd042 100644 --- a/src/_pakfiremodule.c +++ b/src/_pakfiremodule.c @@ -58,6 +58,7 @@ static PyMethodDef Repo_methods[] = { {"write", (PyCFunction)Repo_write, METH_VARARGS, NULL}, {"read", (PyCFunction)Repo_read, METH_VARARGS, NULL}, {"clear", (PyCFunction)Repo_clear, METH_NOARGS, NULL}, + {"get_all", (PyCFunction)Repo_get_all, METH_NOARGS, NULL}, { NULL, NULL, 0, NULL } }; diff --git a/src/repo.c b/src/repo.c index f08b67d10..db47c2b01 100644 --- a/src/repo.c +++ b/src/repo.c @@ -1,10 +1,13 @@ +#include #include +#include #include #include #include "pool.h" #include "repo.h" +#include "solvable.h" PyTypeObject RepoType = { PyObject_HEAD_INIT(NULL) @@ -145,3 +148,27 @@ PyObject *Repo_clear(RepoObject *self) { Py_RETURN_NONE; } + +PyObject *Repo_get_all(RepoObject *self) { + Solvable *s; + Id p; + Repo *r = self->_repo; + + PyObject *list = PyList_New(0); + + FOR_REPO_SOLVABLES(r, p, s) { + SolvableObject *solv; + + solv = PyObject_New(SolvableObject, &SolvableType); + if (solv == NULL) + return NULL; + + solv->_pool = self->_repo->pool; + solv->_id = p; + + PyList_Append(list, (PyObject *)solv); + } + + Py_INCREF(list); + return list; +} diff --git a/src/repo.h b/src/repo.h index c790ed5c9..2b89a3282 100644 --- a/src/repo.h +++ b/src/repo.h @@ -23,6 +23,7 @@ extern PyObject *Repo_set_priority(RepoObject *self, PyObject *args); extern PyObject *Repo_write(RepoObject *self, PyObject *args); extern PyObject *Repo_read(RepoObject *self, PyObject *args); extern PyObject *Repo_clear(RepoObject *self); +extern PyObject *Repo_get_all(RepoObject *self); extern PyTypeObject RepoType; -- 2.39.5