From eac44f69edabee2200a5ad5ed34483a0afc2a788 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Mon, 24 Jun 2013 23:47:31 +0200 Subject: [PATCH] pakfire: Sync with upstream. Imports a set of patches which fix various bugs and errors. --- pakfire/pakfire.nm | 21 +- ...t_size-column-when-creating-new-data.patch | 5 +- ...s-a-tmpfs-into-the-build-environment.patch | 24 ++ ...tions-Use-tmp-and-enable-compression.patch | 75 +++++ ...namespace-for-build-environments-and.patch | 271 ++++++++++++++++++ ...se-cgroups-if-the-system-supports-it.patch | 153 ++++++++++ ...t-be-verbose-about-killing-processes.patch | 25 ++ ...der-process-out-of-cgroup-before-des.patch | 35 +++ ...rt-Catch-503-Bad-Gateway-HTTP-errors.patch | 41 +++ ...eepalive-daemon-after-it-has-crashed.patch | 57 ++++ pakfire/patches/0010-Fix-typo.patch | 25 ++ .../0011-system-Fix-reading-CPU-info.patch | 41 +++ ...-copy-dev-nodes-if-they-really-exist.patch | 30 ++ ...bar-Remove-overwriting-the-Bar-class.patch | 45 +++ ...-Better-formatting-for-progress-bars.patch | 25 ++ ...rt-Handle-503-in-the-same-way-as-502.patch | 36 +++ ...temd-service-file-for-pakfire-daemon.patch | 50 ++++ ...the-host-key-is-not-in-the-key-store.patch | 37 +++ .../0018-Fix-installing-unit-files.patch | 40 +++ .../0019-Fix-typo-in-exception-name.patch | 25 ++ 20 files changed, 1058 insertions(+), 3 deletions(-) create mode 100644 pakfire/patches/0002-Mount-tmp-as-a-tmpfs-into-the-build-environment.patch create mode 100644 pakfire/patches/0003-ccache-optimizations-Use-tmp-and-enable-compression.patch create mode 100644 pakfire/patches/0004-Create-an-extra-namespace-for-build-environments-and.patch create mode 100644 pakfire/patches/0005-builder-Use-cgroups-if-the-system-supports-it.patch create mode 100644 pakfire/patches/0006-cgroup-Don-t-be-verbose-about-killing-processes.patch create mode 100644 pakfire/patches/0007-cgroup-Move-builder-process-out-of-cgroup-before-des.patch create mode 100644 pakfire/patches/0008-transport-Catch-503-Bad-Gateway-HTTP-errors.patch create mode 100644 pakfire/patches/0009-daemon-Restart-keepalive-daemon-after-it-has-crashed.patch create mode 100644 pakfire/patches/0010-Fix-typo.patch create mode 100644 pakfire/patches/0011-system-Fix-reading-CPU-info.patch create mode 100644 pakfire/patches/0012-buildroot-Only-copy-dev-nodes-if-they-really-exist.patch create mode 100644 pakfire/patches/0013-progressbar-Remove-overwriting-the-Bar-class.patch create mode 100644 pakfire/patches/0014-Better-formatting-for-progress-bars.patch create mode 100644 pakfire/patches/0015-transport-Handle-503-in-the-same-way-as-502.patch create mode 100644 pakfire/patches/0016-Add-systemd-service-file-for-pakfire-daemon.patch create mode 100644 pakfire/patches/0017-Warn-if-the-host-key-is-not-in-the-key-store.patch create mode 100644 pakfire/patches/0018-Fix-installing-unit-files.patch create mode 100644 pakfire/patches/0019-Fix-typo-in-exception-name.patch diff --git a/pakfire/pakfire.nm b/pakfire/pakfire.nm index 406be2135..8f0453184 100644 --- a/pakfire/pakfire.nm +++ b/pakfire/pakfire.nm @@ -5,7 +5,7 @@ name = pakfire version = 0.9.25 -release = 2 +release = 3 maintainer = Michael Tremer groups = System/Packaging @@ -29,6 +29,7 @@ build libsolv-devel >= 0.0.0-4 popt-devel python-devel + systemd-units xz-devel end @@ -151,6 +152,24 @@ packages %{bindir}/pakfire-daemon end configfiles = %{sysconfdir}/pakfire/daemon.conf + + script postin + systemctl daemon-reload >/dev/null 2>&1 || : + end + + script preun + systemctl --no-reload disable pakfire-daemon.service > /dev/null 2>&1 || : + systemctl stop pakfire-daemon.service > /dev/null 2>&1 || : + end + + script postun + systemctl daemon-reload >/dev/null 2>&1 || : + end + + script postup + systemctl daemon-reload 2>&1 || : + systemctl reload-or-try-restart pakfire-daemon.service >/dev/null 2>&1 || : + end end package quality-agent diff --git a/pakfire/patches/0001-database-Add-inst_size-column-when-creating-new-data.patch b/pakfire/patches/0001-database-Add-inst_size-column-when-creating-new-data.patch index f366f216d..b4173928c 100644 --- a/pakfire/patches/0001-database-Add-inst_size-column-when-creating-new-data.patch +++ b/pakfire/patches/0001-database-Add-inst_size-column-when-creating-new-data.patch @@ -1,7 +1,8 @@ From 0d6d6fd2b89c609a6f8daff225c6c98b54a041cc Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Mon, 4 Mar 2013 17:18:52 +0100 -Subject: [PATCH] database: Add inst_size column when creating new databases. +Subject: [PATCH 01/19] database: Add inst_size column when creating new + databases. --- python/pakfire/repository/database.py | 1 + @@ -20,5 +21,5 @@ index fc8a1a9..d8751e9 100644 provides TEXT, requires TEXT, -- -1.7.11.7 +1.8.1.4 diff --git a/pakfire/patches/0002-Mount-tmp-as-a-tmpfs-into-the-build-environment.patch b/pakfire/patches/0002-Mount-tmp-as-a-tmpfs-into-the-build-environment.patch new file mode 100644 index 000000000..db2ada811 --- /dev/null +++ b/pakfire/patches/0002-Mount-tmp-as-a-tmpfs-into-the-build-environment.patch @@ -0,0 +1,24 @@ +From 4252a54980c0996a9b46ad2043aefe0910bc5da7 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Tue, 5 Mar 2013 01:20:22 +0100 +Subject: [PATCH 02/19] Mount /tmp as a tmpfs into the build environment. + +--- + python/pakfire/builder.py | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/python/pakfire/builder.py b/python/pakfire/builder.py +index 24027d9..1cf1e8e 100644 +--- a/python/pakfire/builder.py ++++ b/python/pakfire/builder.py +@@ -572,6 +572,7 @@ class BuildEnviron(object): + ("pakfire_tmpfs", "/dev", "tmpfs", "mode=755,nosuid"), + ("/dev/pts", "/dev/pts", "bind", "bind"), + ("pakfire_tmpfs", "/run", "tmpfs", "mode=755,nosuid,nodev"), ++ ("pakfire_tmpfs", "/tmp", "tmpfs", "mode=755,nosuid,nodev"), + ] + + # If selinux is enabled. +-- +1.8.1.4 + diff --git a/pakfire/patches/0003-ccache-optimizations-Use-tmp-and-enable-compression.patch b/pakfire/patches/0003-ccache-optimizations-Use-tmp-and-enable-compression.patch new file mode 100644 index 000000000..7358615f2 --- /dev/null +++ b/pakfire/patches/0003-ccache-optimizations-Use-tmp-and-enable-compression.patch @@ -0,0 +1,75 @@ +From 27f559296e5301c16e2be42acfde3bcf7e374290 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Tue, 5 Mar 2013 01:20:54 +0100 +Subject: [PATCH 03/19] ccache optimizations: Use /tmp and enable compression. + +When the cache is on an NFS share, a lot of preprocessed files +will be transferred to that share, because ccache usually uses +$CCACHE_DIR/tmp as directory for its temporary files. +/tmp is always local and only the final data has to be live in +the cache is transferred over the wire. + +To decrease the size of the cache, we enable compression. That +will cost a bit of CPU performance, but processors are usually +very fast nowadays and the bottleneck when compiling code is IO. +--- + examples/builder.conf | 5 +++++ + python/pakfire/builder.py | 15 +++++++++++++++ + 2 files changed, 20 insertions(+) + +diff --git a/examples/builder.conf b/examples/builder.conf +index ff42733..128a118 100644 +--- a/examples/builder.conf ++++ b/examples/builder.conf +@@ -9,6 +9,7 @@ file = /var/log/pakfire-builder.log + #distro = ipfire3 + + # Use ccache in order to build rebuilds in less time. ++# See also the [ccache] section. + #use_ccache = true + + # Use icecream in order to speed up builds. +@@ -21,3 +22,7 @@ file = /var/log/pakfire-builder.log + + # Create loop devices in build environment. + #use_loop_devices = true ++ ++[ccache] ++# Turn on compression to get more files into the cache. ++#compress = true +diff --git a/python/pakfire/builder.py b/python/pakfire/builder.py +index 1cf1e8e..250a659 100644 +--- a/python/pakfire/builder.py ++++ b/python/pakfire/builder.py +@@ -119,6 +119,12 @@ class BuildEnviron(object): + "buildroot_tmpfs" : self.config.get_bool("builder", "use_tmpfs", False), + } + ++ # Get ccache settings. ++ if self.settings.get("enable_ccache", False): ++ self.settings.update({ ++ "ccache_compress" : self.config.get_bool("ccache", "compress", True), ++ }) ++ + # Try to get the configured host key. If it is available, + # we will automatically sign all packages with it. + if self.keyring.get_host_key(secret=True): +@@ -615,6 +621,15 @@ class BuildEnviron(object): + # Inherit environment from distro + env.update(self.pakfire.distro.environ) + ++ # ccache environment settings ++ if self.settings.get("enable_ccache", False): ++ compress = self.settings.get("ccache_compress", False) ++ if compress: ++ env["CCACHE_COMPRESS"] = "1" ++ ++ # Let ccache create its temporary files in /tmp. ++ env["CCACHE_TEMPDIR"] = "/tmp" ++ + # Icecream environment settings + if self.settings.get("enable_icecream", False): + # Set the toolchain path +-- +1.8.1.4 + diff --git a/pakfire/patches/0004-Create-an-extra-namespace-for-build-environments-and.patch b/pakfire/patches/0004-Create-an-extra-namespace-for-build-environments-and.patch new file mode 100644 index 000000000..fd1eeeb0b --- /dev/null +++ b/pakfire/patches/0004-Create-an-extra-namespace-for-build-environments-and.patch @@ -0,0 +1,271 @@ +From 392371f70db6fe3df79a6e2306092857c4615a4b Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Fri, 8 Mar 2013 11:02:18 +0100 +Subject: [PATCH 04/19] Create an extra namespace for build environments and + private network. + +Create a python binding for unshare(2) and use this to +unshare the IPC and UTS namespace (if the kernel supports that). + +Also add the option to use private networking in the container. +--- + examples/builder.conf | 6 ++++++ + python/pakfire/builder.py | 18 +++++++++++++++++- + python/pakfire/cli.py | 40 ++++++++++++++++++++++++++++------------ + python/src/_pakfiremodule.c | 13 +++++++++++++ + python/src/util.c | 22 ++++++++++++++++++++++ + python/src/util.h | 1 + + 6 files changed, 87 insertions(+), 13 deletions(-) + +diff --git a/examples/builder.conf b/examples/builder.conf +index 128a118..978c7d9 100644 +--- a/examples/builder.conf ++++ b/examples/builder.conf +@@ -23,6 +23,12 @@ file = /var/log/pakfire-builder.log + # Create loop devices in build environment. + #use_loop_devices = true + ++# Use private network. ++# Setting this to true will result in the build ++# chroot having its own network - i.e. no network connection ++# to the outside world. ++#private_network = false ++ + [ccache] + # Turn on compression to get more files into the cache. + #compress = true +diff --git a/python/pakfire/builder.py b/python/pakfire/builder.py +index 250a659..5cb00aa 100644 +--- a/python/pakfire/builder.py ++++ b/python/pakfire/builder.py +@@ -69,7 +69,7 @@ class BuildEnviron(object): + # The version of the kernel this machine is running. + kernel_version = os.uname()[2] + +- def __init__(self, pakfire, filename=None, distro_name=None, build_id=None, logfile=None, release_build=True): ++ def __init__(self, pakfire, filename=None, distro_name=None, build_id=None, logfile=None, release_build=True, **kwargs): + self.pakfire = pakfire + + # Check if the given pakfire instance is of the correct type. +@@ -117,6 +117,7 @@ class BuildEnviron(object): + "enable_icecream" : self.config.get_bool("builder", "use_icecream", False), + "sign_packages" : False, + "buildroot_tmpfs" : self.config.get_bool("builder", "use_tmpfs", False), ++ "private_network" : self.config.get_bool("builder", "private_network", False), + } + + # Get ccache settings. +@@ -130,6 +131,9 @@ class BuildEnviron(object): + if self.keyring.get_host_key(secret=True): + self.settings["sign_packages"] = True + ++ # Add settings from keyword arguments. ++ self.settings.update(kwargs) ++ + # Where do we put the result? + self.resultdir = os.path.join(self.pakfire.path, "result") + +@@ -164,6 +168,14 @@ class BuildEnviron(object): + def start(self): + assert not self.pakfire.initialized, "Pakfire has already been initialized" + ++ # Unshare namepsace. ++ # If this fails because the kernel has no support for CLONE_NEWIPC or CLONE_NEWUTS, ++ # we try to fall back to just set CLONE_NEWNS. ++ try: ++ _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNS|_pakfire.SCHED_CLONE_NEWIPC|_pakfire.SCHED_CLONE_NEWUTS) ++ except RuntimeError, e: ++ _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNS) ++ + # Mount the directories. + self._mountall() + +@@ -173,6 +185,10 @@ class BuildEnviron(object): + # Initialize pakfire instance. + self.pakfire.initialize() + ++ # Optionally enable private networking. ++ if self.settings.get("private_network", None): ++ _pakfire.unshare(_pakfire.SCHED_CLONE_NEWNET) ++ + # Populate /dev. + self.populate_dev() + +diff --git a/python/pakfire/cli.py b/python/pakfire/cli.py +index 232aad8..a80b397 100644 +--- a/python/pakfire/cli.py ++++ b/python/pakfire/cli.py +@@ -543,6 +543,8 @@ class CliBuilder(Cli): + help=_("Run a shell after a successful build.")) + sub_build.add_argument("--no-install-test", action="store_true", + help=_("Do not perform the install test.")) ++ sub_build.add_argument("--private-network", action="store_true", ++ help=_("Disable network in container.")) + + def parse_command_shell(self): + # Implement the "shell" command. +@@ -554,6 +556,8 @@ class CliBuilder(Cli): + + sub_shell.add_argument("-m", "--mode", nargs="?", default="development", + help=_("Mode to run in. Is either 'release' or 'development' (default).")) ++ sub_shell.add_argument("--private-network", action="store_true", ++ help=_("Disable network in container.")) + + def parse_command_dist(self): + # Implement the "dist" command. +@@ -580,22 +584,25 @@ class CliBuilder(Cli): + else: + raise FileNotFoundError, pkg + +- # Check whether to enable the install test. +- install_test = not self.args.no_install_test ++ # Build argument list. ++ kwargs = { ++ "after_shell" : self.args.after_shell, ++ # Check whether to enable the install test. ++ "install_test" : not self.args.no_install_test, ++ "result_dir" : [self.args.resultdir,], ++ "shell" : True, ++ } + + if self.args.mode == "release": +- release_build = True ++ kwargs["release_build"] = True + else: +- release_build = False ++ kwargs["release_build"] = False ++ ++ if self.args.private_network: ++ kwargs["private_network"] = True + + p = self.create_pakfire() +- p.build(pkg, +- install_test=install_test, +- resultdirs=[self.args.resultdir,], +- shell=True, +- after_shell=self.args.after_shell, +- release_build=release_build, +- ) ++ p.build(pkg, **kwargs) + + def handle_shell(self): + pkg = None +@@ -617,7 +624,16 @@ class CliBuilder(Cli): + release_build = False + + p = self.create_pakfire() +- p.shell(pkg, release_build=release_build) ++ ++ kwargs = { ++ "release_build" : release_build, ++ } ++ ++ # Private network ++ if self.args.private_network: ++ kwargs["private_network"] = True ++ ++ p.shell(pkg, **kwargs) + + def handle_dist(self): + # Get the packages from the command line options +diff --git a/python/src/_pakfiremodule.c b/python/src/_pakfiremodule.c +index 4c94c5a..c208634 100644 +--- a/python/src/_pakfiremodule.c ++++ b/python/src/_pakfiremodule.c +@@ -18,9 +18,14 @@ + # # + #############################################################################*/ + ++#ifndef _GNU_SOURCE ++#define _GNU_SOURCE ++#endif ++ + #include + + #include ++#include + #include + + #include "capabilities.h" +@@ -43,6 +48,7 @@ static PyMethodDef pakfireModuleMethods[] = { + {"set_capabilities", (PyCFunction)set_capabilities, METH_VARARGS, NULL}, + {"personality", (PyCFunction)_personality, METH_VARARGS, NULL}, + {"sync", (PyCFunction)_sync, METH_NOARGS, NULL}, ++ {"unshare", (PyCFunction)_unshare, METH_VARARGS, NULL}, + { NULL, NULL, 0, NULL } + }; + +@@ -275,6 +281,13 @@ void init_pakfire(void) { + PyDict_SetItemString(d, "PERSONALITY_LINUX", Py_BuildValue("i", PER_LINUX)); + PyDict_SetItemString(d, "PERSONALITY_LINUX32", Py_BuildValue("i", PER_LINUX32)); + ++ // Namespace stuff ++ PyDict_SetItemString(d, "SCHED_CLONE_NEWIPC", Py_BuildValue("i", CLONE_NEWIPC)); ++ PyDict_SetItemString(d, "SCHED_CLONE_NEWPID", Py_BuildValue("i", CLONE_NEWPID)); ++ PyDict_SetItemString(d, "SCHED_CLONE_NEWNET", Py_BuildValue("i", CLONE_NEWNET)); ++ PyDict_SetItemString(d, "SCHED_CLONE_NEWNS", Py_BuildValue("i", CLONE_NEWNS)); ++ PyDict_SetItemString(d, "SCHED_CLONE_NEWUTS", Py_BuildValue("i", CLONE_NEWUTS)); ++ + // Add constants for relations + PyDict_SetItemString(d, "REL_EQ", Py_BuildValue("i", REL_EQ)); + PyDict_SetItemString(d, "REL_LT", Py_BuildValue("i", REL_LT)); +diff --git a/python/src/util.c b/python/src/util.c +index ed555f5..acea90a 100644 +--- a/python/src/util.c ++++ b/python/src/util.c +@@ -18,11 +18,18 @@ + # # + #############################################################################*/ + ++#ifndef _GNU_SOURCE ++#define _GNU_SOURCE ++#endif ++ + #include + ++#include ++#include + #include + #include + ++#include "config.h" + #include "util.h" + + PyObject *_personality(PyObject *self, PyObject *args) { +@@ -52,6 +59,21 @@ PyObject *_sync(PyObject *self, PyObject *args) { + Py_RETURN_NONE; + } + ++PyObject *_unshare(PyObject *self, PyObject *args) { ++ int flags = 0; ++ ++ if (!PyArg_ParseTuple(args, "i", &flags)) { ++ return NULL; ++ } ++ ++ int ret = unshare(flags); ++ if (ret < 0) { ++ return PyErr_SetFromErrno(PyExc_RuntimeError); ++ } ++ ++ return Py_BuildValue("i", ret); ++} ++ + PyObject *version_compare(PyObject *self, PyObject *args) { + Pool *pool; + const char *evr1, *evr2; +diff --git a/python/src/util.h b/python/src/util.h +index 6ed9d14..0322d1c 100644 +--- a/python/src/util.h ++++ b/python/src/util.h +@@ -27,6 +27,7 @@ + + extern PyObject *_personality(PyObject *self, PyObject *args); + extern PyObject *_sync(PyObject *self, PyObject *args); ++extern PyObject *_unshare(PyObject *self, PyObject *args); + extern PyObject *version_compare(PyObject *self, PyObject *args); + + #endif +-- +1.8.1.4 + diff --git a/pakfire/patches/0005-builder-Use-cgroups-if-the-system-supports-it.patch b/pakfire/patches/0005-builder-Use-cgroups-if-the-system-supports-it.patch new file mode 100644 index 000000000..dbd323213 --- /dev/null +++ b/pakfire/patches/0005-builder-Use-cgroups-if-the-system-supports-it.patch @@ -0,0 +1,153 @@ +From 904c8f1136b683a90a08cd4cf671a26f24e45f03 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Fri, 8 Mar 2013 16:08:16 +0100 +Subject: [PATCH 05/19] builder: Use cgroups if the system supports it. + +A systemd-based system is required right now. +--- + python/pakfire/builder.py | 12 ++++++-- + python/pakfire/cgroup.py | 73 ++++++++++++++++++++++++++++------------------- + 2 files changed, 52 insertions(+), 33 deletions(-) + +diff --git a/python/pakfire/builder.py b/python/pakfire/builder.py +index 5cb00aa..7427a8c 100644 +--- a/python/pakfire/builder.py ++++ b/python/pakfire/builder.py +@@ -309,10 +309,16 @@ class BuildEnviron(object): + self.cgroup = None + return + +- self.cgroup = cgroup.CGroup("pakfire/builder/%s" % self.build_id) ++ # Search for the cgroup this process is currently running in. ++ parent_cgroup = cgroup.find_by_pid(os.getpid()) ++ if not parent_cgroup: ++ return ++ ++ # Create our own cgroup inside the parent cgroup. ++ self.cgroup = parent_cgroup.create_child_cgroup("pakfire/builder/%s" % self.build_id) + +- # Attach the pakfire-builder process to the parent group. +- self.cgroup.parent.attach() ++ # Attach the pakfire-builder process to the group. ++ self.cgroup.attach() + + def init_logging(self, logfile): + if logfile: +diff --git a/python/pakfire/cgroup.py b/python/pakfire/cgroup.py +index f372a6c..6c85937 100644 +--- a/python/pakfire/cgroup.py ++++ b/python/pakfire/cgroup.py +@@ -8,40 +8,14 @@ import time + import logging + log = logging.getLogger("pakfire.cgroups") + +-CGROUP_PATH_CANDIDATES = ( +- "/sys/fs/cgroup", +-) +- +-def find_cgroup_path(): +- """ +- This function tries to find the right place +- where to put the cgroups. +- """ +- for path in CGROUP_PATH_CANDIDATES: +- check_path = os.path.join(path, "tasks") +- if not os.path.exists(check_path): +- continue +- +- return path +- +-CGROUP_PATH = find_cgroup_path() +- +-def supported(): +- """ +- Returns True or False depending on +- whether cgroups are supported or not. +- """ +- if CGROUP_PATH is None: +- return False +- +- return True ++CGROUP_MOUNTPOINT = "/sys/fs/cgroup/systemd" + + class CGroup(object): + def __init__(self, name): + assert supported(), "cgroups are not supported by this kernel" + + self.name = name +- self.path = os.path.join(CGROUP_PATH, name) ++ self.path = os.path.join(CGROUP_MOUNTPOINT, name) + self.path = os.path.abspath(self.path) + + # The parent cgroup. +@@ -58,6 +32,31 @@ class CGroup(object): + def __cmp__(self, other): + return cmp(self.path, other.path) + ++ @classmethod ++ def find_by_pid(cls, pid): ++ """ ++ Returns the cgroup of the process with the given PID. ++ ++ If no cgroup can be found, None is returned. ++ """ ++ if not cls.supported: ++ return ++ ++ for d, subdirs, files in os.walk(CGROUP_MOUNTPOINT): ++ if not "tasks" in files: ++ continue ++ ++ cgroup = cls(d) ++ if pid in cgroup.tasks: ++ return cgroup ++ ++ @staticmethod ++ def supported(): ++ """ ++ Returns true, if this hosts supports cgroups. ++ """ ++ return os.path.ismount(CGROUP_MOUNTPOINT) ++ + def create(self): + """ + Creates the filesystem structure for +@@ -69,6 +68,13 @@ class CGroup(object): + log.debug("cgroup '%s' has been created." % self.name) + os.makedirs(self.path) + ++ def create_child_cgroup(self, name): ++ """ ++ Create a child cgroup with name relative to the ++ parent cgroup. ++ """ ++ return self.__class__(os.path.join(self.name, name)) ++ + def attach(self): + """ + Attaches this task to the cgroup. +@@ -152,8 +158,8 @@ class CGroup(object): + + @property + def parent(self): +- # Cannot go above CGROUP_PATH. +- if self.path == CGROUP_PATH: ++ # Cannot go above CGROUP_MOUNTPOINT. ++ if self.path == CGROUP_MOUNTPOINT: + return + + if self._parent is None: +@@ -317,3 +323,10 @@ class CGroup(object): + time.sleep(0.2) + + return self.is_empty() ++ ++ ++# Alias for simple access to check if this host supports cgroups. ++supported = CGroup.supported ++ ++# Alias for simple access to find the cgroup of a certain process. ++find_by_pid = CGroup.find_by_pid +-- +1.8.1.4 + diff --git a/pakfire/patches/0006-cgroup-Don-t-be-verbose-about-killing-processes.patch b/pakfire/patches/0006-cgroup-Don-t-be-verbose-about-killing-processes.patch new file mode 100644 index 000000000..837819fc7 --- /dev/null +++ b/pakfire/patches/0006-cgroup-Don-t-be-verbose-about-killing-processes.patch @@ -0,0 +1,25 @@ +From 337a733a278a81c1ad44f9d1fed65ec6581dfa44 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Sun, 10 Mar 2013 01:22:19 +0100 +Subject: [PATCH 06/19] cgroup: Don't be verbose about killing processes. + +--- + python/pakfire/cgroup.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/python/pakfire/cgroup.py b/python/pakfire/cgroup.py +index 6c85937..207f270 100644 +--- a/python/pakfire/cgroup.py ++++ b/python/pakfire/cgroup.py +@@ -316,7 +316,7 @@ class CGroup(object): + + if sig: + # Send sig to all processes in the cgroup. +- log.info("Sending signal %s to all processes in '%s'." % (sig, self.name)) ++ log.debug("Sending signal %s to all processes in '%s'." % (sig, self.name)) + self.kill(sig=sig, recursive=True) + + # Sleep for 200ms. +-- +1.8.1.4 + diff --git a/pakfire/patches/0007-cgroup-Move-builder-process-out-of-cgroup-before-des.patch b/pakfire/patches/0007-cgroup-Move-builder-process-out-of-cgroup-before-des.patch new file mode 100644 index 000000000..4b93d8561 --- /dev/null +++ b/pakfire/patches/0007-cgroup-Move-builder-process-out-of-cgroup-before-des.patch @@ -0,0 +1,35 @@ +From 3503b1493c09bbcca6d8dff203126703a7fc5822 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Sun, 10 Mar 2013 01:22:37 +0100 +Subject: [PATCH 07/19] cgroup: Move builder process out of cgroup before + destroying it. + +Before all processes in the cgroup are killed, the builder process +is migrated to the parent group, because kill_and_wait() waits +for nothing because it will never kill its own process. +--- + python/pakfire/builder.py | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/python/pakfire/builder.py b/python/pakfire/builder.py +index 7427a8c..944c1a0 100644 +--- a/python/pakfire/builder.py ++++ b/python/pakfire/builder.py +@@ -200,11 +200,13 @@ class BuildEnviron(object): + + def stop(self): + if self.cgroup: ++ # Move the builder process out of the cgroup. ++ self.cgroup.migrate_task(self.cgroup.parent, os.getpid()) ++ + # Kill all still running processes in the cgroup. + self.cgroup.kill_and_wait() + + # Remove cgroup and all parent cgroups if they are empty. +- self.cgroup.migrate_task(self.cgroup.root, os.getpid()) + self.cgroup.destroy() + + parent = self.cgroup.parent +-- +1.8.1.4 + diff --git a/pakfire/patches/0008-transport-Catch-503-Bad-Gateway-HTTP-errors.patch b/pakfire/patches/0008-transport-Catch-503-Bad-Gateway-HTTP-errors.patch new file mode 100644 index 000000000..c8e79ab01 --- /dev/null +++ b/pakfire/patches/0008-transport-Catch-503-Bad-Gateway-HTTP-errors.patch @@ -0,0 +1,41 @@ +From 3577457e20575bc2feff3f012f83c52fff85b9ac Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Sat, 30 Mar 2013 12:18:39 +0000 +Subject: [PATCH 08/19] transport: Catch 503 Bad Gateway HTTP errors. + +--- + python/pakfire/errors.py | 4 ++++ + python/pakfire/transport.py | 2 ++ + 2 files changed, 6 insertions(+) + +diff --git a/python/pakfire/errors.py b/python/pakfire/errors.py +index 87c64e3..2da3822 100644 +--- a/python/pakfire/errors.py ++++ b/python/pakfire/errors.py +@@ -102,6 +102,10 @@ class TransportError(Error): + pass + + ++class TransportBadGatewayError(TransportError): ++ pass ++ ++ + class TransportConnectionError(TransportError): + pass + +diff --git a/python/pakfire/transport.py b/python/pakfire/transport.py +index 1adb1c6..f301652 100644 +--- a/python/pakfire/transport.py ++++ b/python/pakfire/transport.py +@@ -260,6 +260,8 @@ class PakfireHubTransport(object): + raise TransportNotFoundError, url + elif e.code == 500: + raise TransportInternalServerError, url ++ elif e.code == 503: ++ raise TransportBadGatewayError, url + elif e.code == 504: + raise TransportConnectionTimeoutError, url + +-- +1.8.1.4 + diff --git a/pakfire/patches/0009-daemon-Restart-keepalive-daemon-after-it-has-crashed.patch b/pakfire/patches/0009-daemon-Restart-keepalive-daemon-after-it-has-crashed.patch new file mode 100644 index 000000000..f4929eb87 --- /dev/null +++ b/pakfire/patches/0009-daemon-Restart-keepalive-daemon-after-it-has-crashed.patch @@ -0,0 +1,57 @@ +From 11ec9629f460bbd6169e8f934d0194912ff257e0 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Sat, 30 Mar 2013 12:21:29 +0000 +Subject: [PATCH 09/19] daemon: Restart keepalive daemon after it has crashed. + +--- + python/pakfire/daemon.py | 27 +++++++++++++++++++++++++++ + 1 file changed, 27 insertions(+) + +diff --git a/python/pakfire/daemon.py b/python/pakfire/daemon.py +index 8c453e0..267b330 100644 +--- a/python/pakfire/daemon.py ++++ b/python/pakfire/daemon.py +@@ -73,6 +73,10 @@ class PakfireDaemon(object): + while self.__running: + time_started = time.time() + ++ # Check if keepalive process is still alive. ++ if not self.keepalive.is_alive(): ++ self.restart_keepalive(wait=10) ++ + # Spawn a sufficient number of worker processes. + self.spawn_workers_if_needed() + +@@ -96,6 +100,29 @@ class PakfireDaemon(object): + log.info(_("Shutting down...")) + self.__running = False + ++ def restart_keepalive(self, wait=None): ++ log.critial(_("Restarting keepalive process")) ++ ++ # Send SIGTERM to really end the process. ++ self.keepalive.terminate() ++ ++ # Wait for the process to terminate. ++ if wait: ++ self.keepalive.join(wait) ++ ++ # Remove the keepalive process from the process list. ++ try: ++ self.__workers.remove(self.keepalive) ++ except ValueError: ++ pass ++ ++ # Create a new process and start it. ++ self.keepalive = PakfireDaemonKeepalive(self.config) ++ self.keepalive.start() ++ ++ # Add the process to the process list. ++ self.__workers.append(self.keepalive) ++ + def spawn_workers_if_needed(self, *args, **kwargs): + """ + Spawns more workers if needed. +-- +1.8.1.4 + diff --git a/pakfire/patches/0010-Fix-typo.patch b/pakfire/patches/0010-Fix-typo.patch new file mode 100644 index 000000000..c4d332a72 --- /dev/null +++ b/pakfire/patches/0010-Fix-typo.patch @@ -0,0 +1,25 @@ +From 0757c740d0551dc3a0013d55595579a71285c124 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Sun, 7 Apr 2013 14:15:01 +0200 +Subject: [PATCH 10/19] Fix typo. + +--- + python/pakfire/daemon.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/python/pakfire/daemon.py b/python/pakfire/daemon.py +index 267b330..67fdd69 100644 +--- a/python/pakfire/daemon.py ++++ b/python/pakfire/daemon.py +@@ -101,7 +101,7 @@ class PakfireDaemon(object): + self.__running = False + + def restart_keepalive(self, wait=None): +- log.critial(_("Restarting keepalive process")) ++ log.critical(_("Restarting keepalive process")) + + # Send SIGTERM to really end the process. + self.keepalive.terminate() +-- +1.8.1.4 + diff --git a/pakfire/patches/0011-system-Fix-reading-CPU-info.patch b/pakfire/patches/0011-system-Fix-reading-CPU-info.patch new file mode 100644 index 000000000..3601ddd0e --- /dev/null +++ b/pakfire/patches/0011-system-Fix-reading-CPU-info.patch @@ -0,0 +1,41 @@ +From 90c9f2492736ab3a1d9b202da10803a248b8c0bc Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Wed, 8 May 2013 20:31:27 +0000 +Subject: [PATCH 11/19] system: Fix reading CPU info. + +--- + python/pakfire/system.py | 12 +++++++----- + 1 file changed, 7 insertions(+), 5 deletions(-) + +diff --git a/python/pakfire/system.py b/python/pakfire/system.py +index 1f0f6de..a917461 100644 +--- a/python/pakfire/system.py ++++ b/python/pakfire/system.py +@@ -130,17 +130,19 @@ class System(object): + def cpu_model(self): + cpuinfo = self.parse_cpuinfo() + +- ret = None +- if self.arch.startswith("arm"): ++ ret = cpuinfo.get("model name", None) ++ ++ # Some ARM platforms do not provide "model name", so we ++ # try an other way. ++ if ret is None: + try: + ret = "%(Hardware)s - %(Processor)s" % cpuinfo + except KeyError: + pass +- else: +- ret = cpuinfo.get("model name", None) + + # Remove too many spaces. +- ret = " ".join(ret.split()) ++ if ret: ++ ret = " ".join(ret.split()) + + return ret or _("Could not be determined") + +-- +1.8.1.4 + diff --git a/pakfire/patches/0012-buildroot-Only-copy-dev-nodes-if-they-really-exist.patch b/pakfire/patches/0012-buildroot-Only-copy-dev-nodes-if-they-really-exist.patch new file mode 100644 index 000000000..23581f27c --- /dev/null +++ b/pakfire/patches/0012-buildroot-Only-copy-dev-nodes-if-they-really-exist.patch @@ -0,0 +1,30 @@ +From efffd953ca54809f1f7d0bb9269568477c660a6d Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Wed, 8 May 2013 20:50:11 +0000 +Subject: [PATCH 12/19] buildroot: Only copy dev nodes if they really exist. + +--- + python/pakfire/builder.py | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/python/pakfire/builder.py b/python/pakfire/builder.py +index 944c1a0..68c9fd4 100644 +--- a/python/pakfire/builder.py ++++ b/python/pakfire/builder.py +@@ -501,7 +501,12 @@ class BuildEnviron(object): + for node in nodes: + # Stat the original node of the host system and copy it to + # the build chroot. +- node_stat = os.stat(node) ++ try: ++ node_stat = os.stat(node) ++ ++ # If it cannot be found, just go on. ++ except OSError: ++ continue + + self._create_node(node, node_stat.st_mode, node_stat.st_rdev) + +-- +1.8.1.4 + diff --git a/pakfire/patches/0013-progressbar-Remove-overwriting-the-Bar-class.patch b/pakfire/patches/0013-progressbar-Remove-overwriting-the-Bar-class.patch new file mode 100644 index 000000000..2b395c5f8 --- /dev/null +++ b/pakfire/patches/0013-progressbar-Remove-overwriting-the-Bar-class.patch @@ -0,0 +1,45 @@ +From 2b0854638bedee5ef9078be8b3062bfbbd584e7a Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 13 Jun 2013 18:51:10 +0200 +Subject: [PATCH 13/19] progressbar: Remove overwriting the Bar class. + +--- + python/pakfire/util.py | 15 +-------------- + 1 file changed, 1 insertion(+), 14 deletions(-) + +diff --git a/python/pakfire/util.py b/python/pakfire/util.py +index 84128cd..8d91709 100644 +--- a/python/pakfire/util.py ++++ b/python/pakfire/util.py +@@ -79,19 +79,6 @@ def random_string(length=20): + + return s + +- +-class Bar(progressbar.Bar): +- def update(self, pbar, width): +- percent = pbar.percentage() +- if pbar.finished: +- return " " * width +- +- cwidth = width - len(self.left) - len(self.right) +- marked_width = int(percent * cwidth / 100) +- m = self._format_marker(pbar) +- bar = (self.left + (m*marked_width).ljust(cwidth) + self.right) +- return bar +- + def make_progress(message, maxval, eta=True, speed=False): + # Return nothing if stdout is not a terminal. + if not sys.stdout.isatty(): +@@ -101,7 +88,7 @@ def make_progress(message, maxval, eta=True, speed=False): + " ", + "%s" % message, + " ", +- Bar(left="[", right="]"), ++ progressbar.Bar(left="[", right="]"), + " ", + ] + +-- +1.8.1.4 + diff --git a/pakfire/patches/0014-Better-formatting-for-progress-bars.patch b/pakfire/patches/0014-Better-formatting-for-progress-bars.patch new file mode 100644 index 000000000..2527844f6 --- /dev/null +++ b/pakfire/patches/0014-Better-formatting-for-progress-bars.patch @@ -0,0 +1,25 @@ +From 493336ff5c795b6d33d78659a69cc16165e7d0d8 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Thu, 13 Jun 2013 19:18:23 +0200 +Subject: [PATCH 14/19] Better formatting for progress bars. + +--- + python/pakfire/util.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/python/pakfire/util.py b/python/pakfire/util.py +index 8d91709..73c62d0 100644 +--- a/python/pakfire/util.py ++++ b/python/pakfire/util.py +@@ -86,7 +86,7 @@ def make_progress(message, maxval, eta=True, speed=False): + + widgets = [ + " ", +- "%s" % message, ++ "%-64s" % message, + " ", + progressbar.Bar(left="[", right="]"), + " ", +-- +1.8.1.4 + diff --git a/pakfire/patches/0015-transport-Handle-503-in-the-same-way-as-502.patch b/pakfire/patches/0015-transport-Handle-503-in-the-same-way-as-502.patch new file mode 100644 index 000000000..fa450673e --- /dev/null +++ b/pakfire/patches/0015-transport-Handle-503-in-the-same-way-as-502.patch @@ -0,0 +1,36 @@ +From bd3e05167686bb559d2439a092c87fe8234b7ae6 Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Sun, 23 Jun 2013 18:03:58 +0200 +Subject: [PATCH 15/19] transport: Handle 503 in the same way as 502. + +--- + python/pakfire/transport.py | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/python/pakfire/transport.py b/python/pakfire/transport.py +index f301652..982f540 100644 +--- a/python/pakfire/transport.py ++++ b/python/pakfire/transport.py +@@ -260,7 +260,7 @@ class PakfireHubTransport(object): + raise TransportNotFoundError, url + elif e.code == 500: + raise TransportInternalServerError, url +- elif e.code == 503: ++ elif e.code in (502, 503): + raise TransportBadGatewayError, url + elif e.code == 504: + raise TransportConnectionTimeoutError, url +@@ -278,8 +278,8 @@ class PakfireHubTransport(object): + try: + return self.one_request(url, **kwargs) + +- # 500 - Internal Server Error +- except TransportInternalServerError, e: ++ # 500 - Internal Server Error, 502 + 503 Bad Gateway Error ++ except (TransportInternalServerError, TransportBadGateway), e: + log.exception("%s" % e.__class__.__name__) + + # Wait a minute before trying again. +-- +1.8.1.4 + diff --git a/pakfire/patches/0016-Add-systemd-service-file-for-pakfire-daemon.patch b/pakfire/patches/0016-Add-systemd-service-file-for-pakfire-daemon.patch new file mode 100644 index 000000000..1687e967c --- /dev/null +++ b/pakfire/patches/0016-Add-systemd-service-file-for-pakfire-daemon.patch @@ -0,0 +1,50 @@ +From 74ffafed27a27f12511274f2e6cf2c5b26ec8d5c Mon Sep 17 00:00:00 2001 +From: Stefan Schantl +Date: Sun, 23 Jun 2013 16:42:19 +0200 +Subject: [PATCH 16/19] Add systemd service file for pakfire-daemon. + +--- + Makefile | 4 ++++ + systemd/pakfire-daemon.service | 15 +++++++++++++++ + 2 files changed, 19 insertions(+) + create mode 100644 systemd/pakfire-daemon.service + +diff --git a/Makefile b/Makefile +index c873463..00ea0fa 100644 +--- a/Makefile ++++ b/Makefile +@@ -38,6 +38,10 @@ install: build + cp -rvf examples/$${file} $(DESTDIR)/etc/pakfire/; \ + done + ++ # Install systemd file. ++ -mkdir -pv $(DESTDIR)/usr/lib/systemd/system ++ cp -vf systemd/*.systemd $(DESTDIR)/usr/lib/systemd/system ++ + .PHONY: check + check: all + PYTHONPATH=python/src/ pylint -E python/pakfire +diff --git a/systemd/pakfire-daemon.service b/systemd/pakfire-daemon.service +new file mode 100644 +index 0000000..f4c8f2b +--- /dev/null ++++ b/systemd/pakfire-daemon.service +@@ -0,0 +1,15 @@ ++[Unit] ++Description=Pakfire Daemon ++After=network.target ++Requires=network.target ++ ++[Service] ++KillMode=process ++SendSIGKILL=false ++TimeoutStopSec=0 ++ ++ExecStart=/usr/bin/pakfire-daemon ++Restart=on-failure ++ ++[Install] ++WantedBy=multi-user.target +-- +1.8.1.4 + diff --git a/pakfire/patches/0017-Warn-if-the-host-key-is-not-in-the-key-store.patch b/pakfire/patches/0017-Warn-if-the-host-key-is-not-in-the-key-store.patch new file mode 100644 index 000000000..52471f67c --- /dev/null +++ b/pakfire/patches/0017-Warn-if-the-host-key-is-not-in-the-key-store.patch @@ -0,0 +1,37 @@ +From 60c7a320b14c490487ae4105ccb125a190de828d Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Mon, 24 Jun 2013 21:24:49 +0000 +Subject: [PATCH 17/19] Warn if the host key is not in the key store. + +--- + python/pakfire/keyring.py | 8 +++++--- + 1 file changed, 5 insertions(+), 3 deletions(-) + +diff --git a/python/pakfire/keyring.py b/python/pakfire/keyring.py +index 8fe052e..7bd54fb 100644 +--- a/python/pakfire/keyring.py ++++ b/python/pakfire/keyring.py +@@ -71,15 +71,17 @@ class Keyring(object): + os.chmod(filename, 600) + + def dump_key(self, keyfp): +- ret = [] +- + key = self.get_key(keyfp, secret=False) +- key_priv = self.get_key(keyfp, secret=True) ++ if not key: ++ return [" " + _("Not in key store: %s") % keyfp, ""] + ++ ret = [] + for uid in key.uids: + ret.append(uid.uid) + + ret.append(" " + _("Fingerprint: %s") % keyfp) ++ ++ key_priv = self.get_key(keyfp, secret=True) + if key_priv: + ret.append(" " + _("Private key available!")) + ret.append("") +-- +1.8.1.4 + diff --git a/pakfire/patches/0018-Fix-installing-unit-files.patch b/pakfire/patches/0018-Fix-installing-unit-files.patch new file mode 100644 index 000000000..7e8e37f7b --- /dev/null +++ b/pakfire/patches/0018-Fix-installing-unit-files.patch @@ -0,0 +1,40 @@ +From cf1301293b1eea70faa2dc47f4c788aa7e52960c Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Mon, 24 Jun 2013 21:34:45 +0000 +Subject: [PATCH 18/19] Fix installing unit files. + +--- + Makeconfig | 3 +++ + Makefile | 5 ++++- + 2 files changed, 7 insertions(+), 1 deletion(-) + +diff --git a/Makeconfig b/Makeconfig +index b9f7898..ec6a2f2 100644 +--- a/Makeconfig ++++ b/Makeconfig +@@ -50,3 +50,6 @@ TOP := $(dir $(lastword $(MAKEFILE_LIST))) + # A list of all files that contain translations and need to + # be indexed. + TRANS_FILES = $(PYTHON_FILES) python/src/*.c ++ ++# systemd unit files ++UNIT_FILES = $(wildcard systemd/*.service) +diff --git a/Makefile b/Makefile +index 00ea0fa..2a1497b 100644 +--- a/Makefile ++++ b/Makefile +@@ -40,7 +40,10 @@ install: build + + # Install systemd file. + -mkdir -pv $(DESTDIR)/usr/lib/systemd/system +- cp -vf systemd/*.systemd $(DESTDIR)/usr/lib/systemd/system ++ for file in $(UNIT_FILES); do \ ++ install -v -m 644 $${file} \ ++ $(DESTDIR)/usr/lib/systemd/system || exit 1; \ ++ done + + .PHONY: check + check: all +-- +1.8.1.4 + diff --git a/pakfire/patches/0019-Fix-typo-in-exception-name.patch b/pakfire/patches/0019-Fix-typo-in-exception-name.patch new file mode 100644 index 000000000..82715f727 --- /dev/null +++ b/pakfire/patches/0019-Fix-typo-in-exception-name.patch @@ -0,0 +1,25 @@ +From 84369401ab3b69541f6b763f050a2a1940cf351d Mon Sep 17 00:00:00 2001 +From: Michael Tremer +Date: Mon, 24 Jun 2013 21:34:55 +0000 +Subject: [PATCH 19/19] Fix typo in exception name. + +--- + python/pakfire/transport.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/python/pakfire/transport.py b/python/pakfire/transport.py +index 982f540..3dbf991 100644 +--- a/python/pakfire/transport.py ++++ b/python/pakfire/transport.py +@@ -279,7 +279,7 @@ class PakfireHubTransport(object): + return self.one_request(url, **kwargs) + + # 500 - Internal Server Error, 502 + 503 Bad Gateway Error +- except (TransportInternalServerError, TransportBadGateway), e: ++ except (TransportInternalServerError, TransportBadGatewayError), e: + log.exception("%s" % e.__class__.__name__) + + # Wait a minute before trying again. +-- +1.8.1.4 + -- 2.47.3