PYTHON_CC = $(CC) -pthread -fPIC
PYTHON_CFLAGS = $(shell python-config --cflags)
-PYTHON_MODULES = pakfire pakfire/packages pakfire/repository
+PYTHON_MODULES = pakfire pakfire/client pakfire/packages pakfire/repository
ifeq "$(DEBIAN)" "1"
PYTHON_DIR = $(LIBDIR)/python$(PYTHON_VERSION)/dist-packages
else
# Don't overwrite already installed configuration file.
[ -e "$(DESTDIR)/etc/pakfire.conf" ] || \
cp -vf examples/pakfire.conf $(DESTDIR)/etc/pakfire.conf
+ [ -e "$(DESTDIR)/etc/pakfire-client.conf" ] || \
+ cp -vf examples/pakfire-client.conf $(DESTDIR)/etc/pakfire-client.conf
+ [ -e "$(DESTDIR)/etc/pakfire-daemon.conf" ] || \
+ cp -vf examples/pakfire-daemon.conf $(DESTDIR)/etc/pakfire-daemon.conf
cp -vf examples/pakfire.repos.d/* $(DESTDIR)/etc/pakfire.repos.d/
.PHONY: check
--- /dev/null
+
+# Configure the pakfire client.
+[client]
+
+# The URL of the server to connect to.
+# server = https://pakfire.ipfire.org/
+
+# Your credentials to log in on the hub.
+# username = ipfire
+# password = 1234...
--- /dev/null
+
+# Configure the pakfire daemon.
+[daemon]
+
+# The URL of the server to connect to.
+# server = https://pakfire.ipfire.org/
+
+# The hostname of this machine.
+# hostname = <automatically detected>
+
+# The authentication secret that is used
+# to identify this host against the pakfire
+# build service.
+# secret = 1234...
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2011-12-23 21:12+0100\n"
+"POT-Creation-Date: 2012-01-26 22:46+0100\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
msgstr ""
#. Log the package information.
-#: ../python/pakfire/builder.py:153
+#: ../python/pakfire/builder.py:155
msgid "Package information:"
msgstr ""
#. Install all packages.
-#: ../python/pakfire/builder.py:324
+#: ../python/pakfire/builder.py:326
msgid "Install packages needed for build..."
msgstr ""
-#: ../python/pakfire/builder.py:329
+#: ../python/pakfire/builder.py:331
msgid "Extracting"
msgstr ""
-#: ../python/pakfire/builder.py:582
+#: ../python/pakfire/builder.py:600
msgid "You cannot run a build when no package was given."
msgstr ""
-#: ../python/pakfire/builder.py:587
+#: ../python/pakfire/builder.py:605
#, python-format
msgid "Could not find makefile in build root: %s"
msgstr ""
-#: ../python/pakfire/builder.py:601
+#: ../python/pakfire/builder.py:619
msgid "The build command failed. See logfile for details."
msgstr ""
#. Walk through the whole tree and collect all files
#. that are on the same disk (not crossing mountpoints).
-#: ../python/pakfire/builder.py:659
+#: ../python/pakfire/builder.py:677
msgid "Creating filelist..."
msgstr ""
#. Create a nice progressbar.
-#: ../python/pakfire/builder.py:678
+#: ../python/pakfire/builder.py:696
msgid "Compressing files..."
msgstr ""
-#: ../python/pakfire/builder.py:697
+#: ../python/pakfire/builder.py:715
#, python-format
msgid "Cache file was successfully created at %s."
msgstr ""
-#: ../python/pakfire/builder.py:698
+#: ../python/pakfire/builder.py:716
#, python-format
msgid " Containing %(files)s files, it has a size of %(size)s."
msgstr ""
#. Make a nice progress bar as always.
-#: ../python/pakfire/builder.py:709
+#: ../python/pakfire/builder.py:727
msgid "Extracting files..."
msgstr ""
#. Update all packages.
-#: ../python/pakfire/builder.py:729
+#: ../python/pakfire/builder.py:747
msgid "Updating packages from cache..."
msgstr ""
#. Package the result.
#. Make all these little package from the build environment.
-#: ../python/pakfire/builder.py:857
+#: ../python/pakfire/builder.py:875
msgid "Creating packages:"
msgstr ""
#. Execute the buildscript of this stage.
-#: ../python/pakfire/builder.py:877
+#: ../python/pakfire/builder.py:895
#, python-format
msgid "Running stage %s:"
msgstr ""
-#: ../python/pakfire/builder.py:895
+#: ../python/pakfire/builder.py:913
#, python-format
msgid "Could not remove static libraries: %s"
msgstr ""
-#: ../python/pakfire/builder.py:901
+#: ../python/pakfire/builder.py:919
msgid "Compressing man pages did not complete successfully."
msgstr ""
-#: ../python/pakfire/builder.py:921
+#: ../python/pakfire/builder.py:939
msgid "Extracting debuginfo did not complete with success. Aborting build."
msgstr ""
-#: ../python/pakfire/cli.py:43
+#: ../python/pakfire/cli.py:47
msgid "Pakfire command line interface."
msgstr ""
-#: ../python/pakfire/cli.py:50
+#: ../python/pakfire/cli.py:54
msgid "The path where pakfire should operate in."
msgstr ""
-#: ../python/pakfire/cli.py:117
+#: ../python/pakfire/cli.py:121
msgid "Enable verbose output."
msgstr ""
-#: ../python/pakfire/cli.py:120
+#: ../python/pakfire/cli.py:124
msgid "Path to a configuration file to load."
msgstr ""
-#: ../python/pakfire/cli.py:123
+#: ../python/pakfire/cli.py:128
msgid "Disable a repository temporarily."
msgstr ""
-#: ../python/pakfire/cli.py:126
+#: ../python/pakfire/cli.py:131
msgid "Enable a repository temporarily."
msgstr ""
-#: ../python/pakfire/cli.py:129
+#: ../python/pakfire/cli.py:135
msgid "Run pakfire in offline mode."
msgstr ""
-#: ../python/pakfire/cli.py:134
+#: ../python/pakfire/cli.py:140
msgid "Install one or more packages to the system."
msgstr ""
-#: ../python/pakfire/cli.py:136
+#: ../python/pakfire/cli.py:142
msgid "Give name of at least one package to install."
msgstr ""
-#: ../python/pakfire/cli.py:142
+#: ../python/pakfire/cli.py:148
msgid "Install one or more packages from the filesystem."
msgstr ""
-#: ../python/pakfire/cli.py:144
+#: ../python/pakfire/cli.py:150
msgid "Give filename of at least one package."
msgstr ""
-#: ../python/pakfire/cli.py:150
+#: ../python/pakfire/cli.py:156
msgid "Reinstall one or more packages."
msgstr ""
-#: ../python/pakfire/cli.py:152
+#: ../python/pakfire/cli.py:158
msgid "Give name of at least one package to reinstall."
msgstr ""
-#: ../python/pakfire/cli.py:158
+#: ../python/pakfire/cli.py:164
msgid "Remove one or more packages from the system."
msgstr ""
-#: ../python/pakfire/cli.py:160
+#: ../python/pakfire/cli.py:166
msgid "Give name of at least one package to remove."
msgstr ""
-#: ../python/pakfire/cli.py:166
+#: ../python/pakfire/cli.py:172
msgid "Give a name of a package to update or leave emtpy for all."
msgstr ""
-#: ../python/pakfire/cli.py:168
+#: ../python/pakfire/cli.py:174
msgid "Exclude package from update."
msgstr ""
-#: ../python/pakfire/cli.py:170 ../python/pakfire/cli.py:195
+#: ../python/pakfire/cli.py:176 ../python/pakfire/cli.py:201
msgid "Allow changing the vendor of packages."
msgstr ""
-#: ../python/pakfire/cli.py:172 ../python/pakfire/cli.py:197
+#: ../python/pakfire/cli.py:178 ../python/pakfire/cli.py:203
msgid "Allow changing the architecture of packages."
msgstr ""
-#: ../python/pakfire/cli.py:177
+#: ../python/pakfire/cli.py:183
msgid "Update the whole system or one specific package."
msgstr ""
-#: ../python/pakfire/cli.py:184
+#: ../python/pakfire/cli.py:190
msgid "Check, if there are any updates available."
msgstr ""
-#: ../python/pakfire/cli.py:191
+#: ../python/pakfire/cli.py:197
msgid "Downgrade one or more packages."
msgstr ""
-#: ../python/pakfire/cli.py:193
+#: ../python/pakfire/cli.py:199
msgid "Give a name of a package to downgrade."
msgstr ""
-#: ../python/pakfire/cli.py:203
+#: ../python/pakfire/cli.py:209
msgid "Print some information about the given package(s)."
msgstr ""
-#: ../python/pakfire/cli.py:205
+#: ../python/pakfire/cli.py:211
msgid "Give at least the name of one package."
msgstr ""
-#: ../python/pakfire/cli.py:211
+#: ../python/pakfire/cli.py:217
msgid "Search for a given pattern."
msgstr ""
-#: ../python/pakfire/cli.py:213
+#: ../python/pakfire/cli.py:219
msgid "A pattern to search for."
msgstr ""
-#: ../python/pakfire/cli.py:219
+#: ../python/pakfire/cli.py:225
msgid "Get a list of packages that provide a given file or feature."
msgstr ""
-#: ../python/pakfire/cli.py:221
+#: ../python/pakfire/cli.py:227
msgid "File or feature to search for."
msgstr ""
-#: ../python/pakfire/cli.py:227
+#: ../python/pakfire/cli.py:233
msgid "Get list of packages that belong to the given group."
msgstr ""
-#: ../python/pakfire/cli.py:229
+#: ../python/pakfire/cli.py:235
msgid "Group name to search for."
msgstr ""
-#: ../python/pakfire/cli.py:235
+#: ../python/pakfire/cli.py:241
msgid "Install all packages that belong to the given group."
msgstr ""
-#: ../python/pakfire/cli.py:237
+#: ../python/pakfire/cli.py:243
msgid "Group name."
msgstr ""
-#: ../python/pakfire/cli.py:243
+#: ../python/pakfire/cli.py:249
msgid "List all currently enabled repositories."
msgstr ""
-#: ../python/pakfire/cli.py:247
+#: ../python/pakfire/cli.py:253
msgid "Cleanup commands."
msgstr ""
-#: ../python/pakfire/cli.py:255
+#: ../python/pakfire/cli.py:261
msgid "Cleanup all temporary files."
msgstr ""
-#: ../python/pakfire/cli.py:261
+#: ../python/pakfire/cli.py:267
msgid "Check the system for any errors."
msgstr ""
-#: ../python/pakfire/cli.py:267
+#: ../python/pakfire/cli.py:273
msgid "Check the dependencies for a particular package."
msgstr ""
-#: ../python/pakfire/cli.py:269
+#: ../python/pakfire/cli.py:275
msgid "Give name of at least one package to check."
msgstr ""
-#: ../python/pakfire/cli.py:348 ../python/pakfire/transaction.py:352
+#: ../python/pakfire/cli.py:351 ../python/pakfire/transaction.py:352
msgid "Repository"
msgstr ""
-#: ../python/pakfire/cli.py:348
+#: ../python/pakfire/cli.py:351
msgid "Enabled"
msgstr ""
-#: ../python/pakfire/cli.py:348
+#: ../python/pakfire/cli.py:351
msgid "Priority"
msgstr ""
-#: ../python/pakfire/cli.py:348
+#: ../python/pakfire/cli.py:351
msgid "Packages"
msgstr ""
-#: ../python/pakfire/cli.py:360
+#: ../python/pakfire/cli.py:363
msgid "Cleaning up everything..."
msgstr ""
-#: ../python/pakfire/cli.py:376
+#: ../python/pakfire/cli.py:379
msgid "You cannot run pakfire-builder in a pakfire chroot."
msgstr ""
-#: ../python/pakfire/cli.py:379 ../python/pakfire/cli.py:688
+#: ../python/pakfire/cli.py:382 ../python/pakfire/cli.py:719
msgid "Pakfire builder command line interface."
msgstr ""
-#: ../python/pakfire/cli.py:437
+#: ../python/pakfire/cli.py:440
msgid "Update the package indexes."
msgstr ""
-#: ../python/pakfire/cli.py:443 ../python/pakfire/cli.py:708
+#: ../python/pakfire/cli.py:446 ../python/pakfire/cli.py:739
msgid "Build one or more packages."
msgstr ""
-#: ../python/pakfire/cli.py:445 ../python/pakfire/cli.py:710
+#: ../python/pakfire/cli.py:448 ../python/pakfire/cli.py:633
+#: ../python/pakfire/cli.py:741
msgid "Give name of at least one package to build."
msgstr ""
-#: ../python/pakfire/cli.py:449 ../python/pakfire/cli.py:714
+#: ../python/pakfire/cli.py:452 ../python/pakfire/cli.py:745
+#: ../python/pakfire/cli.py:815
msgid "Build the package for the given architecture."
msgstr ""
-#: ../python/pakfire/cli.py:451 ../python/pakfire/cli.py:479
-#: ../python/pakfire/cli.py:716
+#: ../python/pakfire/cli.py:454 ../python/pakfire/cli.py:482
+#: ../python/pakfire/cli.py:747
msgid "Path were the output files should be copied to."
msgstr ""
-#: ../python/pakfire/cli.py:453 ../python/pakfire/cli.py:468
-#: ../python/pakfire/cli.py:718
+#: ../python/pakfire/cli.py:456 ../python/pakfire/cli.py:471
+#: ../python/pakfire/cli.py:749
msgid "Mode to run in. Is either 'release' or 'development' (default)."
msgstr ""
-#: ../python/pakfire/cli.py:455
+#: ../python/pakfire/cli.py:458
msgid "Run a shell after a successful build."
msgstr ""
-#: ../python/pakfire/cli.py:460
+#: ../python/pakfire/cli.py:463
msgid "Go into a shell."
msgstr ""
-#: ../python/pakfire/cli.py:462
+#: ../python/pakfire/cli.py:465
msgid "Give name of a package."
msgstr ""
-#: ../python/pakfire/cli.py:466
+#: ../python/pakfire/cli.py:469
msgid "Emulated architecture in the shell."
msgstr ""
-#: ../python/pakfire/cli.py:473
+#: ../python/pakfire/cli.py:476
msgid "Generate a source package."
msgstr ""
-#: ../python/pakfire/cli.py:475
+#: ../python/pakfire/cli.py:478
msgid "Give name(s) of a package(s)."
msgstr ""
-#: ../python/pakfire/cli.py:484
+#: ../python/pakfire/cli.py:487
msgid "Create a build environment cache."
msgstr ""
-#: ../python/pakfire/cli.py:494
+#: ../python/pakfire/cli.py:497
msgid "Create a new build environment cache."
msgstr ""
-#: ../python/pakfire/cli.py:499
+#: ../python/pakfire/cli.py:502
msgid "Remove all cached build environments."
msgstr ""
-#: ../python/pakfire/cli.py:577
+#: ../python/pakfire/cli.py:580
#, python-format
msgid "Removing environment cache file: %s..."
msgstr ""
-#: ../python/pakfire/cli.py:583
+#: ../python/pakfire/cli.py:586
#, python-format
msgid "Could not remove file: %s"
msgstr ""
-#: ../python/pakfire/cli.py:589
+#: ../python/pakfire/cli.py:592
msgid "Pakfire server command line interface."
msgstr ""
-#: ../python/pakfire/cli.py:628
-msgid "Request a build job from the server."
+#: ../python/pakfire/cli.py:631
+msgid "Send a scrach build job to the server."
msgstr ""
-#: ../python/pakfire/cli.py:634
-msgid "Send a keepalive to the server."
+#: ../python/pakfire/cli.py:635
+msgid "Limit build to only these architecture(s)."
msgstr ""
#: ../python/pakfire/cli.py:641
+msgid "Send a keepalive to the server."
+msgstr ""
+
+#: ../python/pakfire/cli.py:648
msgid "Update all repositories."
msgstr ""
-#: ../python/pakfire/cli.py:647
+#: ../python/pakfire/cli.py:654
msgid "Repository management commands."
msgstr ""
-#: ../python/pakfire/cli.py:655
+#: ../python/pakfire/cli.py:662
msgid "Create a new repository index."
msgstr ""
-#: ../python/pakfire/cli.py:656
+#: ../python/pakfire/cli.py:663
msgid "Path to the packages."
msgstr ""
-#: ../python/pakfire/cli.py:657
+#: ../python/pakfire/cli.py:664
msgid "Path to input packages."
msgstr ""
-#: ../python/pakfire/cli.py:662
+#: ../python/pakfire/cli.py:669
msgid "Dump some information about this machine."
msgstr ""
-#: ../python/pakfire/cli.py:720
+#: ../python/pakfire/cli.py:751
msgid "Do not verify build dependencies."
msgstr ""
+#: ../python/pakfire/cli.py:775
+msgid "Pakfire client command line interface."
+msgstr ""
+
+#: ../python/pakfire/cli.py:809
+msgid "Build a package remotely."
+msgstr ""
+
+#: ../python/pakfire/cli.py:811
+msgid "Give name of a package to build."
+msgstr ""
+
+#: ../python/pakfire/cli.py:820
+msgid "Print some information about this host."
+msgstr ""
+
+#: ../python/pakfire/cli.py:826
+msgid "Check the connection to the hub."
+msgstr ""
+
+#: ../python/pakfire/cli.py:853 ../python/pakfire/server.py:302
+msgid "Hostname"
+msgstr ""
+
+#: ../python/pakfire/cli.py:854
+msgid "Pakfire hub"
+msgstr ""
+
+#: ../python/pakfire/cli.py:857
+msgid "Username"
+msgstr ""
+
+#. Hardware information
+#: ../python/pakfire/cli.py:861 ../python/pakfire/server.py:306
+msgid "Hardware information"
+msgstr ""
+
+#: ../python/pakfire/cli.py:862 ../python/pakfire/server.py:307
+msgid "CPU model"
+msgstr ""
+
+#: ../python/pakfire/cli.py:863 ../python/pakfire/server.py:308
+msgid "Memory"
+msgstr ""
+
+#: ../python/pakfire/cli.py:865 ../python/pakfire/server.py:310
+msgid "Native arch"
+msgstr ""
+
+#: ../python/pakfire/cli.py:867 ../python/pakfire/server.py:312
+msgid "Supported arches"
+msgstr ""
+
+#: ../python/pakfire/cli.py:880
+msgid "Your IP address"
+msgstr ""
+
+#: ../python/pakfire/cli.py:885
+msgid "You are authenticated to the build service:"
+msgstr ""
+
+#: ../python/pakfire/cli.py:891
+msgid "User name"
+msgstr ""
+
+#: ../python/pakfire/cli.py:892
+msgid "Real name"
+msgstr ""
+
+#: ../python/pakfire/cli.py:893
+msgid "Email address"
+msgstr ""
+
+#: ../python/pakfire/cli.py:894
+msgid "Registered"
+msgstr ""
+
+#: ../python/pakfire/cli.py:901
+msgid "You could not be authenticated to the build service."
+msgstr ""
+
+#: ../python/pakfire/cli.py:910
+msgid "Pakfire daemon command line interface."
+msgstr ""
+
+#: ../python/pakfire/client/transport.py:55
+#, python-format
+msgid "Socket error: %s"
+msgstr ""
+
+#: ../python/pakfire/client/transport.py:79
+#, python-format
+msgid "Trying again in %s seconds. %s tries left."
+msgstr ""
+
#: ../python/pakfire/compress.py:85 ../python/pakfire/compress.py:95
#, python-format
msgid "Given algorithm '%s' is not supported."
msgstr ""
+#. Parse the file.
+#: ../python/pakfire/config.py:256
+#, python-format
+msgid "Reading from configuration file: %s"
+msgstr ""
+
+#: ../python/pakfire/config.py:306
+msgid "Configuration:"
+msgstr ""
+
+#: ../python/pakfire/config.py:308
+#, python-format
+msgid "Section: %s"
+msgstr ""
+
+#: ../python/pakfire/config.py:313
+msgid "No settings in this section."
+msgstr ""
+
+#: ../python/pakfire/config.py:315
+msgid "Loaded from files:"
+msgstr ""
+
#: ../python/pakfire/downloader.py:134
msgid "Downloading source files:"
msgstr ""
msgid "File"
msgstr ""
-#: ../python/pakfire/packages/base.py:361
+#: ../python/pakfire/packages/base.py:362
msgid "Not set"
msgstr ""
-#: ../python/pakfire/packages/base.py:526
+#: ../python/pakfire/packages/base.py:527
#, python-format
msgid "Config file saved as %s."
msgstr ""
msgid "Could not remove file: /%s"
msgstr ""
-#: ../python/pakfire/packages/make.py:78
+#: ../python/pakfire/packages/make.py:79
msgid "Package name is undefined."
msgstr ""
-#: ../python/pakfire/packages/make.py:81
+#: ../python/pakfire/packages/make.py:82
msgid "Package version is undefined."
msgstr ""
-#: ../python/pakfire/packages/make.py:407
+#: ../python/pakfire/packages/make.py:416
#, python-format
msgid "Searching for automatic dependencies for %s..."
msgstr ""
+#: ../python/pakfire/packages/make.py:463
+#, python-format
+msgid "Regular experession is invalid and has been skipped: %s"
+msgstr ""
+
+#. Let the user know what has been done.
+#: ../python/pakfire/packages/make.py:479
+#, python-format
+msgid "Filter %s filtered %s."
+msgstr ""
+
#. Load progressbar.
#: ../python/pakfire/packages/packager.py:362
msgid "Packaging"
msgid " Solutions:"
msgstr ""
-#: ../python/pakfire/server.py:267
+#: ../python/pakfire/server.py:278 ../python/pakfire/system.py:114
msgid "Could not be determined"
msgstr ""
-#: ../python/pakfire/server.py:291
-msgid "Hostname"
-msgstr ""
-
-#. Hardware information
-#: ../python/pakfire/server.py:295
-msgid "Hardware information"
-msgstr ""
-
-#: ../python/pakfire/server.py:296
-msgid "CPU model"
-msgstr ""
-
-#: ../python/pakfire/server.py:297
-msgid "Memory"
-msgstr ""
-
-#: ../python/pakfire/server.py:299
-msgid "Native arch"
-msgstr ""
-
-#: ../python/pakfire/server.py:301
-msgid "Supported arches"
-msgstr ""
-
#: ../python/pakfire/transaction.py:91
#, python-format
msgid "file %s from %s conflicts with file from package %s"
msgid "The error that lead to this:"
msgstr ""
-#: ../tools/pakfire-multicall.py:67
+#: ../tools/pakfire-multicall.py:69
msgid "An error has occured when running Pakfire."
msgstr ""
-#: ../tools/pakfire-multicall.py:70
+#: ../tools/pakfire-multicall.py:72
msgid "Error message:"
msgstr ""
-#: ../tools/pakfire-multicall.py:74
+#: ../tools/pakfire-multicall.py:76
msgid "Further description:"
msgstr ""
###############################################################################
import base
+import client
from errors import *
return pakfire.grouplist(group)
def _build(pkg, resultdir, **kwargs):
- pakfire = Pakfire(**kwargs)
+ pakfire = Pakfire(mode="builder", **kwargs)
return pakfire._build(pkg, resultdir, **kwargs)
import logging
log = logging.getLogger("pakfire")
+from system import system
from constants import *
from i18n import _
from errors import BuildError, BuildRootLocked, Error
| __/ (_| | <| _| | | | __/ | |_) | |_| | | | (_| | __/ |
|_| \__,_|_|\_\_| |_|_| \___| |_.__/ \__,_|_|_|\__,_|\___|_|
- Time : %(time)s
- Host : %(host)s
Version : %(version)s
+ Host : %(hostname)s (%(host_arch)s)
+ Time : %(time)s
"""
# Setup the logging.
if logfile:
- self.log = logging.getLogger(self.build_id)
+ self.log = log.getChild(self.build_id)
# Propage everything to the root logger that we will see something
# on the terminal.
self.log.propagate = 1
# are running in release mode.
if self.mode == "release":
logdata = {
- "host" : socket.gethostname(),
- "time" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
- "version" : "Pakfire %s" % PAKFIRE_VERSION,
+ "host_arch" : system.arch,
+ "hostname" : system.hostname,
+ "time" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
+ "version" : "Pakfire %s" % PAKFIRE_VERSION,
}
for line in BUILD_LOG_HEADER.splitlines():
if not requires:
return
- self.pakfire.install(requires, interactive=False,
- allow_downgrade=True, logger=self.log)
+ try:
+ self.pakfire.install(requires, interactive=False,
+ allow_downgrade=True, logger=self.log)
+
+ # Catch dependency errors and log it.
+ except DependencyError, e:
+ raise
def install_test(self):
pkgs = []
return env
+ @property
+ def installed_packages(self):
+ """
+ Returns an iterator over all installed packages in this build environment.
+ """
+ # Get the repository of all installed packages.
+ repo = self.pakfire.repos.get_repo("@system")
+
+ # Return an iterator over the packages.
+ return iter(repo)
+
def do(self, command, shell=True, personality=None, logger=None, *args, **kwargs):
ret = None
import os
import sys
+import pakfire.api as pakfire
+
+import client
+import config
import logger
import packages
import repository
import server
import util
-import pakfire.api as pakfire
+from system import system
from constants import *
from i18n import _
return ret
- def parse_common_arguments(self):
+ def parse_common_arguments(self, repo_manage_switches=True, offline_switch=True):
self.parser.add_argument("--version", action="version",
version="%(prog)s " + PAKFIRE_VERSION)
self.parser.add_argument("-c", "--config", nargs="?",
help=_("Path to a configuration file to load."))
- self.parser.add_argument("--disable-repo", nargs="*", metavar="REPO",
- help=_("Disable a repository temporarily."))
+ if repo_manage_switches:
+ self.parser.add_argument("--disable-repo", nargs="*", metavar="REPO",
+ help=_("Disable a repository temporarily."))
- self.parser.add_argument("--enabled-repo", nargs="*", metavar="REPO",
- help=_("Enable a repository temporarily."))
+ self.parser.add_argument("--enabled-repo", nargs="*", metavar="REPO",
+ help=_("Enable a repository temporarily."))
- self.parser.add_argument("--offline", action="store_true",
- help=_("Run pakfire in offline mode."))
+ if offline_switch:
+ self.parser.add_argument("--offline", action="store_true",
+ help=_("Run pakfire in offline mode."))
def parse_command_install(self):
# Implement the "install" command.
def run(self):
action = self.args.action
- if not self.action2func.has_key(action):
- raise
-
try:
func = self.action2func[action]
except KeyError:
- raise # XXX catch and return better error message
+ raise Exception, "Unhandled action: %s" % action
return func()
def parse_command_build(self):
# Implement the "build" command.
- sub_keepalive = self.sub_commands.add_parser("build",
- help=_("Request a build job from the server."))
- sub_keepalive.add_argument("action", action="store_const", const="build")
+ sub_build = self.sub_commands.add_parser("build",
+ help=_("Send a scrach build job to the server."))
+ sub_build.add_argument("package", nargs=1,
+ help=_("Give name of at least one package to build."))
+ sub_build.add_argument("--arch", "-a",
+ help=_("Limit build to only these architecture(s)."))
+ sub_build.add_argument("action", action="store_const", const="build")
def parse_command_keepalive(self):
# Implement the "keepalive" command.
self.server.update_info()
def handle_build(self):
- self.server.build_job()
+ # Arch.
+ if self.args.arch:
+ arches = self.args.arch.split()
+
+ (package,) = self.args.package
+
+ self.server.create_scratch_build({})
+ return
+
+ # Temporary folter for source package.
+ tmpdir = "/tmp/pakfire-%s" % util.random_string()
+
+ try:
+ os.makedirs(tmpdir)
+
+ pakfire.dist(package, resultdir=[tmpdir,])
+
+ for file in os.listdir(tmpdir):
+ file = os.path.join(tmpdir, file)
+
+ print file
+
+ finally:
+ if os.path.exists(tmpdir):
+ util.rm(tmpdir)
def handle_repoupdate(self):
self.server.update_repositories()
}
pakfire._build(pkg, builder_mode=self.args.mode,
- distro_config=distro_config, resultdir=self.args.resultdir,
- nodeps=self.args.nodeps, **self.pakfire_args)
+ distro_config=distro_config, resultdir=self.args.resultdir,)
+
+
+class CliClient(Cli):
+ def __init__(self):
+ self.parser = argparse.ArgumentParser(
+ description = _("Pakfire client command line interface."),
+ )
+
+ self.parse_common_arguments(repo_manage_switches=True, offline_switch=True)
+
+ # Add sub-commands.
+ self.sub_commands = self.parser.add_subparsers()
+
+ self.parse_command_build()
+ self.parse_command_connection_check()
+ self.parse_command_info()
+
+ # Finally parse all arguments from the command line and save them.
+ self.args = self.parser.parse_args()
+
+ self.action2func = {
+ "build" : self.handle_build,
+ "conn-check" : self.handle_connection_check,
+ "info" : self.handle_info,
+ }
+
+ # Read configuration for the pakfire client.
+ self.conf = conf = config.ConfigClient()
+
+ # Create connection to pakfire hub.
+ self.client = client.PakfireUserClient(
+ conf.get("client", "server"),
+ conf.get("client", "username"),
+ conf.get("client", "password"),
+ )
+
+ def parse_command_build(self):
+ # Parse "build" command.
+ sub_build = self.sub_commands.add_parser("build",
+ help=_("Build a package remotely."))
+ sub_build.add_argument("package", nargs=1,
+ help=_("Give name of a package to build."))
+ sub_build.add_argument("action", action="store_const", const="build")
+
+ sub_build.add_argument("-a", "--arch",
+ help=_("Build the package for the given architecture."))
+
+ def parse_command_info(self):
+ # Implement the "info" command.
+ sub_info = self.sub_commands.add_parser("info",
+ help=_("Print some information about this host."))
+ sub_info.add_argument("action", action="store_const", const="info")
+
+ def parse_command_connection_check(self):
+ # Implement the "conn-check" command.
+ sub_conn_check = self.sub_commands.add_parser("conn-check",
+ help=_("Check the connection to the hub."))
+ sub_conn_check.add_argument("action", action="store_const", const="conn-check")
+
+ def handle_build(self):
+ (package,) = self.args.package
+
+ # XXX just for now, we do only upload source pfm files.
+ assert os.path.exists(package)
+
+ # Format arches.
+ if self.args.arch:
+ arches = self.args.arch.replace(",", " ")
+ else:
+ arches = None
+
+ # Create a new build on the server.
+ build = self.client.build_create(package, arches=arches)
+
+ # XXX Print the resulting build.
+ print build
+
+ def handle_info(self):
+ ret = []
+
+ ret.append("")
+ ret.append(" PAKFIRE %s" % PAKFIRE_VERSION)
+ ret.append("")
+ ret.append(" %-20s: %s" % (_("Hostname"), system.hostname))
+ ret.append(" %-20s: %s" % (_("Pakfire hub"), self.conf.get("client", "server")))
+ if self.conf.get("client", "username") and self.conf.get("client", "password"):
+ ret.append(" %-20s: %s" % \
+ (_("Username"), self.conf.get("client", "username")))
+ ret.append("")
+
+ # Hardware information
+ ret.append(" %s:" % _("Hardware information"))
+ ret.append(" %-16s: %s" % (_("CPU model"), system.cpu_model))
+ ret.append(" %-16s: %s" % (_("Memory"), util.format_size(system.memory)))
+ ret.append("")
+ ret.append(" %-16s: %s" % (_("Native arch"), system.arch))
+
+ header = _("Supported arches")
+ for arch in system.supported_arches:
+ ret.append(" %-16s: %s" % (header, arch))
+ header = ""
+ ret.append("")
+
+ for line in ret:
+ print line
+
+ def handle_connection_check(self):
+ ret = []
+
+ address = self.client.get_my_address()
+ ret.append(" %-20s: %s" % (_("Your IP address"), address))
+ ret.append("")
+
+ authenticated = self.client.check_auth()
+ if authenticated:
+ ret.append(" %s" % _("You are authenticated to the build service:"))
+
+ user = self.client.get_user_profile()
+ assert user, "Could not fetch user infomation"
+
+ keys = [
+ ("name", _("User name")),
+ ("realname", _("Real name")),
+ ("email", _("Email address")),
+ ("registered", _("Registered")),
+ ]
+
+ for key, desc in keys:
+ ret.append(" %-18s: %s" % (desc, user.get(key)))
+
+ else:
+ ret.append(_("You could not be authenticated to the build service."))
+
+ for line in ret:
+ print line
+
+
+class CliDaemon(Cli):
+ def __init__(self):
+ self.parser = argparse.ArgumentParser(
+ description = _("Pakfire daemon command line interface."),
+ )
+
+ self.parse_common_arguments(repo_manage_switches=True, offline_switch=True)
+
+ # Finally parse all arguments from the command line and save them.
+ self.args = self.parser.parse_args()
+
+ def run(self):
+ """
+ Runs the pakfire daemon with provided settings.
+ """
+ # Read the configuration file for the daemon.
+ conf = config.ConfigDaemon()
+
+ # Create daemon instance.
+ d = pakfire.client.PakfireDaemon(
+ server = conf.get("daemon", "server"),
+ hostname = conf.get("daemon", "hostname"),
+ secret = conf.get("daemon", "secret"),
+ )
+
+ try:
+ d.run()
+
+ # We cannot just kill the daemon, it needs a smooth shutdown.
+ except (SystemExit, KeyboardInterrupt):
+ d.shutdown()
--- /dev/null
+#!/usr/bin/python
+
+from base import PakfireUserClient, PakfireBuilderClient
+from builder import PakfireDaemon, ClientBuilder
--- /dev/null
+#!/usr/bin/python
+
+from __future__ import division
+
+import os
+import socket
+import urlparse
+import xmlrpclib
+
+import pakfire.util
+import pakfire.packages as packages
+from pakfire.system import system
+
+# Local modules.
+import transport
+
+from pakfire.constants import *
+
+import logging
+log = logging.getLogger("pakfire.client")
+
+class PakfireClient(object):
+ type = None
+
+ def __init__(self, server, username, password):
+ self.url = self._join_url(server, username, password)
+
+ # Create a secure XMLRPC connection to the server.
+ self.conn = transport.Connection(self.url)
+
+ def _join_url(self, server, username, password):
+ """
+ Construct a right URL out of the given
+ server, username and password.
+
+ Basicly this just adds the credentials
+ to the URL.
+ """
+ assert self.type
+
+ # Parse the given URL.
+ url = urlparse.urlparse(server)
+ assert url.scheme in ("http", "https")
+
+ # Build new URL.
+ ret = "%s://" % url.scheme
+
+ # Add credentials if provided.
+ if username and password:
+ ret += "%s:%s@" % (username, password)
+
+ # Add host and path components.
+ ret += "%s/pakfirehub/%s" % (url.netloc, self.type)
+
+ return ret
+
+ ### Misc. actions
+
+ def noop(self):
+ """
+ No operation. Just to check if the connection is
+ working. Returns a random number.
+ """
+ return self.conn.noop()
+
+ def get_my_address(self):
+ """
+ Get my own address (as seen by the hub).
+ """
+ return self.conn.get_my_address()
+
+ def get_hub_status(self):
+ """
+ Get some status information about the hub.
+ """
+ return self.conn.get_hub_status()
+
+
+class BuildMixin(object):
+ ### Build actions
+
+ def build_create(self, filename, arches=None, distro=None):
+ """
+ Create a new build on the hub.
+ """
+
+ # Upload the source file to the server.
+ upload_id = self._upload_file(filename)
+
+ # Then create the build.
+ build = self.conn.build_create(upload_id, distro, arches)
+
+ print build
+
+ def _upload_file(self, filename):
+ # Get the hash of the file.
+ hash = pakfire.util.calc_hash1(filename)
+
+ # Get the size of the file.
+ size = os.path.getsize(filename)
+
+ # Get an upload ID from the server.
+ upload_id = self.conn.upload_create(os.path.basename(filename),
+ size, hash)
+
+ try:
+ # Calculate the number of chunks.
+ chunks = (size // CHUNK_SIZE) + 1
+
+ # Cut the file in pieces and upload them one after another.
+ with open(filename) as f:
+ chunk = 0
+ while True:
+ data = f.read(CHUNK_SIZE)
+ if not data:
+ break
+
+ chunk += 1
+ log.info("Uploading chunk %s/%s of %s." % (chunk, chunks,
+ os.path.basename(filename)))
+
+ data = xmlrpclib.Binary(data)
+ self.conn.upload_chunk(upload_id, data)
+
+ # Tell the server, that we finished the upload.
+ ret = self.conn.upload_finished(upload_id)
+
+ except:
+ # If anything goes wrong, try to delete the upload and raise
+ # the exception.
+ self.conn.upload_remove(upload_id)
+
+ raise
+
+ # If the server sends false, something happened with the upload that
+ # could not be recovered.
+ if ret:
+ logging.info("Upload of %s succeeded." % filename)
+ else:
+ logging.error("Upload of %s was not successful." % filename)
+ raise Exception, "Upload failed."
+
+ return upload_id
+
+
+class PakfireUserClient(BuildMixin, PakfireClient):
+ type = "user"
+
+ def check_auth(self):
+ """
+ Check if the user was successfully authenticated.
+ """
+ return self.conn.check_auth()
+
+ def get_user_profile(self):
+ """
+ Get information about the user profile.
+ """
+ return self.conn.get_user_profile()
+
+
+class PakfireBuilderClient(BuildMixin, PakfireClient):
+ type = "builder"
+
+ def send_keepalive(self, overload=None):
+ """
+ Sends a little keepalive to the server and
+ updates the hardware information if the server
+ requests it.
+ """
+ log.debug("Sending keepalive to the hub.")
+
+ # Collect the current loadavg and send it to the hub.
+ loadavg = ", ".join(("%.2f" % round(l, 2) for l in os.getloadavg()))
+
+ needs_update = self.conn.send_keepalive(loadavg, overload)
+
+ if needs_update:
+ log.debug("The hub is requesting an update.")
+ self.send_update()
+
+ def send_update(self):
+ log.info("Sending host information update to hub...")
+
+ self.conn.send_update(
+ # Supported architectures.
+ system.supported_arches,
+
+ # CPU information.
+ system.cpu_model,
+ system.cpu_count,
+
+ # Amount of memory in bytes.
+ system.memory / 1024,
+ )
--- /dev/null
+#!/usr/bin/python
+
+import hashlib
+import multiprocessing
+import os
+import sys
+import tempfile
+import time
+
+import pakfire.api
+import pakfire.builder
+import pakfire.config
+import pakfire.downloader
+import pakfire.system
+import pakfire.util
+from pakfire.system import system
+
+import base
+
+from pakfire.constants import *
+
+import logging
+log = logging.getLogger("pakfire.client")
+
+def fork_builder(*args, **kwargs):
+ """
+ Wrapper that runs ClientBuilder in a new process and catches
+ any exception to report it to the main process.
+ """
+ try:
+ # Create new instance of the builder.
+ cb = ClientBuilder(*args, **kwargs)
+
+ # Run the build:
+ cb.build()
+
+ except Exception, e:
+ # XXX catch the exception and log it.
+ print e
+
+ # End the process with an exit code.
+ sys.exit(1)
+
+
+class PakfireDaemon(object):
+ """
+ The PakfireDaemon class that creates a a new process per build
+ job and also handles the keepalive/abort stuff.
+ """
+ def __init__(self, server, hostname, secret):
+ self.client = base.PakfireBuilderClient(server, hostname, secret)
+ self.conn = self.client.conn
+
+ # Save login data (to create child processes).
+ self.server = server
+ self.hostname = hostname
+ self.__secret = secret
+
+ # A list with all running processes.
+ self.processes = []
+ self.pid2jobid = {}
+
+ # Save when last keepalive was sent.
+ self._last_keepalive = 0
+
+ def run(self, heartbeat=1, max_processes=None):
+ # By default do not start more than two processes per CPU core.
+ if max_processes is None:
+ max_processes = system.cpu_count * 2
+ log.debug("Maximum number of simultaneous processes is: %s" % max_processes)
+
+ # Indicates when to try to request a new job or aborted builds.
+ last_job_request = 0
+ last_abort_request = 0
+
+ # Main loop.
+ while True:
+ # Send the keepalive regularly.
+ self.send_keepalive()
+
+ # Remove all finished builds.
+ # "removed" indicates, if a process has actually finished.
+ removed = self.remove_finished_builders()
+
+ # If a build slot was freed, search immediately for a new job.
+ if removed:
+ last_job_request = 0
+
+ # Kill aborted jobs.
+ if time.time() - last_abort_request >= 60:
+ aborted = self.kill_aborted_jobs()
+
+ # If a build slot was freed, search immediately for a new job.
+ if aborted:
+ last_job_request = 0
+
+ last_abort_request = time.time()
+
+ # Check if the maximum number of processes was reached.
+ # Actually the hub does manage this but this is an emergency
+ # condition if anything goes wrong.
+ if self.num_processes >= max_processes:
+ log.debug("Reached maximum number of allowed processes (%s)." % max_processes)
+
+ time.sleep(heartbeat)
+ continue
+
+ # Get new job.
+ if time.time() - last_job_request >= 60 and not self.has_overload():
+ # If the last job request is older than a minute and we don't
+ # have too much load, we go and check if there is something
+ # to do for us.
+ job = self.get_job()
+
+ # If we got a job, we start a child process to work on it.
+ if job:
+ log.debug("Got a new job.")
+ self.fork_builder(job)
+ else:
+ log.debug("No new job.")
+
+ # Update the time when we requested a job.
+ last_job_request = time.time()
+
+ # Wait a moment before starting over.
+ time.sleep(heartbeat)
+
+ def shutdown(self):
+ """
+ Shut down the daemon.
+ This means to kill all child processes.
+
+ The method blocks until all processes are shut down.
+ """
+ for process in self.processes:
+ log.info("Sending %s to terminate..." % process)
+
+ process.terminate()
+ else:
+ log.info("No processes to kill. Shutting down immediately.")
+
+ while self.processes:
+ log.debug("%s process(es) is/are still running..." % len(self.processes))
+
+ for process in self.processes[:]:
+ if not process.is_alive():
+ # The process has terminated.
+ log.info("Process %s terminated with exit code: %s" % \
+ (process, process.exitcode))
+
+ self.processes.remove(process)
+
+ @property
+ def num_processes(self):
+ # Return the number of processes.
+ return len(self.processes)
+
+ def get_job(self):
+ """
+ Get a build job from the hub.
+ """
+ log.info("Requesting a new job from the server...")
+
+ # Get some information about this system.
+ s = pakfire.system.System()
+
+ # Fetch a build job from the hub.
+ return self.client.conn.build_get_job(s.supported_arches)
+
+ def has_overload(self):
+ """
+ Checks, if the load average is not too high.
+
+ On this is to be decided if a new job is taken.
+ """
+ try:
+ load1, load5, load15 = os.getloadavg()
+ except OSError:
+ # Could not determine the current loadavg. In that case we
+ # assume that we don't have overload.
+ return False
+
+ # If there are more than 2 processes in the process queue per CPU
+ # core we will assume that the system has heavy load and to not request
+ # a new job.
+ return load5 >= system.cpu_count * 2
+
+ def send_keepalive(self):
+ """
+ When triggered, this method sends a keepalive to the hub.
+ """
+ # Do not send a keepalive more often than twice a minute.
+ if time.time() - self._last_keepalive < 30:
+ return
+
+ self.client.send_keepalive(overload=self.has_overload())
+ self._last_keepalive = time.time()
+
+ def remove_finished_builders(self):
+ # Return if any processes have been removed.
+ ret = False
+
+ # Search for any finished processes.
+ for process in self.processes[:]:
+ # If the process is not alive anymore...
+ if not process.is_alive():
+ ret = True
+
+ # ... check the exit code and log a message on errors.
+ if process.exitcode == 0:
+ log.debug("Process %s exited normally." % process)
+
+ elif process.exitcode > 0:
+ log.error("Process did not exit normally: %s code: %s" \
+ % (process, process.exitcode))
+
+ elif process.exitcode < 0:
+ log.error("Process killed by signal: %s: code: %s" \
+ % (process, process.exitcode))
+
+ # If a program has crashed, we send that to the hub.
+ job_id = self.pid2jobid.get(process.pid, None)
+ if job_id:
+ self.conn.build_job_crashed(job_id, process.exitcode)
+
+ # Finally, remove the process from the process list.
+ self.processes.remove(process)
+
+ return ret
+
+ def kill_aborted_jobs(self):
+ log.debug("Requesting aborted jobs...")
+
+ # Get a list of running job ids:
+ running_jobs = self.pid2jobid.values()
+
+ # If there are no running jobs, there is nothing to do.
+ if not running_jobs:
+ return False
+
+ # Ask the hub for any build jobs to abort.
+ aborted_jobs = self.conn.build_jobs_aborted(running_jobs)
+
+ # If no build jobs were aborted, there is nothing to do.
+ if not aborted_jobs:
+ return False
+
+ for process in self.processes[:]:
+ job_id = self.pid2jobid.get(process.pid, None)
+ if job_id and job_id in aborted_jobs:
+
+ # Kill the process.
+ log.info("Killing process %s which was aborted by the user." \
+ % process.pid)
+ process.terminate()
+
+ # Remove the process from the process list to avoid
+ # that is will be cleaned up in the normal way.
+ self.processes.remove(process)
+
+ return True
+
+ def fork_builder(self, job):
+ """
+ For a new child process to create a new independent builder.
+ """
+ # Create the Process object.
+ process = multiprocessing.Process(target=fork_builder,
+ args=(self.server, self.hostname, self.__secret, job))
+ # The process is running in daemon mode so it will try to kill
+ # all child processes when exiting.
+ process.daemon = True
+
+ # Start the process.
+ process.start()
+ log.info("Started new process %s with PID %s." % (process, process.pid))
+
+ # Save the PID and the build id to track down
+ # crashed builds.
+ self.pid2jobid[process.pid] = job.get("id", None)
+
+ # Append it to the process list.
+ self.processes.append(process)
+
+
+class ClientBuilder(object):
+ def __init__(self, server, hostname, secret, job):
+ self.client = base.PakfireBuilderClient(server, hostname, secret)
+ self.conn = self.client.conn
+
+ # Store the information sent by the server here.
+ self.build_job = job
+
+ def update_state(self, state, message=None):
+ self.conn.build_job_update_state(self.build_id, state, message)
+
+ def upload_file(self, filename, type):
+ assert os.path.exists(filename)
+ assert type in ("package", "log")
+
+ # First upload the file data and save the upload_id.
+ upload_id = self.client._upload_file(filename)
+
+ # Add the file to the build.
+ return self.conn.build_job_add_file(self.build_id, upload_id, type)
+
+ def upload_buildroot(self, installed_packages):
+ pkgs = []
+
+ for pkg in installed_packages:
+ pkgs.append((pkg.friendly_name, pkg.uuid))
+
+ return self.conn.build_upload_buildroot(self.build_id, pkgs)
+
+ @property
+ def build_id(self):
+ if self.build_job:
+ return self.build_job.get("id", None)
+
+ @property
+ def build_arch(self):
+ if self.build_job:
+ return self.build_job.get("arch", None)
+
+ @property
+ def build_source_url(self):
+ if self.build_job:
+ return self.build_job.get("source_url", None)
+
+ @property
+ def build_source_filename(self):
+ if self.build_source_url:
+ return os.path.basename(self.build_source_url)
+
+ @property
+ def build_source_hash512(self):
+ if self.build_job:
+ return self.build_job.get("source_hash512", None)
+
+ @property
+ def build_type(self):
+ if self.build_job:
+ return self.build_job.get("type", None)
+
+ def build(self):
+ # Cannot go on if I got no build job.
+ if not self.build_job:
+ logging.info("No job to work on...")
+ return
+
+ # Call the function that processes the build and try to catch general
+ # exceptions and report them to the server.
+ # If everything goes okay, we tell this the server, too.
+ try:
+ # Create a temporary file and a directory for the resulting files.
+ tmpdir = tempfile.mkdtemp()
+ tmpfile = os.path.join(tmpdir, self.build_source_filename)
+ logfile = os.path.join(tmpdir, "build.log")
+
+ # Get a package grabber and add mirror download capabilities to it.
+ grabber = pakfire.downloader.PackageDownloader(pakfire.config.Config())
+
+ try:
+ ## Download the source.
+ grabber.urlgrab(self.build_source_url, filename=tmpfile)
+
+ # Check if the download checksum matches (if provided).
+ if self.build_source_hash512:
+ h = hashlib.sha512()
+ f = open(tmpfile, "rb")
+ while True:
+ buf = f.read(BUFFER_SIZE)
+ if not buf:
+ break
+
+ h.update(buf)
+ f.close()
+
+ if not self.build_source_hash512 == h.hexdigest():
+ raise DownloadError, "Hash check did not succeed."
+
+ # Create dist with arguments that are passed to the pakfire
+ # builder.
+ kwargs = {
+ # Of course this is a release build.
+ # i.e. don't use local packages.
+ "builder_mode" : "release",
+
+ # Set the build_id we got from the build service.
+ "build_id" : self.build_id,
+
+ # Files and directories (should be self explaining).
+ "logfile" : logfile,
+
+ # Distro configuration.
+ "distro_config" : {
+ "arch" : self.build_arch,
+ },
+ }
+
+ # Create a new instance of the builder.
+ build = pakfire.builder.BuildEnviron(tmpfile, **kwargs)
+
+ try:
+ # Create the build environment.
+ build.start()
+
+ # Update the build status on the server.
+ self.upload_buildroot(build.installed_packages)
+ self.update_state("running")
+
+ # Run the build (with install test).
+ build.build(install_test=True)
+
+ # Copy the created packages to the tempdir.
+ build.copy_result(tmpdir)
+
+ finally:
+ # Cleanup the build environment.
+ build.stop()
+
+ # Jippie, build is finished, we are going to upload the files.
+ self.update_state("uploading")
+
+ # Walk through the result directory and upload all (binary) files.
+ # Skip that for test builds.
+ if not self.build_type == "test":
+ for dir, subdirs, files in os.walk(tmpdir):
+ for file in files:
+ file = os.path.join(dir, file)
+ if file in (logfile, tmpfile,):
+ continue
+
+ self.upload_file(file, "package")
+
+ except DependencyError, e:
+ message = "%s: %s" % (e.__class__.__name__, e)
+ self.update_state("dependency_error", message)
+ raise
+
+ except DownloadError, e:
+ message = "%s: %s" % (e.__class__.__name__, e)
+ self.update_state("download_error", message)
+ raise
+
+ finally:
+ # Upload the logfile in any case and if it exists.
+ if os.path.exists(logfile):
+ self.upload_file(logfile, "log")
+
+ # Cleanup the files we created.
+ pakfire.util.rm(tmpdir)
+
+ except DependencyError:
+ # This has already been reported.
+ raise
+
+ except (DownloadError,):
+ # Do not take any further action for these exceptions.
+ pass
+
+ except Exception, e:
+ # Format the exception and send it to the server.
+ message = "%s: %s" % (e.__class__.__name__, e)
+
+ self.update_state("failed", message)
+ raise
+
+ else:
+ self.update_state("finished")
--- /dev/null
+#!/usr/bin/python
+
+import random
+import sys
+import time
+
+def fork_builder(*args, **kwargs):
+ cb = ClientBuilder(*args, **kwargs)
+
+ try:
+ cb()
+ except Exception, e:
+ print e
+ sys.exit(1)
+
+class ClientBuilder(object):
+ def __init__(self, id):
+ self.id = id
+
+ def __call__(self, *args):
+ print "Running", self.id, args
+
+ time.sleep(2)
+
+ if random.choice((False, False, False, True)):
+ raise Exception, "Process died"
+
+
+import multiprocessing
+
+
+processes = []
+
+while True:
+ # Check if there are at least 2 processes running.
+ if len(processes) < 2:
+ process = multiprocessing.Process(target=fork_builder, args=(len(processes),))
+
+ process.daemon = True
+ process.start()
+
+ processes.append(process)
+
+ print len(processes), "in process list:", processes
+
+ for process in processes:
+ time.sleep(0.5)
+
+ print process.name, "is alive?", process.is_alive()
+
+ if not process.is_alive():
+ print "Removing process", process
+ print " Exitcode:", process.exitcode
+ processes.remove(process)
--- /dev/null
+#!/usr/bin/python
+###############################################################################
+# #
+# Pakfire - The IPFire package management system #
+# Copyright (C) 2011 Pakfire development team #
+# #
+# This program is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# This program is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with this program. If not, see <http://www.gnu.org/licenses/>. #
+# #
+###############################################################################
+
+import socket
+import time
+import xmlrpclib
+
+import logging
+log = logging.getLogger("pakfire.client")
+
+from pakfire.constants import *
+from pakfire.i18n import _
+
+class XMLRPCMixin:
+ user_agent = "pakfire/%s" % PAKFIRE_VERSION
+
+ def single_request(self, *args, **kwargs):
+ ret = None
+
+ # Tries can be passed to this method.
+ tries = kwargs.pop("tries", 100)
+ timeout = 1
+
+ while tries:
+ try:
+ ret = xmlrpclib.Transport.single_request(self, *args, **kwargs)
+
+ except socket.error, e:
+ # These kinds of errors are not fatal, but they can happen on
+ # a bad internet connection or whatever.
+ # 32 Broken pipe
+ # 110 Connection timeout
+ # 111 Connection refused
+ if not e.errno in (32, 110, 111,):
+ raise
+
+ log.warning(_("Socket error: %s") % e)
+
+ except xmlrpclib.ProtocolError, e:
+ # Log all XMLRPC protocol errors.
+ log.error("XMLRPC protocol error:")
+ log.error(" URL: %s" % e.url)
+ log.error(" HTTP headers:")
+ for header in e.headers.items():
+ log.error(" %s: %s" % header)
+ log.error(" Error code: %s" % e.errcode)
+ log.error(" Error message: %s" % e.errmsg)
+ raise
+
+ else:
+ # If request was successful, we can break the loop.
+ break
+
+ # If the request was not successful, we wait a little time to try
+ # it again.
+ tries -= 1
+ timeout *= 2
+ if timeout > 60:
+ timeout = 60
+
+ log.warning(_("Trying again in %s seconds. %s tries left.") % (timeout, tries))
+ time.sleep(timeout)
+
+ else:
+ log.error("Maximum number of tries was reached. Giving up.")
+ # XXX need better exception here.
+ raise Exception, "Could not fulfill request."
+
+ return ret
+
+
+class XMLRPCTransport(XMLRPCMixin, xmlrpclib.Transport):
+ """
+ Handles the XMLRPC connection over HTTP.
+ """
+ pass
+
+
+class SafeXMLRPCTransport(XMLRPCMixin, xmlrpclib.SafeTransport):
+ """
+ Handles the XMLRPC connection over HTTPS.
+ """
+ pass
+
+
+class Connection(xmlrpclib.ServerProxy):
+ """
+ Class wrapper that automatically chooses the right transport
+ method depending on the given URL.
+ """
+
+ def __init__(self, url):
+ # Create transport channel to the server.
+ if url.startswith("https://"):
+ transport = SafeXMLRPCTransport()
+ elif url.startswith("http://"):
+ transport = XMLRPCTransport()
+
+ xmlrpclib.ServerProxy.__init__(self, url, transport=transport,
+ allow_none=True)
###############################################################################
import os
+import socket
from ConfigParser import ConfigParser
log = logging.getLogger("pakfire")
import base
+from system import system
from constants import *
+from i18n import _
class Config(object):
def __init__(self, type=None):
if self.type == "builder":
path = os.getcwd()
+ _path = None
while not path == "/":
_path = os.path.join(path, "config")
if os.path.exists(_path):
Check if this host can build for the target architecture "arch".
"""
return arch in self.supported_arches
+
+class _Config(object):
+ files = []
+
+ # A dict with default settings for this config class.
+ default_settings = {}
+
+ def __init__(self, path="/etc"):
+ # Configuration settings.
+ self._config = self.default_settings.copy()
+
+ # List of files that were already loaded.
+ self._files = []
+
+ # Read default configuration file.
+ self.read(*[os.path.join(path, f) for f in self.files])
+
+ # Dump read configuration.
+ self.dump()
+
+ def _read_hook(self, config):
+ """
+ Method to be overwritten when addition stuff
+ should be done with the ConfigParser object.
+ """
+ pass
+
+ def read(self, *files):
+ # Do nothing for no files.
+ if not files:
+ return
+
+ # Create configparser and read the content of the file
+ # to parse it.
+ config = ConfigParser()
+ for f in files:
+ # Normalize filename.
+ f = os.path.abspath(f)
+
+ # Check if file has already been read or
+ # does not exist. Then skip it.
+ if f in self._files or not os.path.exists(f):
+ continue
+
+ # Parse the file.
+ log.debug(_("Reading from configuration file: %s") % f)
+ config.read(f)
+
+ # Save the filename to the list of read files.
+ self._files.append(f)
+
+ # Read all data from the configuration file in the _config dict.
+ for section in config.sections():
+ items = dict(config.items(section))
+
+ try:
+ self._config[section].update(items)
+ except KeyError:
+ self._config[section] = items
+
+ def set(self, section, key, value):
+ try:
+ self._config[section][key] = value
+ except KeyError:
+ self._config[section] = { key : value }
+
+ def get(self, section, key, default=None):
+ try:
+ return self._config[section][key]
+ except KeyError:
+ return default
+
+ def get_int(self, section, key, default=None):
+ val = self.get(section=section, key=key, default=default)
+ try:
+ val = int(val)
+ except ValueError:
+ return default
+
+ def get_bool(self, section, key, default=None):
+ val = self.get(section=section, key=key, default=default)
+
+ if val in (True, "true", "1", "on"):
+ return True
+ elif val in (False, "false", "0", "off"):
+ return False
+
+ return default
+
+ def dump(self):
+ """
+ Dump the configuration that was read.
+
+ (Only in debugging mode.)
+ """
+ log.debug(_("Configuration:"))
+ for section, settings in self._config.items():
+ log.debug(" " + _("Section: %s") % section)
+
+ for k, v in settings.items():
+ log.debug(" %-20s: %s" % (k, v))
+ else:
+ log.debug(" " + _("No settings in this section."))
+
+ log.debug(" " + _("Loaded from files:"))
+ for f in self._files:
+ log.debug(" %s" % f)
+
+
+class ConfigBuilder(_Config):
+ files = ["pakfire.conf", "pakfire-builder.conf"]
+
+
+class ConfigClient(_Config):
+ files = ["pakfire.conf", "pakfire-client.conf"]
+
+ default_settings = {
+ "client" : {
+ # The default server is the official Pakfire
+ # server.
+ "server" : "https://pakfire.ipfire.org",
+ },
+ }
+
+
+class ConfigDaemon(_Config):
+ files = ["pakfire.conf", "pakfire-daemon.conf"]
+
+ default_settings = {
+ "daemon" : {
+ # The default server is the official Pakfire
+ # server.
+ "server" : "https://pakfire.ipfire.org",
+
+ # The default hostname is the host name of this
+ # machine.
+ "hostname" : system.hostname,
+ },
+ }
BUFFER_SIZE = 102400
+# The size of the data chunks that are uploaded to the
+# pakfire hub.
+CHUNK_SIZE = BUFFER_SIZE
+
MIRRORLIST_MAXSIZE = 1024**2
MACRO_FILE_DIR = "/usr/lib/pakfire/macros"
"release" : self.release,
"epoch" : self.epoch,
"arch" : self.arch,
+ "supported_arches" : self.supported_arches,
"groups" : self.groups,
"summary" : self.summary,
"description" : self.description,
import shutil
import socket
import tarfile
+import uuid
from urlgrabber.grabber import URLGrabber, URLGrabError
from urlgrabber.progress import TextMeter
# Not existant for Makefiles
return None
+ @property
+ def supported_arches(self):
+ """
+ These are the supported arches. Which means, packages of these
+ architectures can be built out of this source package.
+ """
+ # If the package architecture is "noarch", the package
+ # needs only to be built for that.
+ if self.arch == "noarch":
+ return "noarch"
+
+ return self.lexer.get_var("sup_arches", "all")
+
class Makefile(MakefileBase):
@property
def arch(self):
return "src"
- @property
- def supported_arches(self):
- """
- These are the supported arches. Which means, packages of these
- architectures can be built out of this source package.
- """
- return self.lexer.get_var("sup_arches", "all")
-
@property
def packages(self):
pkgs = []
# Store additional dependencies in here.
self._dependencies = {}
+ # Generate a random identifier.
+ self._uuid = "%s" % uuid.uuid4()
+
@property
def name(self):
return self._name
@property
def uuid(self):
- return None
+ return self._uuid
def track_dependencies(self, builder, path):
# Build filelist with all files that have been installed.
# Collect all dependencies that were discovered by the tracker.
deps += self._dependencies.get(key, [])
+ # Add the UUID.
+ if key == "provides":
+ deps.append("uuid(%s)" % self.uuid)
+
# Remove duplicates.
deps = set(deps)
deps = list(deps)
@property
def requires(self):
- return self.get_deps("requires")
+ # Make sure that no self-provides are in the list
+ # of requirements.
+ provides = self.provides
+
+ return [r for r in self.get_deps("requires") if not r in provides]
@property
def provides(self):
# Generic package information including Pakfire information.
info.update({
"pakfire_version" : PAKFIRE_VERSION,
- "uuid" : uuid.uuid4(),
+ "uuid" : self.pkg.uuid,
"type" : "binary",
})
return ret
+class ServerProxy(xmlrpclib.ServerProxy):
+ def __init__(self, server, *args, **kwargs):
+
+ # Some default settings.
+ if not kwargs.has_key("transport"):
+ kwargs["transport"] = XMLRPCTransport()
+
+ kwargs["allow_none"] = True
+
+ xmlrpclib.ServerProxy.__init__(self, server, *args, **kwargs)
+
+
class Server(object):
def __init__(self, **pakfire_args):
self.config = pakfire.config.Config()
log.info("Establishing RPC connection to: %s" % server)
- self.conn = xmlrpclib.ServerProxy(server, transport=XMLRPCTransport(),
- allow_none=True)
+ self.conn = ServerProxy(server)
self.pakfire_args = pakfire_args
(repo["distro"]["sname"], repo["name"], arch)
pakfire.api.repo_create(path, files)
+
+ def create_scratch_build(self, *args, **kwargs):
+ return self.conn.create_scratch_build(*args, **kwargs)
from __future__ import division
+import multiprocessing
import os
+import socket
+
+from i18n import _
+
+class System(object):
+ """
+ Class that grants access to several information about
+ the system this software is running on.
+ """
+ @property
+ def hostname(self):
+ return socket.gethostname()
+
+ @property
+ def arch(self):
+ """
+ Return the architecture of the host we are running on.
+ """
+ return os.uname()[4]
+
+ @property
+ def supported_arches(self):
+ """
+ Check what architectures can be built on this host.
+ """
+ host_can_build = {
+ # Host arch : Can build these arches.
+
+ # x86
+ "x86_64" : ["x86_64", "i686",],
+ "i686" : ["i686",],
+
+ # ARM
+ "armv5tel" : ["armv5tel",],
+ "armv5tejl" : ["armv5tel",],
+ "armv7l" : ["armv5tel",],
+ "armv7hl" : ["armv7hl", "armv5tel",],
+ }
+
+ try:
+ return host_can_build[self.arch]
+ except KeyError:
+ return []
+
+ def host_supports_arch(self, arch):
+ """
+ Check if this host can build for the target architecture "arch".
+ """
+ return arch in self.supported_arches
+
+ @property
+ def cpu_count(self):
+ """
+ Count the number of CPU cores.
+ """
+ return multiprocessing.cpu_count()
+
+ @property
+ def cpu_model(self):
+ # Determine CPU model
+ cpuinfo = {}
+ with open("/proc/cpuinfo") as f:
+ for line in f.readlines():
+ # Break at an empty line, because all information after that
+ # is redundant.
+ if not line:
+ break
+
+ try:
+ key, value = line.split(":")
+ except:
+ pass # Skip invalid lines
+
+ key, value = key.strip(), value.strip()
+ cpuinfo[key] = value
+
+ ret = None
+ if self.arch.startswith("arm"):
+ try:
+ ret = "%(Hardware)s - %(Processor)s" % cpuinfo
+ except KeyError:
+ pass
+ else:
+ ret = cpuinfo.get("model name", None)
+
+ # Remove too many spaces.
+ ret = " ".join(ret.split())
+
+ return ret or _("Could not be determined")
+
+ @property
+ def memory(self):
+ # Determine memory size
+ memory = 0
+ with open("/proc/meminfo") as f:
+ line = f.readline()
+
+ try:
+ a, b, c = line.split()
+ except:
+ pass
+ else:
+ memory = int(b) * 1024
+
+ return memory
+
+
+# Create an instance of this class to only keep it once in memory.
+system = System()
class Mountpoints(object):
def __init__(self, pakfire, root="/"):
assert file.name.startswith(self.path)
self.disk_usage += file.size
+
+
+if __name__ == "__main__":
+ print "Hostname", system.hostname
+ print "Arch", system.arch
+ print "Supported arches", system.supported_arches
+
+ print "CPU Model", system.cpu_model
+ print "CPU count", system.cpu_count
+ print "Memory", system.memory
s /= 1024
unit += 1
- return "%d %s" % (int(s) * sign, units[unit])
+ return "%d%s" % (round(s) * sign, units[unit])
def format_time(s):
return "%02d:%02d" % (s // 60, s % 60)
-mkdir -pv $(DESTDIR)/usr/bin
ln -svf ../..$(SCRIPT_DIR)/pakfire-multicall.py $(DESTDIR)/usr/bin/pakfire
ln -svf ../..$(SCRIPT_DIR)/pakfire-multicall.py $(DESTDIR)/usr/bin/pakfire-builder
+ ln -svf ../..$(SCRIPT_DIR)/pakfire-multicall.py $(DESTDIR)/usr/bin/pakfire-client
+ ln -svf ../..$(SCRIPT_DIR)/pakfire-multicall.py $(DESTDIR)/usr/bin/pakfire-daemon
ln -svf ../..$(SCRIPT_DIR)/pakfire-multicall.py $(DESTDIR)/usr/bin/pakfire-server
ln -svf pakfire-multicall.py $(DESTDIR)$(SCRIPT_DIR)/builder
basename2cls = {
"pakfire" : Cli,
"pakfire-builder" : CliBuilder,
+ "pakfire-client" : CliClient,
+ "pakfire-daemon" : CliDaemon,
"pakfire-server" : CliServer,
"builder" : CliBuilderIntern,
}
projects / pakfire.git / commitdiff
