From: Michael Tremer Date: Mon, 9 Jan 2012 10:13:23 +0000 (+0100) Subject: Huge change: Introduce pakfire-client and -daemon. X-Git-Tag: 0.9.20~18^2~3 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=c62d93f14f200f9c7915c473d06486da307d96c2;p=pakfire.git Huge change: Introduce pakfire-client and -daemon. This introduces the pakfire-client and the daemon which communicate both with the pakfire build service. The client is for users, that can send new builds to the service. The daemon fetches build jobs from the build service, builds them and uploades the binary packages. --- diff --git a/Makeconfig b/Makeconfig index 2c2708f09..1647544fc 100644 --- a/Makeconfig +++ b/Makeconfig @@ -29,7 +29,7 @@ endif PYTHON_CC = $(CC) -pthread -fPIC PYTHON_CFLAGS = $(shell python-config --cflags) -PYTHON_MODULES = pakfire pakfire/packages pakfire/repository +PYTHON_MODULES = pakfire pakfire/client pakfire/packages pakfire/repository ifeq "$(DEBIAN)" "1" PYTHON_DIR = $(LIBDIR)/python$(PYTHON_VERSION)/dist-packages else diff --git a/Makefile b/Makefile index ffacd122f..e9d80ed48 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,10 @@ install: build # Don't overwrite already installed configuration file. [ -e "$(DESTDIR)/etc/pakfire.conf" ] || \ cp -vf examples/pakfire.conf $(DESTDIR)/etc/pakfire.conf + [ -e "$(DESTDIR)/etc/pakfire-client.conf" ] || \ + cp -vf examples/pakfire-client.conf $(DESTDIR)/etc/pakfire-client.conf + [ -e "$(DESTDIR)/etc/pakfire-daemon.conf" ] || \ + cp -vf examples/pakfire-daemon.conf $(DESTDIR)/etc/pakfire-daemon.conf cp -vf examples/pakfire.repos.d/* $(DESTDIR)/etc/pakfire.repos.d/ .PHONY: check diff --git a/examples/pakfire-client.conf b/examples/pakfire-client.conf new file mode 100644 index 000000000..da57d03fd --- /dev/null +++ b/examples/pakfire-client.conf @@ -0,0 +1,10 @@ + +# Configure the pakfire client. +[client] + +# The URL of the server to connect to. +# server = https://pakfire.ipfire.org/ + +# Your credentials to log in on the hub. +# username = ipfire +# password = 1234... diff --git a/examples/pakfire-daemon.conf b/examples/pakfire-daemon.conf new file mode 100644 index 000000000..525cc1f36 --- /dev/null +++ b/examples/pakfire-daemon.conf @@ -0,0 +1,14 @@ + +# Configure the pakfire daemon. +[daemon] + +# The URL of the server to connect to. +# server = https://pakfire.ipfire.org/ + +# The hostname of this machine. +# hostname = + +# The authentication secret that is used +# to identify this host against the pakfire +# build service. +# secret = 1234... diff --git a/po/pakfire.pot b/po/pakfire.pot index ffd851cb3..58419eb63 100644 --- a/po/pakfire.pot +++ b/po/pakfire.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2011-12-23 21:12+0100\n" +"POT-Creation-Date: 2012-01-26 22:46+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -119,389 +119,513 @@ msgid "Everything is fine." msgstr "" #. Log the package information. -#: ../python/pakfire/builder.py:153 +#: ../python/pakfire/builder.py:155 msgid "Package information:" msgstr "" #. Install all packages. -#: ../python/pakfire/builder.py:324 +#: ../python/pakfire/builder.py:326 msgid "Install packages needed for build..." msgstr "" -#: ../python/pakfire/builder.py:329 +#: ../python/pakfire/builder.py:331 msgid "Extracting" msgstr "" -#: ../python/pakfire/builder.py:582 +#: ../python/pakfire/builder.py:600 msgid "You cannot run a build when no package was given." msgstr "" -#: ../python/pakfire/builder.py:587 +#: ../python/pakfire/builder.py:605 #, python-format msgid "Could not find makefile in build root: %s" msgstr "" -#: ../python/pakfire/builder.py:601 +#: ../python/pakfire/builder.py:619 msgid "The build command failed. See logfile for details." msgstr "" #. Walk through the whole tree and collect all files #. that are on the same disk (not crossing mountpoints). -#: ../python/pakfire/builder.py:659 +#: ../python/pakfire/builder.py:677 msgid "Creating filelist..." msgstr "" #. Create a nice progressbar. -#: ../python/pakfire/builder.py:678 +#: ../python/pakfire/builder.py:696 msgid "Compressing files..." msgstr "" -#: ../python/pakfire/builder.py:697 +#: ../python/pakfire/builder.py:715 #, python-format msgid "Cache file was successfully created at %s." msgstr "" -#: ../python/pakfire/builder.py:698 +#: ../python/pakfire/builder.py:716 #, python-format msgid " Containing %(files)s files, it has a size of %(size)s." msgstr "" #. Make a nice progress bar as always. -#: ../python/pakfire/builder.py:709 +#: ../python/pakfire/builder.py:727 msgid "Extracting files..." msgstr "" #. Update all packages. -#: ../python/pakfire/builder.py:729 +#: ../python/pakfire/builder.py:747 msgid "Updating packages from cache..." msgstr "" #. Package the result. #. Make all these little package from the build environment. -#: ../python/pakfire/builder.py:857 +#: ../python/pakfire/builder.py:875 msgid "Creating packages:" msgstr "" #. Execute the buildscript of this stage. -#: ../python/pakfire/builder.py:877 +#: ../python/pakfire/builder.py:895 #, python-format msgid "Running stage %s:" msgstr "" -#: ../python/pakfire/builder.py:895 +#: ../python/pakfire/builder.py:913 #, python-format msgid "Could not remove static libraries: %s" msgstr "" -#: ../python/pakfire/builder.py:901 +#: ../python/pakfire/builder.py:919 msgid "Compressing man pages did not complete successfully." msgstr "" -#: ../python/pakfire/builder.py:921 +#: ../python/pakfire/builder.py:939 msgid "Extracting debuginfo did not complete with success. Aborting build." msgstr "" -#: ../python/pakfire/cli.py:43 +#: ../python/pakfire/cli.py:47 msgid "Pakfire command line interface." msgstr "" -#: ../python/pakfire/cli.py:50 +#: ../python/pakfire/cli.py:54 msgid "The path where pakfire should operate in." msgstr "" -#: ../python/pakfire/cli.py:117 +#: ../python/pakfire/cli.py:121 msgid "Enable verbose output." msgstr "" -#: ../python/pakfire/cli.py:120 +#: ../python/pakfire/cli.py:124 msgid "Path to a configuration file to load." msgstr "" -#: ../python/pakfire/cli.py:123 +#: ../python/pakfire/cli.py:128 msgid "Disable a repository temporarily." msgstr "" -#: ../python/pakfire/cli.py:126 +#: ../python/pakfire/cli.py:131 msgid "Enable a repository temporarily." msgstr "" -#: ../python/pakfire/cli.py:129 +#: ../python/pakfire/cli.py:135 msgid "Run pakfire in offline mode." msgstr "" -#: ../python/pakfire/cli.py:134 +#: ../python/pakfire/cli.py:140 msgid "Install one or more packages to the system." msgstr "" -#: ../python/pakfire/cli.py:136 +#: ../python/pakfire/cli.py:142 msgid "Give name of at least one package to install." msgstr "" -#: ../python/pakfire/cli.py:142 +#: ../python/pakfire/cli.py:148 msgid "Install one or more packages from the filesystem." msgstr "" -#: ../python/pakfire/cli.py:144 +#: ../python/pakfire/cli.py:150 msgid "Give filename of at least one package." msgstr "" -#: ../python/pakfire/cli.py:150 +#: ../python/pakfire/cli.py:156 msgid "Reinstall one or more packages." msgstr "" -#: ../python/pakfire/cli.py:152 +#: ../python/pakfire/cli.py:158 msgid "Give name of at least one package to reinstall." msgstr "" -#: ../python/pakfire/cli.py:158 +#: ../python/pakfire/cli.py:164 msgid "Remove one or more packages from the system." msgstr "" -#: ../python/pakfire/cli.py:160 +#: ../python/pakfire/cli.py:166 msgid "Give name of at least one package to remove." msgstr "" -#: ../python/pakfire/cli.py:166 +#: ../python/pakfire/cli.py:172 msgid "Give a name of a package to update or leave emtpy for all." msgstr "" -#: ../python/pakfire/cli.py:168 +#: ../python/pakfire/cli.py:174 msgid "Exclude package from update." msgstr "" -#: ../python/pakfire/cli.py:170 ../python/pakfire/cli.py:195 +#: ../python/pakfire/cli.py:176 ../python/pakfire/cli.py:201 msgid "Allow changing the vendor of packages." msgstr "" -#: ../python/pakfire/cli.py:172 ../python/pakfire/cli.py:197 +#: ../python/pakfire/cli.py:178 ../python/pakfire/cli.py:203 msgid "Allow changing the architecture of packages." msgstr "" -#: ../python/pakfire/cli.py:177 +#: ../python/pakfire/cli.py:183 msgid "Update the whole system or one specific package." msgstr "" -#: ../python/pakfire/cli.py:184 +#: ../python/pakfire/cli.py:190 msgid "Check, if there are any updates available." msgstr "" -#: ../python/pakfire/cli.py:191 +#: ../python/pakfire/cli.py:197 msgid "Downgrade one or more packages." msgstr "" -#: ../python/pakfire/cli.py:193 +#: ../python/pakfire/cli.py:199 msgid "Give a name of a package to downgrade." msgstr "" -#: ../python/pakfire/cli.py:203 +#: ../python/pakfire/cli.py:209 msgid "Print some information about the given package(s)." msgstr "" -#: ../python/pakfire/cli.py:205 +#: ../python/pakfire/cli.py:211 msgid "Give at least the name of one package." msgstr "" -#: ../python/pakfire/cli.py:211 +#: ../python/pakfire/cli.py:217 msgid "Search for a given pattern." msgstr "" -#: ../python/pakfire/cli.py:213 +#: ../python/pakfire/cli.py:219 msgid "A pattern to search for." msgstr "" -#: ../python/pakfire/cli.py:219 +#: ../python/pakfire/cli.py:225 msgid "Get a list of packages that provide a given file or feature." msgstr "" -#: ../python/pakfire/cli.py:221 +#: ../python/pakfire/cli.py:227 msgid "File or feature to search for." msgstr "" -#: ../python/pakfire/cli.py:227 +#: ../python/pakfire/cli.py:233 msgid "Get list of packages that belong to the given group." msgstr "" -#: ../python/pakfire/cli.py:229 +#: ../python/pakfire/cli.py:235 msgid "Group name to search for." msgstr "" -#: ../python/pakfire/cli.py:235 +#: ../python/pakfire/cli.py:241 msgid "Install all packages that belong to the given group." msgstr "" -#: ../python/pakfire/cli.py:237 +#: ../python/pakfire/cli.py:243 msgid "Group name." msgstr "" -#: ../python/pakfire/cli.py:243 +#: ../python/pakfire/cli.py:249 msgid "List all currently enabled repositories." msgstr "" -#: ../python/pakfire/cli.py:247 +#: ../python/pakfire/cli.py:253 msgid "Cleanup commands." msgstr "" -#: ../python/pakfire/cli.py:255 +#: ../python/pakfire/cli.py:261 msgid "Cleanup all temporary files." msgstr "" -#: ../python/pakfire/cli.py:261 +#: ../python/pakfire/cli.py:267 msgid "Check the system for any errors." msgstr "" -#: ../python/pakfire/cli.py:267 +#: ../python/pakfire/cli.py:273 msgid "Check the dependencies for a particular package." msgstr "" -#: ../python/pakfire/cli.py:269 +#: ../python/pakfire/cli.py:275 msgid "Give name of at least one package to check." msgstr "" -#: ../python/pakfire/cli.py:348 ../python/pakfire/transaction.py:352 +#: ../python/pakfire/cli.py:351 ../python/pakfire/transaction.py:352 msgid "Repository" msgstr "" -#: ../python/pakfire/cli.py:348 +#: ../python/pakfire/cli.py:351 msgid "Enabled" msgstr "" -#: ../python/pakfire/cli.py:348 +#: ../python/pakfire/cli.py:351 msgid "Priority" msgstr "" -#: ../python/pakfire/cli.py:348 +#: ../python/pakfire/cli.py:351 msgid "Packages" msgstr "" -#: ../python/pakfire/cli.py:360 +#: ../python/pakfire/cli.py:363 msgid "Cleaning up everything..." msgstr "" -#: ../python/pakfire/cli.py:376 +#: ../python/pakfire/cli.py:379 msgid "You cannot run pakfire-builder in a pakfire chroot." msgstr "" -#: ../python/pakfire/cli.py:379 ../python/pakfire/cli.py:688 +#: ../python/pakfire/cli.py:382 ../python/pakfire/cli.py:719 msgid "Pakfire builder command line interface." msgstr "" -#: ../python/pakfire/cli.py:437 +#: ../python/pakfire/cli.py:440 msgid "Update the package indexes." msgstr "" -#: ../python/pakfire/cli.py:443 ../python/pakfire/cli.py:708 +#: ../python/pakfire/cli.py:446 ../python/pakfire/cli.py:739 msgid "Build one or more packages." msgstr "" -#: ../python/pakfire/cli.py:445 ../python/pakfire/cli.py:710 +#: ../python/pakfire/cli.py:448 ../python/pakfire/cli.py:633 +#: ../python/pakfire/cli.py:741 msgid "Give name of at least one package to build." msgstr "" -#: ../python/pakfire/cli.py:449 ../python/pakfire/cli.py:714 +#: ../python/pakfire/cli.py:452 ../python/pakfire/cli.py:745 +#: ../python/pakfire/cli.py:815 msgid "Build the package for the given architecture." msgstr "" -#: ../python/pakfire/cli.py:451 ../python/pakfire/cli.py:479 -#: ../python/pakfire/cli.py:716 +#: ../python/pakfire/cli.py:454 ../python/pakfire/cli.py:482 +#: ../python/pakfire/cli.py:747 msgid "Path were the output files should be copied to." msgstr "" -#: ../python/pakfire/cli.py:453 ../python/pakfire/cli.py:468 -#: ../python/pakfire/cli.py:718 +#: ../python/pakfire/cli.py:456 ../python/pakfire/cli.py:471 +#: ../python/pakfire/cli.py:749 msgid "Mode to run in. Is either 'release' or 'development' (default)." msgstr "" -#: ../python/pakfire/cli.py:455 +#: ../python/pakfire/cli.py:458 msgid "Run a shell after a successful build." msgstr "" -#: ../python/pakfire/cli.py:460 +#: ../python/pakfire/cli.py:463 msgid "Go into a shell." msgstr "" -#: ../python/pakfire/cli.py:462 +#: ../python/pakfire/cli.py:465 msgid "Give name of a package." msgstr "" -#: ../python/pakfire/cli.py:466 +#: ../python/pakfire/cli.py:469 msgid "Emulated architecture in the shell." msgstr "" -#: ../python/pakfire/cli.py:473 +#: ../python/pakfire/cli.py:476 msgid "Generate a source package." msgstr "" -#: ../python/pakfire/cli.py:475 +#: ../python/pakfire/cli.py:478 msgid "Give name(s) of a package(s)." msgstr "" -#: ../python/pakfire/cli.py:484 +#: ../python/pakfire/cli.py:487 msgid "Create a build environment cache." msgstr "" -#: ../python/pakfire/cli.py:494 +#: ../python/pakfire/cli.py:497 msgid "Create a new build environment cache." msgstr "" -#: ../python/pakfire/cli.py:499 +#: ../python/pakfire/cli.py:502 msgid "Remove all cached build environments." msgstr "" -#: ../python/pakfire/cli.py:577 +#: ../python/pakfire/cli.py:580 #, python-format msgid "Removing environment cache file: %s..." msgstr "" -#: ../python/pakfire/cli.py:583 +#: ../python/pakfire/cli.py:586 #, python-format msgid "Could not remove file: %s" msgstr "" -#: ../python/pakfire/cli.py:589 +#: ../python/pakfire/cli.py:592 msgid "Pakfire server command line interface." msgstr "" -#: ../python/pakfire/cli.py:628 -msgid "Request a build job from the server." +#: ../python/pakfire/cli.py:631 +msgid "Send a scrach build job to the server." msgstr "" -#: ../python/pakfire/cli.py:634 -msgid "Send a keepalive to the server." +#: ../python/pakfire/cli.py:635 +msgid "Limit build to only these architecture(s)." msgstr "" #: ../python/pakfire/cli.py:641 +msgid "Send a keepalive to the server." +msgstr "" + +#: ../python/pakfire/cli.py:648 msgid "Update all repositories." msgstr "" -#: ../python/pakfire/cli.py:647 +#: ../python/pakfire/cli.py:654 msgid "Repository management commands." msgstr "" -#: ../python/pakfire/cli.py:655 +#: ../python/pakfire/cli.py:662 msgid "Create a new repository index." msgstr "" -#: ../python/pakfire/cli.py:656 +#: ../python/pakfire/cli.py:663 msgid "Path to the packages." msgstr "" -#: ../python/pakfire/cli.py:657 +#: ../python/pakfire/cli.py:664 msgid "Path to input packages." msgstr "" -#: ../python/pakfire/cli.py:662 +#: ../python/pakfire/cli.py:669 msgid "Dump some information about this machine." msgstr "" -#: ../python/pakfire/cli.py:720 +#: ../python/pakfire/cli.py:751 msgid "Do not verify build dependencies." msgstr "" +#: ../python/pakfire/cli.py:775 +msgid "Pakfire client command line interface." +msgstr "" + +#: ../python/pakfire/cli.py:809 +msgid "Build a package remotely." +msgstr "" + +#: ../python/pakfire/cli.py:811 +msgid "Give name of a package to build." +msgstr "" + +#: ../python/pakfire/cli.py:820 +msgid "Print some information about this host." +msgstr "" + +#: ../python/pakfire/cli.py:826 +msgid "Check the connection to the hub." +msgstr "" + +#: ../python/pakfire/cli.py:853 ../python/pakfire/server.py:302 +msgid "Hostname" +msgstr "" + +#: ../python/pakfire/cli.py:854 +msgid "Pakfire hub" +msgstr "" + +#: ../python/pakfire/cli.py:857 +msgid "Username" +msgstr "" + +#. Hardware information +#: ../python/pakfire/cli.py:861 ../python/pakfire/server.py:306 +msgid "Hardware information" +msgstr "" + +#: ../python/pakfire/cli.py:862 ../python/pakfire/server.py:307 +msgid "CPU model" +msgstr "" + +#: ../python/pakfire/cli.py:863 ../python/pakfire/server.py:308 +msgid "Memory" +msgstr "" + +#: ../python/pakfire/cli.py:865 ../python/pakfire/server.py:310 +msgid "Native arch" +msgstr "" + +#: ../python/pakfire/cli.py:867 ../python/pakfire/server.py:312 +msgid "Supported arches" +msgstr "" + +#: ../python/pakfire/cli.py:880 +msgid "Your IP address" +msgstr "" + +#: ../python/pakfire/cli.py:885 +msgid "You are authenticated to the build service:" +msgstr "" + +#: ../python/pakfire/cli.py:891 +msgid "User name" +msgstr "" + +#: ../python/pakfire/cli.py:892 +msgid "Real name" +msgstr "" + +#: ../python/pakfire/cli.py:893 +msgid "Email address" +msgstr "" + +#: ../python/pakfire/cli.py:894 +msgid "Registered" +msgstr "" + +#: ../python/pakfire/cli.py:901 +msgid "You could not be authenticated to the build service." +msgstr "" + +#: ../python/pakfire/cli.py:910 +msgid "Pakfire daemon command line interface." +msgstr "" + +#: ../python/pakfire/client/transport.py:55 +#, python-format +msgid "Socket error: %s" +msgstr "" + +#: ../python/pakfire/client/transport.py:79 +#, python-format +msgid "Trying again in %s seconds. %s tries left." +msgstr "" + #: ../python/pakfire/compress.py:85 ../python/pakfire/compress.py:95 #, python-format msgid "Given algorithm '%s' is not supported." msgstr "" +#. Parse the file. +#: ../python/pakfire/config.py:256 +#, python-format +msgid "Reading from configuration file: %s" +msgstr "" + +#: ../python/pakfire/config.py:306 +msgid "Configuration:" +msgstr "" + +#: ../python/pakfire/config.py:308 +#, python-format +msgid "Section: %s" +msgstr "" + +#: ../python/pakfire/config.py:313 +msgid "No settings in this section." +msgstr "" + +#: ../python/pakfire/config.py:315 +msgid "Loaded from files:" +msgstr "" + #: ../python/pakfire/downloader.py:134 msgid "Downloading source files:" msgstr "" @@ -630,11 +754,11 @@ msgstr "" msgid "File" msgstr "" -#: ../python/pakfire/packages/base.py:361 +#: ../python/pakfire/packages/base.py:362 msgid "Not set" msgstr "" -#: ../python/pakfire/packages/base.py:526 +#: ../python/pakfire/packages/base.py:527 #, python-format msgid "Config file saved as %s." msgstr "" @@ -664,19 +788,30 @@ msgstr "" msgid "Could not remove file: /%s" msgstr "" -#: ../python/pakfire/packages/make.py:78 +#: ../python/pakfire/packages/make.py:79 msgid "Package name is undefined." msgstr "" -#: ../python/pakfire/packages/make.py:81 +#: ../python/pakfire/packages/make.py:82 msgid "Package version is undefined." msgstr "" -#: ../python/pakfire/packages/make.py:407 +#: ../python/pakfire/packages/make.py:416 #, python-format msgid "Searching for automatic dependencies for %s..." msgstr "" +#: ../python/pakfire/packages/make.py:463 +#, python-format +msgid "Regular experession is invalid and has been skipped: %s" +msgstr "" + +#. Let the user know what has been done. +#: ../python/pakfire/packages/make.py:479 +#, python-format +msgid "Filter %s filtered %s." +msgstr "" + #. Load progressbar. #: ../python/pakfire/packages/packager.py:362 msgid "Packaging" @@ -764,35 +899,10 @@ msgstr "" msgid " Solutions:" msgstr "" -#: ../python/pakfire/server.py:267 +#: ../python/pakfire/server.py:278 ../python/pakfire/system.py:114 msgid "Could not be determined" msgstr "" -#: ../python/pakfire/server.py:291 -msgid "Hostname" -msgstr "" - -#. Hardware information -#: ../python/pakfire/server.py:295 -msgid "Hardware information" -msgstr "" - -#: ../python/pakfire/server.py:296 -msgid "CPU model" -msgstr "" - -#: ../python/pakfire/server.py:297 -msgid "Memory" -msgstr "" - -#: ../python/pakfire/server.py:299 -msgid "Native arch" -msgstr "" - -#: ../python/pakfire/server.py:301 -msgid "Supported arches" -msgstr "" - #: ../python/pakfire/transaction.py:91 #, python-format msgid "file %s from %s conflicts with file from package %s" @@ -1085,14 +1195,14 @@ msgstr "" msgid "The error that lead to this:" msgstr "" -#: ../tools/pakfire-multicall.py:67 +#: ../tools/pakfire-multicall.py:69 msgid "An error has occured when running Pakfire." msgstr "" -#: ../tools/pakfire-multicall.py:70 +#: ../tools/pakfire-multicall.py:72 msgid "Error message:" msgstr "" -#: ../tools/pakfire-multicall.py:74 +#: ../tools/pakfire-multicall.py:76 msgid "Further description:" msgstr "" diff --git a/python/pakfire/api.py b/python/pakfire/api.py index 58c5f8537..c8bff043f 100644 --- a/python/pakfire/api.py +++ b/python/pakfire/api.py @@ -20,6 +20,7 @@ ############################################################################### import base +import client from errors import * @@ -85,7 +86,7 @@ def grouplist(group, **pakfire_args): return pakfire.grouplist(group) def _build(pkg, resultdir, **kwargs): - pakfire = Pakfire(**kwargs) + pakfire = Pakfire(mode="builder", **kwargs) return pakfire._build(pkg, resultdir, **kwargs) diff --git a/python/pakfire/builder.py b/python/pakfire/builder.py index 9b33694bb..f7a1f0f17 100644 --- a/python/pakfire/builder.py +++ b/python/pakfire/builder.py @@ -42,6 +42,7 @@ import _pakfire import logging log = logging.getLogger("pakfire") +from system import system from constants import * from i18n import _ from errors import BuildError, BuildRootLocked, Error @@ -54,9 +55,9 @@ BUILD_LOG_HEADER = """ | __/ (_| | <| _| | | | __/ | |_) | |_| | | | (_| | __/ | |_| \__,_|_|\_\_| |_|_| \___| |_.__/ \__,_|_|_|\__,_|\___|_| - Time : %(time)s - Host : %(host)s Version : %(version)s + Host : %(hostname)s (%(host_arch)s) + Time : %(time)s """ @@ -85,7 +86,7 @@ class BuildEnviron(object): # Setup the logging. if logfile: - self.log = logging.getLogger(self.build_id) + self.log = log.getChild(self.build_id) # Propage everything to the root logger that we will see something # on the terminal. self.log.propagate = 1 @@ -106,9 +107,10 @@ class BuildEnviron(object): # are running in release mode. if self.mode == "release": logdata = { - "host" : socket.gethostname(), - "time" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()), - "version" : "Pakfire %s" % PAKFIRE_VERSION, + "host_arch" : system.arch, + "hostname" : system.hostname, + "time" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()), + "version" : "Pakfire %s" % PAKFIRE_VERSION, } for line in BUILD_LOG_HEADER.splitlines(): @@ -336,8 +338,13 @@ class BuildEnviron(object): if not requires: return - self.pakfire.install(requires, interactive=False, - allow_downgrade=True, logger=self.log) + try: + self.pakfire.install(requires, interactive=False, + allow_downgrade=True, logger=self.log) + + # Catch dependency errors and log it. + except DependencyError, e: + raise def install_test(self): pkgs = [] @@ -540,6 +547,17 @@ class BuildEnviron(object): return env + @property + def installed_packages(self): + """ + Returns an iterator over all installed packages in this build environment. + """ + # Get the repository of all installed packages. + repo = self.pakfire.repos.get_repo("@system") + + # Return an iterator over the packages. + return iter(repo) + def do(self, command, shell=True, personality=None, logger=None, *args, **kwargs): ret = None diff --git a/python/pakfire/cli.py b/python/pakfire/cli.py index 3e13dbc04..bdb564a81 100644 --- a/python/pakfire/cli.py +++ b/python/pakfire/cli.py @@ -23,13 +23,17 @@ import argparse import os import sys +import pakfire.api as pakfire + +import client +import config import logger import packages import repository import server import util -import pakfire.api as pakfire +from system import system from constants import * from i18n import _ @@ -109,7 +113,7 @@ class Cli(object): return ret - def parse_common_arguments(self): + def parse_common_arguments(self, repo_manage_switches=True, offline_switch=True): self.parser.add_argument("--version", action="version", version="%(prog)s " + PAKFIRE_VERSION) @@ -119,14 +123,16 @@ class Cli(object): self.parser.add_argument("-c", "--config", nargs="?", help=_("Path to a configuration file to load.")) - self.parser.add_argument("--disable-repo", nargs="*", metavar="REPO", - help=_("Disable a repository temporarily.")) + if repo_manage_switches: + self.parser.add_argument("--disable-repo", nargs="*", metavar="REPO", + help=_("Disable a repository temporarily.")) - self.parser.add_argument("--enabled-repo", nargs="*", metavar="REPO", - help=_("Enable a repository temporarily.")) + self.parser.add_argument("--enabled-repo", nargs="*", metavar="REPO", + help=_("Enable a repository temporarily.")) - self.parser.add_argument("--offline", action="store_true", - help=_("Run pakfire in offline mode.")) + if offline_switch: + self.parser.add_argument("--offline", action="store_true", + help=_("Run pakfire in offline mode.")) def parse_command_install(self): # Implement the "install" command. @@ -272,13 +278,10 @@ class Cli(object): def run(self): action = self.args.action - if not self.action2func.has_key(action): - raise - try: func = self.action2func[action] except KeyError: - raise # XXX catch and return better error message + raise Exception, "Unhandled action: %s" % action return func() @@ -624,9 +627,13 @@ class CliServer(Cli): def parse_command_build(self): # Implement the "build" command. - sub_keepalive = self.sub_commands.add_parser("build", - help=_("Request a build job from the server.")) - sub_keepalive.add_argument("action", action="store_const", const="build") + sub_build = self.sub_commands.add_parser("build", + help=_("Send a scrach build job to the server.")) + sub_build.add_argument("package", nargs=1, + help=_("Give name of at least one package to build.")) + sub_build.add_argument("--arch", "-a", + help=_("Limit build to only these architecture(s).")) + sub_build.add_argument("action", action="store_const", const="build") def parse_command_keepalive(self): # Implement the "keepalive" command. @@ -666,7 +673,31 @@ class CliServer(Cli): self.server.update_info() def handle_build(self): - self.server.build_job() + # Arch. + if self.args.arch: + arches = self.args.arch.split() + + (package,) = self.args.package + + self.server.create_scratch_build({}) + return + + # Temporary folter for source package. + tmpdir = "/tmp/pakfire-%s" % util.random_string() + + try: + os.makedirs(tmpdir) + + pakfire.dist(package, resultdir=[tmpdir,]) + + for file in os.listdir(tmpdir): + file = os.path.join(tmpdir, file) + + print file + + finally: + if os.path.exists(tmpdir): + util.rm(tmpdir) def handle_repoupdate(self): self.server.update_repositories() @@ -735,5 +766,172 @@ class CliBuilderIntern(Cli): } pakfire._build(pkg, builder_mode=self.args.mode, - distro_config=distro_config, resultdir=self.args.resultdir, - nodeps=self.args.nodeps, **self.pakfire_args) + distro_config=distro_config, resultdir=self.args.resultdir,) + + +class CliClient(Cli): + def __init__(self): + self.parser = argparse.ArgumentParser( + description = _("Pakfire client command line interface."), + ) + + self.parse_common_arguments(repo_manage_switches=True, offline_switch=True) + + # Add sub-commands. + self.sub_commands = self.parser.add_subparsers() + + self.parse_command_build() + self.parse_command_connection_check() + self.parse_command_info() + + # Finally parse all arguments from the command line and save them. + self.args = self.parser.parse_args() + + self.action2func = { + "build" : self.handle_build, + "conn-check" : self.handle_connection_check, + "info" : self.handle_info, + } + + # Read configuration for the pakfire client. + self.conf = conf = config.ConfigClient() + + # Create connection to pakfire hub. + self.client = client.PakfireUserClient( + conf.get("client", "server"), + conf.get("client", "username"), + conf.get("client", "password"), + ) + + def parse_command_build(self): + # Parse "build" command. + sub_build = self.sub_commands.add_parser("build", + help=_("Build a package remotely.")) + sub_build.add_argument("package", nargs=1, + help=_("Give name of a package to build.")) + sub_build.add_argument("action", action="store_const", const="build") + + sub_build.add_argument("-a", "--arch", + help=_("Build the package for the given architecture.")) + + def parse_command_info(self): + # Implement the "info" command. + sub_info = self.sub_commands.add_parser("info", + help=_("Print some information about this host.")) + sub_info.add_argument("action", action="store_const", const="info") + + def parse_command_connection_check(self): + # Implement the "conn-check" command. + sub_conn_check = self.sub_commands.add_parser("conn-check", + help=_("Check the connection to the hub.")) + sub_conn_check.add_argument("action", action="store_const", const="conn-check") + + def handle_build(self): + (package,) = self.args.package + + # XXX just for now, we do only upload source pfm files. + assert os.path.exists(package) + + # Format arches. + if self.args.arch: + arches = self.args.arch.replace(",", " ") + else: + arches = None + + # Create a new build on the server. + build = self.client.build_create(package, arches=arches) + + # XXX Print the resulting build. + print build + + def handle_info(self): + ret = [] + + ret.append("") + ret.append(" PAKFIRE %s" % PAKFIRE_VERSION) + ret.append("") + ret.append(" %-20s: %s" % (_("Hostname"), system.hostname)) + ret.append(" %-20s: %s" % (_("Pakfire hub"), self.conf.get("client", "server"))) + if self.conf.get("client", "username") and self.conf.get("client", "password"): + ret.append(" %-20s: %s" % \ + (_("Username"), self.conf.get("client", "username"))) + ret.append("") + + # Hardware information + ret.append(" %s:" % _("Hardware information")) + ret.append(" %-16s: %s" % (_("CPU model"), system.cpu_model)) + ret.append(" %-16s: %s" % (_("Memory"), util.format_size(system.memory))) + ret.append("") + ret.append(" %-16s: %s" % (_("Native arch"), system.arch)) + + header = _("Supported arches") + for arch in system.supported_arches: + ret.append(" %-16s: %s" % (header, arch)) + header = "" + ret.append("") + + for line in ret: + print line + + def handle_connection_check(self): + ret = [] + + address = self.client.get_my_address() + ret.append(" %-20s: %s" % (_("Your IP address"), address)) + ret.append("") + + authenticated = self.client.check_auth() + if authenticated: + ret.append(" %s" % _("You are authenticated to the build service:")) + + user = self.client.get_user_profile() + assert user, "Could not fetch user infomation" + + keys = [ + ("name", _("User name")), + ("realname", _("Real name")), + ("email", _("Email address")), + ("registered", _("Registered")), + ] + + for key, desc in keys: + ret.append(" %-18s: %s" % (desc, user.get(key))) + + else: + ret.append(_("You could not be authenticated to the build service.")) + + for line in ret: + print line + + +class CliDaemon(Cli): + def __init__(self): + self.parser = argparse.ArgumentParser( + description = _("Pakfire daemon command line interface."), + ) + + self.parse_common_arguments(repo_manage_switches=True, offline_switch=True) + + # Finally parse all arguments from the command line and save them. + self.args = self.parser.parse_args() + + def run(self): + """ + Runs the pakfire daemon with provided settings. + """ + # Read the configuration file for the daemon. + conf = config.ConfigDaemon() + + # Create daemon instance. + d = pakfire.client.PakfireDaemon( + server = conf.get("daemon", "server"), + hostname = conf.get("daemon", "hostname"), + secret = conf.get("daemon", "secret"), + ) + + try: + d.run() + + # We cannot just kill the daemon, it needs a smooth shutdown. + except (SystemExit, KeyboardInterrupt): + d.shutdown() diff --git a/python/pakfire/client/__init__.py b/python/pakfire/client/__init__.py new file mode 100644 index 000000000..64adfd012 --- /dev/null +++ b/python/pakfire/client/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/python + +from base import PakfireUserClient, PakfireBuilderClient +from builder import PakfireDaemon, ClientBuilder diff --git a/python/pakfire/client/base.py b/python/pakfire/client/base.py new file mode 100644 index 000000000..eda836ef2 --- /dev/null +++ b/python/pakfire/client/base.py @@ -0,0 +1,195 @@ +#!/usr/bin/python + +from __future__ import division + +import os +import socket +import urlparse +import xmlrpclib + +import pakfire.util +import pakfire.packages as packages +from pakfire.system import system + +# Local modules. +import transport + +from pakfire.constants import * + +import logging +log = logging.getLogger("pakfire.client") + +class PakfireClient(object): + type = None + + def __init__(self, server, username, password): + self.url = self._join_url(server, username, password) + + # Create a secure XMLRPC connection to the server. + self.conn = transport.Connection(self.url) + + def _join_url(self, server, username, password): + """ + Construct a right URL out of the given + server, username and password. + + Basicly this just adds the credentials + to the URL. + """ + assert self.type + + # Parse the given URL. + url = urlparse.urlparse(server) + assert url.scheme in ("http", "https") + + # Build new URL. + ret = "%s://" % url.scheme + + # Add credentials if provided. + if username and password: + ret += "%s:%s@" % (username, password) + + # Add host and path components. + ret += "%s/pakfirehub/%s" % (url.netloc, self.type) + + return ret + + ### Misc. actions + + def noop(self): + """ + No operation. Just to check if the connection is + working. Returns a random number. + """ + return self.conn.noop() + + def get_my_address(self): + """ + Get my own address (as seen by the hub). + """ + return self.conn.get_my_address() + + def get_hub_status(self): + """ + Get some status information about the hub. + """ + return self.conn.get_hub_status() + + +class BuildMixin(object): + ### Build actions + + def build_create(self, filename, arches=None, distro=None): + """ + Create a new build on the hub. + """ + + # Upload the source file to the server. + upload_id = self._upload_file(filename) + + # Then create the build. + build = self.conn.build_create(upload_id, distro, arches) + + print build + + def _upload_file(self, filename): + # Get the hash of the file. + hash = pakfire.util.calc_hash1(filename) + + # Get the size of the file. + size = os.path.getsize(filename) + + # Get an upload ID from the server. + upload_id = self.conn.upload_create(os.path.basename(filename), + size, hash) + + try: + # Calculate the number of chunks. + chunks = (size // CHUNK_SIZE) + 1 + + # Cut the file in pieces and upload them one after another. + with open(filename) as f: + chunk = 0 + while True: + data = f.read(CHUNK_SIZE) + if not data: + break + + chunk += 1 + log.info("Uploading chunk %s/%s of %s." % (chunk, chunks, + os.path.basename(filename))) + + data = xmlrpclib.Binary(data) + self.conn.upload_chunk(upload_id, data) + + # Tell the server, that we finished the upload. + ret = self.conn.upload_finished(upload_id) + + except: + # If anything goes wrong, try to delete the upload and raise + # the exception. + self.conn.upload_remove(upload_id) + + raise + + # If the server sends false, something happened with the upload that + # could not be recovered. + if ret: + logging.info("Upload of %s succeeded." % filename) + else: + logging.error("Upload of %s was not successful." % filename) + raise Exception, "Upload failed." + + return upload_id + + +class PakfireUserClient(BuildMixin, PakfireClient): + type = "user" + + def check_auth(self): + """ + Check if the user was successfully authenticated. + """ + return self.conn.check_auth() + + def get_user_profile(self): + """ + Get information about the user profile. + """ + return self.conn.get_user_profile() + + +class PakfireBuilderClient(BuildMixin, PakfireClient): + type = "builder" + + def send_keepalive(self, overload=None): + """ + Sends a little keepalive to the server and + updates the hardware information if the server + requests it. + """ + log.debug("Sending keepalive to the hub.") + + # Collect the current loadavg and send it to the hub. + loadavg = ", ".join(("%.2f" % round(l, 2) for l in os.getloadavg())) + + needs_update = self.conn.send_keepalive(loadavg, overload) + + if needs_update: + log.debug("The hub is requesting an update.") + self.send_update() + + def send_update(self): + log.info("Sending host information update to hub...") + + self.conn.send_update( + # Supported architectures. + system.supported_arches, + + # CPU information. + system.cpu_model, + system.cpu_count, + + # Amount of memory in bytes. + system.memory / 1024, + ) diff --git a/python/pakfire/client/builder.py b/python/pakfire/client/builder.py new file mode 100644 index 000000000..7df6b50cb --- /dev/null +++ b/python/pakfire/client/builder.py @@ -0,0 +1,470 @@ +#!/usr/bin/python + +import hashlib +import multiprocessing +import os +import sys +import tempfile +import time + +import pakfire.api +import pakfire.builder +import pakfire.config +import pakfire.downloader +import pakfire.system +import pakfire.util +from pakfire.system import system + +import base + +from pakfire.constants import * + +import logging +log = logging.getLogger("pakfire.client") + +def fork_builder(*args, **kwargs): + """ + Wrapper that runs ClientBuilder in a new process and catches + any exception to report it to the main process. + """ + try: + # Create new instance of the builder. + cb = ClientBuilder(*args, **kwargs) + + # Run the build: + cb.build() + + except Exception, e: + # XXX catch the exception and log it. + print e + + # End the process with an exit code. + sys.exit(1) + + +class PakfireDaemon(object): + """ + The PakfireDaemon class that creates a a new process per build + job and also handles the keepalive/abort stuff. + """ + def __init__(self, server, hostname, secret): + self.client = base.PakfireBuilderClient(server, hostname, secret) + self.conn = self.client.conn + + # Save login data (to create child processes). + self.server = server + self.hostname = hostname + self.__secret = secret + + # A list with all running processes. + self.processes = [] + self.pid2jobid = {} + + # Save when last keepalive was sent. + self._last_keepalive = 0 + + def run(self, heartbeat=1, max_processes=None): + # By default do not start more than two processes per CPU core. + if max_processes is None: + max_processes = system.cpu_count * 2 + log.debug("Maximum number of simultaneous processes is: %s" % max_processes) + + # Indicates when to try to request a new job or aborted builds. + last_job_request = 0 + last_abort_request = 0 + + # Main loop. + while True: + # Send the keepalive regularly. + self.send_keepalive() + + # Remove all finished builds. + # "removed" indicates, if a process has actually finished. + removed = self.remove_finished_builders() + + # If a build slot was freed, search immediately for a new job. + if removed: + last_job_request = 0 + + # Kill aborted jobs. + if time.time() - last_abort_request >= 60: + aborted = self.kill_aborted_jobs() + + # If a build slot was freed, search immediately for a new job. + if aborted: + last_job_request = 0 + + last_abort_request = time.time() + + # Check if the maximum number of processes was reached. + # Actually the hub does manage this but this is an emergency + # condition if anything goes wrong. + if self.num_processes >= max_processes: + log.debug("Reached maximum number of allowed processes (%s)." % max_processes) + + time.sleep(heartbeat) + continue + + # Get new job. + if time.time() - last_job_request >= 60 and not self.has_overload(): + # If the last job request is older than a minute and we don't + # have too much load, we go and check if there is something + # to do for us. + job = self.get_job() + + # If we got a job, we start a child process to work on it. + if job: + log.debug("Got a new job.") + self.fork_builder(job) + else: + log.debug("No new job.") + + # Update the time when we requested a job. + last_job_request = time.time() + + # Wait a moment before starting over. + time.sleep(heartbeat) + + def shutdown(self): + """ + Shut down the daemon. + This means to kill all child processes. + + The method blocks until all processes are shut down. + """ + for process in self.processes: + log.info("Sending %s to terminate..." % process) + + process.terminate() + else: + log.info("No processes to kill. Shutting down immediately.") + + while self.processes: + log.debug("%s process(es) is/are still running..." % len(self.processes)) + + for process in self.processes[:]: + if not process.is_alive(): + # The process has terminated. + log.info("Process %s terminated with exit code: %s" % \ + (process, process.exitcode)) + + self.processes.remove(process) + + @property + def num_processes(self): + # Return the number of processes. + return len(self.processes) + + def get_job(self): + """ + Get a build job from the hub. + """ + log.info("Requesting a new job from the server...") + + # Get some information about this system. + s = pakfire.system.System() + + # Fetch a build job from the hub. + return self.client.conn.build_get_job(s.supported_arches) + + def has_overload(self): + """ + Checks, if the load average is not too high. + + On this is to be decided if a new job is taken. + """ + try: + load1, load5, load15 = os.getloadavg() + except OSError: + # Could not determine the current loadavg. In that case we + # assume that we don't have overload. + return False + + # If there are more than 2 processes in the process queue per CPU + # core we will assume that the system has heavy load and to not request + # a new job. + return load5 >= system.cpu_count * 2 + + def send_keepalive(self): + """ + When triggered, this method sends a keepalive to the hub. + """ + # Do not send a keepalive more often than twice a minute. + if time.time() - self._last_keepalive < 30: + return + + self.client.send_keepalive(overload=self.has_overload()) + self._last_keepalive = time.time() + + def remove_finished_builders(self): + # Return if any processes have been removed. + ret = False + + # Search for any finished processes. + for process in self.processes[:]: + # If the process is not alive anymore... + if not process.is_alive(): + ret = True + + # ... check the exit code and log a message on errors. + if process.exitcode == 0: + log.debug("Process %s exited normally." % process) + + elif process.exitcode > 0: + log.error("Process did not exit normally: %s code: %s" \ + % (process, process.exitcode)) + + elif process.exitcode < 0: + log.error("Process killed by signal: %s: code: %s" \ + % (process, process.exitcode)) + + # If a program has crashed, we send that to the hub. + job_id = self.pid2jobid.get(process.pid, None) + if job_id: + self.conn.build_job_crashed(job_id, process.exitcode) + + # Finally, remove the process from the process list. + self.processes.remove(process) + + return ret + + def kill_aborted_jobs(self): + log.debug("Requesting aborted jobs...") + + # Get a list of running job ids: + running_jobs = self.pid2jobid.values() + + # If there are no running jobs, there is nothing to do. + if not running_jobs: + return False + + # Ask the hub for any build jobs to abort. + aborted_jobs = self.conn.build_jobs_aborted(running_jobs) + + # If no build jobs were aborted, there is nothing to do. + if not aborted_jobs: + return False + + for process in self.processes[:]: + job_id = self.pid2jobid.get(process.pid, None) + if job_id and job_id in aborted_jobs: + + # Kill the process. + log.info("Killing process %s which was aborted by the user." \ + % process.pid) + process.terminate() + + # Remove the process from the process list to avoid + # that is will be cleaned up in the normal way. + self.processes.remove(process) + + return True + + def fork_builder(self, job): + """ + For a new child process to create a new independent builder. + """ + # Create the Process object. + process = multiprocessing.Process(target=fork_builder, + args=(self.server, self.hostname, self.__secret, job)) + # The process is running in daemon mode so it will try to kill + # all child processes when exiting. + process.daemon = True + + # Start the process. + process.start() + log.info("Started new process %s with PID %s." % (process, process.pid)) + + # Save the PID and the build id to track down + # crashed builds. + self.pid2jobid[process.pid] = job.get("id", None) + + # Append it to the process list. + self.processes.append(process) + + +class ClientBuilder(object): + def __init__(self, server, hostname, secret, job): + self.client = base.PakfireBuilderClient(server, hostname, secret) + self.conn = self.client.conn + + # Store the information sent by the server here. + self.build_job = job + + def update_state(self, state, message=None): + self.conn.build_job_update_state(self.build_id, state, message) + + def upload_file(self, filename, type): + assert os.path.exists(filename) + assert type in ("package", "log") + + # First upload the file data and save the upload_id. + upload_id = self.client._upload_file(filename) + + # Add the file to the build. + return self.conn.build_job_add_file(self.build_id, upload_id, type) + + def upload_buildroot(self, installed_packages): + pkgs = [] + + for pkg in installed_packages: + pkgs.append((pkg.friendly_name, pkg.uuid)) + + return self.conn.build_upload_buildroot(self.build_id, pkgs) + + @property + def build_id(self): + if self.build_job: + return self.build_job.get("id", None) + + @property + def build_arch(self): + if self.build_job: + return self.build_job.get("arch", None) + + @property + def build_source_url(self): + if self.build_job: + return self.build_job.get("source_url", None) + + @property + def build_source_filename(self): + if self.build_source_url: + return os.path.basename(self.build_source_url) + + @property + def build_source_hash512(self): + if self.build_job: + return self.build_job.get("source_hash512", None) + + @property + def build_type(self): + if self.build_job: + return self.build_job.get("type", None) + + def build(self): + # Cannot go on if I got no build job. + if not self.build_job: + logging.info("No job to work on...") + return + + # Call the function that processes the build and try to catch general + # exceptions and report them to the server. + # If everything goes okay, we tell this the server, too. + try: + # Create a temporary file and a directory for the resulting files. + tmpdir = tempfile.mkdtemp() + tmpfile = os.path.join(tmpdir, self.build_source_filename) + logfile = os.path.join(tmpdir, "build.log") + + # Get a package grabber and add mirror download capabilities to it. + grabber = pakfire.downloader.PackageDownloader(pakfire.config.Config()) + + try: + ## Download the source. + grabber.urlgrab(self.build_source_url, filename=tmpfile) + + # Check if the download checksum matches (if provided). + if self.build_source_hash512: + h = hashlib.sha512() + f = open(tmpfile, "rb") + while True: + buf = f.read(BUFFER_SIZE) + if not buf: + break + + h.update(buf) + f.close() + + if not self.build_source_hash512 == h.hexdigest(): + raise DownloadError, "Hash check did not succeed." + + # Create dist with arguments that are passed to the pakfire + # builder. + kwargs = { + # Of course this is a release build. + # i.e. don't use local packages. + "builder_mode" : "release", + + # Set the build_id we got from the build service. + "build_id" : self.build_id, + + # Files and directories (should be self explaining). + "logfile" : logfile, + + # Distro configuration. + "distro_config" : { + "arch" : self.build_arch, + }, + } + + # Create a new instance of the builder. + build = pakfire.builder.BuildEnviron(tmpfile, **kwargs) + + try: + # Create the build environment. + build.start() + + # Update the build status on the server. + self.upload_buildroot(build.installed_packages) + self.update_state("running") + + # Run the build (with install test). + build.build(install_test=True) + + # Copy the created packages to the tempdir. + build.copy_result(tmpdir) + + finally: + # Cleanup the build environment. + build.stop() + + # Jippie, build is finished, we are going to upload the files. + self.update_state("uploading") + + # Walk through the result directory and upload all (binary) files. + # Skip that for test builds. + if not self.build_type == "test": + for dir, subdirs, files in os.walk(tmpdir): + for file in files: + file = os.path.join(dir, file) + if file in (logfile, tmpfile,): + continue + + self.upload_file(file, "package") + + except DependencyError, e: + message = "%s: %s" % (e.__class__.__name__, e) + self.update_state("dependency_error", message) + raise + + except DownloadError, e: + message = "%s: %s" % (e.__class__.__name__, e) + self.update_state("download_error", message) + raise + + finally: + # Upload the logfile in any case and if it exists. + if os.path.exists(logfile): + self.upload_file(logfile, "log") + + # Cleanup the files we created. + pakfire.util.rm(tmpdir) + + except DependencyError: + # This has already been reported. + raise + + except (DownloadError,): + # Do not take any further action for these exceptions. + pass + + except Exception, e: + # Format the exception and send it to the server. + message = "%s: %s" % (e.__class__.__name__, e) + + self.update_state("failed", message) + raise + + else: + self.update_state("finished") diff --git a/python/pakfire/client/test.py b/python/pakfire/client/test.py new file mode 100644 index 000000000..8bf5fd44f --- /dev/null +++ b/python/pakfire/client/test.py @@ -0,0 +1,54 @@ +#!/usr/bin/python + +import random +import sys +import time + +def fork_builder(*args, **kwargs): + cb = ClientBuilder(*args, **kwargs) + + try: + cb() + except Exception, e: + print e + sys.exit(1) + +class ClientBuilder(object): + def __init__(self, id): + self.id = id + + def __call__(self, *args): + print "Running", self.id, args + + time.sleep(2) + + if random.choice((False, False, False, True)): + raise Exception, "Process died" + + +import multiprocessing + + +processes = [] + +while True: + # Check if there are at least 2 processes running. + if len(processes) < 2: + process = multiprocessing.Process(target=fork_builder, args=(len(processes),)) + + process.daemon = True + process.start() + + processes.append(process) + + print len(processes), "in process list:", processes + + for process in processes: + time.sleep(0.5) + + print process.name, "is alive?", process.is_alive() + + if not process.is_alive(): + print "Removing process", process + print " Exitcode:", process.exitcode + processes.remove(process) diff --git a/python/pakfire/client/transport.py b/python/pakfire/client/transport.py new file mode 100644 index 000000000..cb78c6668 --- /dev/null +++ b/python/pakfire/client/transport.py @@ -0,0 +1,118 @@ +#!/usr/bin/python +############################################################################### +# # +# Pakfire - The IPFire package management system # +# Copyright (C) 2011 Pakfire development team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +import socket +import time +import xmlrpclib + +import logging +log = logging.getLogger("pakfire.client") + +from pakfire.constants import * +from pakfire.i18n import _ + +class XMLRPCMixin: + user_agent = "pakfire/%s" % PAKFIRE_VERSION + + def single_request(self, *args, **kwargs): + ret = None + + # Tries can be passed to this method. + tries = kwargs.pop("tries", 100) + timeout = 1 + + while tries: + try: + ret = xmlrpclib.Transport.single_request(self, *args, **kwargs) + + except socket.error, e: + # These kinds of errors are not fatal, but they can happen on + # a bad internet connection or whatever. + # 32 Broken pipe + # 110 Connection timeout + # 111 Connection refused + if not e.errno in (32, 110, 111,): + raise + + log.warning(_("Socket error: %s") % e) + + except xmlrpclib.ProtocolError, e: + # Log all XMLRPC protocol errors. + log.error("XMLRPC protocol error:") + log.error(" URL: %s" % e.url) + log.error(" HTTP headers:") + for header in e.headers.items(): + log.error(" %s: %s" % header) + log.error(" Error code: %s" % e.errcode) + log.error(" Error message: %s" % e.errmsg) + raise + + else: + # If request was successful, we can break the loop. + break + + # If the request was not successful, we wait a little time to try + # it again. + tries -= 1 + timeout *= 2 + if timeout > 60: + timeout = 60 + + log.warning(_("Trying again in %s seconds. %s tries left.") % (timeout, tries)) + time.sleep(timeout) + + else: + log.error("Maximum number of tries was reached. Giving up.") + # XXX need better exception here. + raise Exception, "Could not fulfill request." + + return ret + + +class XMLRPCTransport(XMLRPCMixin, xmlrpclib.Transport): + """ + Handles the XMLRPC connection over HTTP. + """ + pass + + +class SafeXMLRPCTransport(XMLRPCMixin, xmlrpclib.SafeTransport): + """ + Handles the XMLRPC connection over HTTPS. + """ + pass + + +class Connection(xmlrpclib.ServerProxy): + """ + Class wrapper that automatically chooses the right transport + method depending on the given URL. + """ + + def __init__(self, url): + # Create transport channel to the server. + if url.startswith("https://"): + transport = SafeXMLRPCTransport() + elif url.startswith("http://"): + transport = XMLRPCTransport() + + xmlrpclib.ServerProxy.__init__(self, url, transport=transport, + allow_none=True) diff --git a/python/pakfire/config.py b/python/pakfire/config.py index 38dd012fa..3bddcf9e7 100644 --- a/python/pakfire/config.py +++ b/python/pakfire/config.py @@ -20,6 +20,7 @@ ############################################################################### import os +import socket from ConfigParser import ConfigParser @@ -27,8 +28,10 @@ import logging log = logging.getLogger("pakfire") import base +from system import system from constants import * +from i18n import _ class Config(object): def __init__(self, type=None): @@ -138,6 +141,7 @@ class Config(object): if self.type == "builder": path = os.getcwd() + _path = None while not path == "/": _path = os.path.join(path, "config") if os.path.exists(_path): @@ -204,3 +208,142 @@ class Config(object): Check if this host can build for the target architecture "arch". """ return arch in self.supported_arches + +class _Config(object): + files = [] + + # A dict with default settings for this config class. + default_settings = {} + + def __init__(self, path="/etc"): + # Configuration settings. + self._config = self.default_settings.copy() + + # List of files that were already loaded. + self._files = [] + + # Read default configuration file. + self.read(*[os.path.join(path, f) for f in self.files]) + + # Dump read configuration. + self.dump() + + def _read_hook(self, config): + """ + Method to be overwritten when addition stuff + should be done with the ConfigParser object. + """ + pass + + def read(self, *files): + # Do nothing for no files. + if not files: + return + + # Create configparser and read the content of the file + # to parse it. + config = ConfigParser() + for f in files: + # Normalize filename. + f = os.path.abspath(f) + + # Check if file has already been read or + # does not exist. Then skip it. + if f in self._files or not os.path.exists(f): + continue + + # Parse the file. + log.debug(_("Reading from configuration file: %s") % f) + config.read(f) + + # Save the filename to the list of read files. + self._files.append(f) + + # Read all data from the configuration file in the _config dict. + for section in config.sections(): + items = dict(config.items(section)) + + try: + self._config[section].update(items) + except KeyError: + self._config[section] = items + + def set(self, section, key, value): + try: + self._config[section][key] = value + except KeyError: + self._config[section] = { key : value } + + def get(self, section, key, default=None): + try: + return self._config[section][key] + except KeyError: + return default + + def get_int(self, section, key, default=None): + val = self.get(section=section, key=key, default=default) + try: + val = int(val) + except ValueError: + return default + + def get_bool(self, section, key, default=None): + val = self.get(section=section, key=key, default=default) + + if val in (True, "true", "1", "on"): + return True + elif val in (False, "false", "0", "off"): + return False + + return default + + def dump(self): + """ + Dump the configuration that was read. + + (Only in debugging mode.) + """ + log.debug(_("Configuration:")) + for section, settings in self._config.items(): + log.debug(" " + _("Section: %s") % section) + + for k, v in settings.items(): + log.debug(" %-20s: %s" % (k, v)) + else: + log.debug(" " + _("No settings in this section.")) + + log.debug(" " + _("Loaded from files:")) + for f in self._files: + log.debug(" %s" % f) + + +class ConfigBuilder(_Config): + files = ["pakfire.conf", "pakfire-builder.conf"] + + +class ConfigClient(_Config): + files = ["pakfire.conf", "pakfire-client.conf"] + + default_settings = { + "client" : { + # The default server is the official Pakfire + # server. + "server" : "https://pakfire.ipfire.org", + }, + } + + +class ConfigDaemon(_Config): + files = ["pakfire.conf", "pakfire-daemon.conf"] + + default_settings = { + "daemon" : { + # The default server is the official Pakfire + # server. + "server" : "https://pakfire.ipfire.org", + + # The default hostname is the host name of this + # machine. + "hostname" : system.hostname, + }, + } diff --git a/python/pakfire/constants.py b/python/pakfire/constants.py index 7e3867f61..082eb27e4 100644 --- a/python/pakfire/constants.py +++ b/python/pakfire/constants.py @@ -49,6 +49,10 @@ REPOSITORY_DB = "index.db" BUFFER_SIZE = 102400 +# The size of the data chunks that are uploaded to the +# pakfire hub. +CHUNK_SIZE = BUFFER_SIZE + MIRRORLIST_MAXSIZE = 1024**2 MACRO_FILE_DIR = "/usr/lib/pakfire/macros" diff --git a/python/pakfire/packages/base.py b/python/pakfire/packages/base.py index 36656a0d9..062bfaefb 100644 --- a/python/pakfire/packages/base.py +++ b/python/pakfire/packages/base.py @@ -190,6 +190,7 @@ class Package(object): "release" : self.release, "epoch" : self.epoch, "arch" : self.arch, + "supported_arches" : self.supported_arches, "groups" : self.groups, "summary" : self.summary, "description" : self.description, diff --git a/python/pakfire/packages/make.py b/python/pakfire/packages/make.py index 9431cadd4..44e17e1e3 100644 --- a/python/pakfire/packages/make.py +++ b/python/pakfire/packages/make.py @@ -24,6 +24,7 @@ import re import shutil import socket import tarfile +import uuid from urlgrabber.grabber import URLGrabber, URLGrabError from urlgrabber.progress import TextMeter @@ -174,6 +175,19 @@ class MakefileBase(Package): # Not existant for Makefiles return None + @property + def supported_arches(self): + """ + These are the supported arches. Which means, packages of these + architectures can be built out of this source package. + """ + # If the package architecture is "noarch", the package + # needs only to be built for that. + if self.arch == "noarch": + return "noarch" + + return self.lexer.get_var("sup_arches", "all") + class Makefile(MakefileBase): @property @@ -192,14 +206,6 @@ class Makefile(MakefileBase): def arch(self): return "src" - @property - def supported_arches(self): - """ - These are the supported arches. Which means, packages of these - architectures can be built out of this source package. - """ - return self.lexer.get_var("sup_arches", "all") - @property def packages(self): pkgs = [] @@ -371,6 +377,9 @@ class MakefilePackage(MakefileBase): # Store additional dependencies in here. self._dependencies = {} + # Generate a random identifier. + self._uuid = "%s" % uuid.uuid4() + @property def name(self): return self._name @@ -389,7 +398,7 @@ class MakefilePackage(MakefileBase): @property def uuid(self): - return None + return self._uuid def track_dependencies(self, builder, path): # Build filelist with all files that have been installed. @@ -485,6 +494,10 @@ class MakefilePackage(MakefileBase): # Collect all dependencies that were discovered by the tracker. deps += self._dependencies.get(key, []) + # Add the UUID. + if key == "provides": + deps.append("uuid(%s)" % self.uuid) + # Remove duplicates. deps = set(deps) deps = list(deps) @@ -497,7 +510,11 @@ class MakefilePackage(MakefileBase): @property def requires(self): - return self.get_deps("requires") + # Make sure that no self-provides are in the list + # of requirements. + provides = self.provides + + return [r for r in self.get_deps("requires") if not r in provides] @property def provides(self): diff --git a/python/pakfire/packages/packager.py b/python/pakfire/packages/packager.py index 1ed2cca37..11fc0c3d2 100644 --- a/python/pakfire/packages/packager.py +++ b/python/pakfire/packages/packager.py @@ -226,7 +226,7 @@ class BinaryPackager(Packager): # Generic package information including Pakfire information. info.update({ "pakfire_version" : PAKFIRE_VERSION, - "uuid" : uuid.uuid4(), + "uuid" : self.pkg.uuid, "type" : "binary", }) diff --git a/python/pakfire/server.py b/python/pakfire/server.py index 9799628ff..f6ec78064 100644 --- a/python/pakfire/server.py +++ b/python/pakfire/server.py @@ -211,6 +211,18 @@ class XMLRPCTransport(xmlrpclib.Transport): return ret +class ServerProxy(xmlrpclib.ServerProxy): + def __init__(self, server, *args, **kwargs): + + # Some default settings. + if not kwargs.has_key("transport"): + kwargs["transport"] = XMLRPCTransport() + + kwargs["allow_none"] = True + + xmlrpclib.ServerProxy.__init__(self, server, *args, **kwargs) + + class Server(object): def __init__(self, **pakfire_args): self.config = pakfire.config.Config() @@ -219,8 +231,7 @@ class Server(object): log.info("Establishing RPC connection to: %s" % server) - self.conn = xmlrpclib.ServerProxy(server, transport=XMLRPCTransport(), - allow_none=True) + self.conn = ServerProxy(server) self.pakfire_args = pakfire_args @@ -483,3 +494,6 @@ class Server(object): (repo["distro"]["sname"], repo["name"], arch) pakfire.api.repo_create(path, files) + + def create_scratch_build(self, *args, **kwargs): + return self.conn.create_scratch_build(*args, **kwargs) diff --git a/python/pakfire/system.py b/python/pakfire/system.py index a5479b2b8..60b6ca081 100644 --- a/python/pakfire/system.py +++ b/python/pakfire/system.py @@ -21,7 +21,117 @@ from __future__ import division +import multiprocessing import os +import socket + +from i18n import _ + +class System(object): + """ + Class that grants access to several information about + the system this software is running on. + """ + @property + def hostname(self): + return socket.gethostname() + + @property + def arch(self): + """ + Return the architecture of the host we are running on. + """ + return os.uname()[4] + + @property + def supported_arches(self): + """ + Check what architectures can be built on this host. + """ + host_can_build = { + # Host arch : Can build these arches. + + # x86 + "x86_64" : ["x86_64", "i686",], + "i686" : ["i686",], + + # ARM + "armv5tel" : ["armv5tel",], + "armv5tejl" : ["armv5tel",], + "armv7l" : ["armv5tel",], + "armv7hl" : ["armv7hl", "armv5tel",], + } + + try: + return host_can_build[self.arch] + except KeyError: + return [] + + def host_supports_arch(self, arch): + """ + Check if this host can build for the target architecture "arch". + """ + return arch in self.supported_arches + + @property + def cpu_count(self): + """ + Count the number of CPU cores. + """ + return multiprocessing.cpu_count() + + @property + def cpu_model(self): + # Determine CPU model + cpuinfo = {} + with open("/proc/cpuinfo") as f: + for line in f.readlines(): + # Break at an empty line, because all information after that + # is redundant. + if not line: + break + + try: + key, value = line.split(":") + except: + pass # Skip invalid lines + + key, value = key.strip(), value.strip() + cpuinfo[key] = value + + ret = None + if self.arch.startswith("arm"): + try: + ret = "%(Hardware)s - %(Processor)s" % cpuinfo + except KeyError: + pass + else: + ret = cpuinfo.get("model name", None) + + # Remove too many spaces. + ret = " ".join(ret.split()) + + return ret or _("Could not be determined") + + @property + def memory(self): + # Determine memory size + memory = 0 + with open("/proc/meminfo") as f: + line = f.readline() + + try: + a, b, c = line.split() + except: + pass + else: + memory = int(b) * 1024 + + return memory + + +# Create an instance of this class to only keep it once in memory. +system = System() class Mountpoints(object): def __init__(self, pakfire, root="/"): @@ -172,3 +282,13 @@ class Mountpoint(object): assert file.name.startswith(self.path) self.disk_usage += file.size + + +if __name__ == "__main__": + print "Hostname", system.hostname + print "Arch", system.arch + print "Supported arches", system.supported_arches + + print "CPU Model", system.cpu_model + print "CPU count", system.cpu_count + print "Memory", system.memory diff --git a/python/pakfire/util.py b/python/pakfire/util.py index 5efa5044f..4bd40beed 100644 --- a/python/pakfire/util.py +++ b/python/pakfire/util.py @@ -182,7 +182,7 @@ def format_size(s): s /= 1024 unit += 1 - return "%d %s" % (int(s) * sign, units[unit]) + return "%d%s" % (round(s) * sign, units[unit]) def format_time(s): return "%02d:%02d" % (s // 60, s % 60) diff --git a/tools/Makefile b/tools/Makefile index 3edf918de..eb1315acb 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -51,6 +51,8 @@ install: $(SCRIPTS) -mkdir -pv $(DESTDIR)/usr/bin ln -svf ../..$(SCRIPT_DIR)/pakfire-multicall.py $(DESTDIR)/usr/bin/pakfire ln -svf ../..$(SCRIPT_DIR)/pakfire-multicall.py $(DESTDIR)/usr/bin/pakfire-builder + ln -svf ../..$(SCRIPT_DIR)/pakfire-multicall.py $(DESTDIR)/usr/bin/pakfire-client + ln -svf ../..$(SCRIPT_DIR)/pakfire-multicall.py $(DESTDIR)/usr/bin/pakfire-daemon ln -svf ../..$(SCRIPT_DIR)/pakfire-multicall.py $(DESTDIR)/usr/bin/pakfire-server ln -svf pakfire-multicall.py $(DESTDIR)$(SCRIPT_DIR)/builder diff --git a/tools/pakfire-multicall.py b/tools/pakfire-multicall.py index 864873e77..0e5ab8073 100755 --- a/tools/pakfire-multicall.py +++ b/tools/pakfire-multicall.py @@ -36,6 +36,8 @@ except ImportError, e: basename2cls = { "pakfire" : Cli, "pakfire-builder" : CliBuilder, + "pakfire-client" : CliClient, + "pakfire-daemon" : CliDaemon, "pakfire-server" : CliServer, "builder" : CliBuilderIntern, }