From 3817ae8e9db55cae60b268d4a9263a35dc1108e9 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Fri, 9 Dec 2011 00:07:45 +0100 Subject: [PATCH] Add option to create build environment cache. Signed-off-by: Michael Tremer --- po/pakfire.pot | 164 +++++++++++++++++++++++---------- python/pakfire/api.py | 4 + python/pakfire/base.py | 32 +++++-- python/pakfire/builder.py | 176 ++++++++++++++++++++++++++++++------ python/pakfire/cli.py | 40 ++++++++ python/pakfire/constants.py | 1 + 6 files changed, 336 insertions(+), 81 deletions(-) diff --git a/po/pakfire.pot b/po/pakfire.pot index 26d18068..b188c099 100644 --- a/po/pakfire.pot +++ b/po/pakfire.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2011-12-04 19:11+0100\n" +"POT-Creation-Date: 2011-12-08 23:57+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -39,38 +39,45 @@ msgstr "" msgid "The scriptlet ran more than %s seconds and was killed." msgstr "" -#: ../python/pakfire/actions.py:251 ../python/pakfire/actions.py:289 -#: ../python/pakfire/actions.py:312 ../python/pakfire/actions.py:335 -#: ../python/pakfire/actions.py:352 ../python/pakfire/actions.py:371 +#: ../python/pakfire/actions.py:200 +#, python-format +msgid "" +"The scriptlet returned with an unhandled error:\n" +"%s" +msgstr "" + +#: ../python/pakfire/actions.py:254 ../python/pakfire/actions.py:292 +#: ../python/pakfire/actions.py:315 ../python/pakfire/actions.py:338 +#: ../python/pakfire/actions.py:355 ../python/pakfire/actions.py:374 #, python-format msgid "Running transaction test for %s" msgstr "" -#: ../python/pakfire/actions.py:260 ../python/pakfire/actions.py:364 +#: ../python/pakfire/actions.py:263 ../python/pakfire/actions.py:367 msgid "Installing" msgstr "" -#: ../python/pakfire/actions.py:298 +#: ../python/pakfire/actions.py:301 msgid "Updating" msgstr "" -#: ../python/pakfire/actions.py:318 +#: ../python/pakfire/actions.py:321 msgid "Removing" msgstr "" #. Cleaning up leftover files and stuff. -#: ../python/pakfire/actions.py:342 +#: ../python/pakfire/actions.py:345 msgid "Cleanup" msgstr "" -#: ../python/pakfire/actions.py:380 +#: ../python/pakfire/actions.py:383 msgid "Downgrading" msgstr "" #: ../python/pakfire/base.py:207 ../python/pakfire/base.py:237 #: ../python/pakfire/base.py:283 ../python/pakfire/base.py:334 -#: ../python/pakfire/base.py:400 ../python/pakfire/base.py:439 -#: ../python/pakfire/base.py:492 ../python/pakfire/base.py:512 +#: ../python/pakfire/base.py:400 ../python/pakfire/base.py:437 +#: ../python/pakfire/base.py:490 ../python/pakfire/base.py:510 msgid "Nothing to do" msgstr "" @@ -98,56 +105,95 @@ msgstr "" msgid "Excluding %s." msgstr "" -#: ../python/pakfire/base.py:480 +#: ../python/pakfire/base.py:478 #, python-format msgid "\"%s\" package does not seem to be installed." msgstr "" -#: ../python/pakfire/base.py:624 +#: ../python/pakfire/base.py:622 msgid "Build command has failed." msgstr "" -#: ../python/pakfire/base.py:704 +#: ../python/pakfire/base.py:702 msgid "Everything is fine." msgstr "" #. Log the package information. -#: ../python/pakfire/builder.py:139 +#: ../python/pakfire/builder.py:153 msgid "Package information:" msgstr "" -#. Copy the makefile and load source tarballs. -#: ../python/pakfire/builder.py:304 +#. Install all packages. +#: ../python/pakfire/builder.py:324 +msgid "Install packages needed for build..." +msgstr "" + +#: ../python/pakfire/builder.py:329 msgid "Extracting" msgstr "" -#: ../python/pakfire/builder.py:554 +#: ../python/pakfire/builder.py:575 +msgid "You cannot run a build when no package was given." +msgstr "" + +#: ../python/pakfire/builder.py:580 #, python-format msgid "Could not find makefile in build root: %s" msgstr "" -#: ../python/pakfire/builder.py:567 +#: ../python/pakfire/builder.py:594 msgid "The build command failed. See logfile for details." msgstr "" +#. Walk through the whole tree and collect all files +#. that are on the same disk (not crossing mountpoints). +#: ../python/pakfire/builder.py:652 +msgid "Creating filelist..." +msgstr "" + +#. Create a nice progressbar. +#: ../python/pakfire/builder.py:671 +msgid "Compressing files..." +msgstr "" + +#: ../python/pakfire/builder.py:690 +#, python-format +msgid "Cache file was successfully created at %s." +msgstr "" + +#: ../python/pakfire/builder.py:691 +#, python-format +msgid " Containing %(files)s files, it has a size of %(size)s." +msgstr "" + +#. Make a nice progress bar as always. +#: ../python/pakfire/builder.py:702 +msgid "Extracting files..." +msgstr "" + +#. Update all packages. +#: ../python/pakfire/builder.py:722 +msgid "Updating packages from cache..." +msgstr "" + #. Package the result. #. Make all these little package from the build environment. -#: ../python/pakfire/builder.py:714 +#: ../python/pakfire/builder.py:837 msgid "Creating packages:" msgstr "" #. Execute the buildscript of this stage. -#: ../python/pakfire/builder.py:734 +#: ../python/pakfire/builder.py:857 #, python-format msgid "Running stage %s:" msgstr "" -#: ../python/pakfire/builder.py:752 +#: ../python/pakfire/builder.py:875 #, python-format msgid "Could not remove static libraries: %s" msgstr "" -#: ../python/pakfire/builder.py:758 +#: ../python/pakfire/builder.py:881 msgid "Compressing man pages did not complete successfully." msgstr "" @@ -331,93 +377,115 @@ msgstr "" msgid "You cannot run pakfire-builder in a pakfire chroot." msgstr "" -#: ../python/pakfire/cli.py:379 ../python/pakfire/cli.py:636 +#: ../python/pakfire/cli.py:379 ../python/pakfire/cli.py:676 msgid "Pakfire builder command line interface." msgstr "" -#: ../python/pakfire/cli.py:434 +#: ../python/pakfire/cli.py:437 msgid "Update the package indexes." msgstr "" -#: ../python/pakfire/cli.py:440 ../python/pakfire/cli.py:656 +#: ../python/pakfire/cli.py:443 ../python/pakfire/cli.py:696 msgid "Build one or more packages." msgstr "" -#: ../python/pakfire/cli.py:442 ../python/pakfire/cli.py:658 +#: ../python/pakfire/cli.py:445 ../python/pakfire/cli.py:698 msgid "Give name of at least one package to build." msgstr "" -#: ../python/pakfire/cli.py:446 ../python/pakfire/cli.py:662 +#: ../python/pakfire/cli.py:449 ../python/pakfire/cli.py:702 msgid "Build the package for the given architecture." msgstr "" -#: ../python/pakfire/cli.py:448 ../python/pakfire/cli.py:476 -#: ../python/pakfire/cli.py:664 +#: ../python/pakfire/cli.py:451 ../python/pakfire/cli.py:479 +#: ../python/pakfire/cli.py:704 msgid "Path were the output files should be copied to." msgstr "" -#: ../python/pakfire/cli.py:450 ../python/pakfire/cli.py:465 -#: ../python/pakfire/cli.py:666 +#: ../python/pakfire/cli.py:453 ../python/pakfire/cli.py:468 +#: ../python/pakfire/cli.py:706 msgid "Mode to run in. Is either 'release' or 'development' (default)." msgstr "" -#: ../python/pakfire/cli.py:452 +#: ../python/pakfire/cli.py:455 msgid "Run a shell after a successful build." msgstr "" -#: ../python/pakfire/cli.py:457 +#: ../python/pakfire/cli.py:460 msgid "Go into a shell." msgstr "" -#: ../python/pakfire/cli.py:459 +#: ../python/pakfire/cli.py:462 msgid "Give name of a package." msgstr "" -#: ../python/pakfire/cli.py:463 +#: ../python/pakfire/cli.py:466 msgid "Emulated architecture in the shell." msgstr "" -#: ../python/pakfire/cli.py:470 +#: ../python/pakfire/cli.py:473 msgid "Generate a source package." msgstr "" -#: ../python/pakfire/cli.py:472 +#: ../python/pakfire/cli.py:475 msgid "Give name(s) of a package(s)." msgstr "" -#: ../python/pakfire/cli.py:549 +#: ../python/pakfire/cli.py:484 +msgid "Create a build environment cache." +msgstr "" + +#: ../python/pakfire/cli.py:494 +msgid "Create a new build environment cache." +msgstr "" + +#: ../python/pakfire/cli.py:499 +msgid "Remove all cached build environments." +msgstr "" + +#: ../python/pakfire/cli.py:577 +#, python-format +msgid "Removing environment cache file: %s..." +msgstr "" + +#: ../python/pakfire/cli.py:583 +#, python-format +msgid "Could not remove file: %s" +msgstr "" + +#: ../python/pakfire/cli.py:589 msgid "Pakfire server command line interface." msgstr "" -#: ../python/pakfire/cli.py:586 +#: ../python/pakfire/cli.py:626 msgid "Request a build job from the server." msgstr "" -#: ../python/pakfire/cli.py:592 +#: ../python/pakfire/cli.py:632 msgid "Send a keepalive to the server." msgstr "" -#: ../python/pakfire/cli.py:599 +#: ../python/pakfire/cli.py:639 msgid "Update all repositories." msgstr "" -#: ../python/pakfire/cli.py:605 +#: ../python/pakfire/cli.py:645 msgid "Repository management commands." msgstr "" -#: ../python/pakfire/cli.py:613 +#: ../python/pakfire/cli.py:653 msgid "Create a new repository index." msgstr "" -#: ../python/pakfire/cli.py:614 +#: ../python/pakfire/cli.py:654 msgid "Path to the packages." msgstr "" -#: ../python/pakfire/cli.py:615 +#: ../python/pakfire/cli.py:655 msgid "Path to input packages." msgstr "" -#: ../python/pakfire/cli.py:668 +#: ../python/pakfire/cli.py:708 msgid "Do not verify build dependencies." msgstr "" @@ -636,7 +704,7 @@ msgid "Loading from %s" msgstr "" #. Add all packages from the database to the index. -#: ../python/pakfire/repository/index.py:447 +#: ../python/pakfire/repository/index.py:438 msgid "Loading installed packages" msgstr "" diff --git a/python/pakfire/api.py b/python/pakfire/api.py index 03380153..58c5f853 100644 --- a/python/pakfire/api.py +++ b/python/pakfire/api.py @@ -131,3 +131,7 @@ def check(**pakfire_args): pakfire = Pakfire(**pakfire_args) return pakfire.check() + +# Cache functions +def cache_create(**pakfire_args): + return Pakfire.cache_create(**pakfire_args) diff --git a/python/pakfire/base.py b/python/pakfire/base.py index cde5992e..b065d712 100644 --- a/python/pakfire/base.py +++ b/python/pakfire/base.py @@ -405,7 +405,7 @@ class Pakfire(object): t.run() - def update(self, pkgs, check=False, excludes=None, allow_vendorchange=False, allow_archchange=False): + def update(self, pkgs=None, check=False, excludes=None, interactive=True, logger=None, **kwargs): """ check indicates, if the method should return after calculation of the transaction. @@ -431,9 +431,7 @@ class Pakfire(object): request.lock(exclude) solver = self.create_solver() - t = solver.solve(request, update=update, - allow_vendorchange=allow_vendorchange, - allow_archchange=allow_archchange) + t = solver.solve(request, update=update, **kwargs) if not t: log.info(_("Nothing to do")) @@ -447,11 +445,11 @@ class Pakfire(object): # Just exit here, because we won't do the transaction in this mode. if check: - t.dump() + t.dump(logger=logger) return # Ask the user if the transaction is okay. - if not t.cli_yesno(): + if interactive and not t.cli_yesno(): return # Run the transaction. @@ -710,3 +708,25 @@ class Pakfire(object): # Process the transaction. t.run() + + @staticmethod + def cache_create(**kwargs): + # Create a build environment that we are going to pack into + # a shiny tarball. + b = builder.BuildEnviron(**kwargs) + p = b.pakfire + + # Get filename of the file from builder instance. + filename = b.cache_file + + try: + b.start() + + # Create directory if not existant. + dirname = os.path.dirname(filename) + if not os.path.exists(dirname): + os.makedirs(dirname) + + b.cache_export(filename) + finally: + b.stop() diff --git a/python/pakfire/builder.py b/python/pakfire/builder.py index fb2a5a16..d587bd72 100644 --- a/python/pakfire/builder.py +++ b/python/pakfire/builder.py @@ -33,9 +33,11 @@ import base import chroot import logger import packages +import packages.file import packages.packager import repository import util +import _pakfire import logging log = logging.getLogger("pakfire") @@ -62,8 +64,8 @@ class BuildEnviron(object): # The version of the kernel this machine is running. kernel_version = os.uname()[2] - def __init__(self, filename, distro_config=None, build_id=None, logfile=None, - builder_mode="release", **pakfire_args): + def __init__(self, filename=None, distro_config=None, build_id=None, logfile=None, + builder_mode="release", use_cache=None, **pakfire_args): # Set mode. assert builder_mode in ("development", "release",) self.mode = builder_mode @@ -122,27 +124,42 @@ class BuildEnviron(object): # Where do we put the result? self.resultdir = os.path.join(self.path, "result") - # Open package. - # If we have a plain makefile, we first build a source package and go with that. - if filename.endswith(".%s" % MAKEFILE_EXTENSION): - pkg = packages.Makefile(self.pakfire, filename) - pkg.dist([self.resultdir,]) - - filename = os.path.join(self.resultdir, "src", pkg.package_filename) - assert os.path.exists(filename), filename - - # Open source package. - self.pkg = packages.SourcePackage(self.pakfire, None, filename) - assert self.pkg, filename + # Check weather to use or not use the cache. + if use_cache is None: + # If use_cache is None, the user did not provide anything and + # so we guess. + if self.mode == "development": + use_cache = True + else: + use_cache = False - # Log the package information. - self.log.info(_("Package information:")) - for line in self.pkg.dump(long=True).splitlines(): - self.log.info(" %s" % line) - self.log.info("") + self.use_cache = use_cache - # Path where we extract the package and put all the source files. - self.build_dir = os.path.join(self.path, "usr/src", self.pkg.friendly_name) + # Open package. + # If we have a plain makefile, we first build a source package and go with that. + if filename: + if filename.endswith(".%s" % MAKEFILE_EXTENSION): + pkg = packages.Makefile(self.pakfire, filename) + pkg.dist([self.resultdir,]) + + filename = os.path.join(self.resultdir, "src", pkg.package_filename) + assert os.path.exists(filename), filename + + # Open source package. + self.pkg = packages.SourcePackage(self.pakfire, None, filename) + assert self.pkg, filename + + # Log the package information. + self.log.info(_("Package information:")) + for line in self.pkg.dump(long=True).splitlines(): + self.log.info(" %s" % line) + self.log.info("") + + # Path where we extract the package and put all the source files. + self.build_dir = os.path.join(self.path, "usr/src", self.pkg.friendly_name) + else: + # No package :( + self.pkg = None # XXX need to make this configureable self.settings = { @@ -280,8 +297,13 @@ class BuildEnviron(object): if not requires: requires = [] - # Add neccessary build dependencies. - requires += BUILD_PACKAGES + if self.use_cache: + # If we are told to use the cache, we just import the + # file. + self.cache_extract() + else: + # Add neccessary build dependencies. + requires += BUILD_PACKAGES # If we have ccache enabled, we need to extract it # to the build chroot. @@ -294,14 +316,17 @@ class BuildEnviron(object): requires.append("icecream") # Get build dependencies from source package. - for req in self.pkg.requires: - requires.append(req) + if self.pkg: + for req in self.pkg.requires: + requires.append(req) # Install all packages. + self.log.info(_("Install packages needed for build...")) self.install(requires) # Copy the makefile and load source tarballs. - self.pkg.extract(_("Extracting"), prefix=self.build_dir) + if self.pkg: + self.pkg.extract(_("Extracting"), prefix=self.build_dir) def install(self, requires): """ @@ -546,7 +571,8 @@ class BuildEnviron(object): return ret def build(self, install_test=True): - assert self.pkg + if not self.pkg: + raise BuildError, _("You cannot run a build when no package was given.") # Search for the package file in build_dir and raise BuildError if it is not present. pkgfile = os.path.join(self.build_dir, "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION)) @@ -601,6 +627,102 @@ class BuildEnviron(object): shell = os.system(command) return os.WEXITSTATUS(shell) + @property + def cache_file(self): + comps = [ + self.pakfire.distro.sname, # name of the distribution + self.pakfire.distro.release, # release version + self.pakfire.distro.arch, # architecture + ] + + return os.path.join(CACHE_ENVIRON_DIR, "%s.cache" %"-".join(comps)) + + def cache_export(self, filename): + # Sync all disk caches. + _pakfire.sync() + + # A list to store all mountpoints, so we don't package them. + mountpoints = [] + + # A list containing all files we want to package. + filelist = [] + + # Walk through the whole tree and collect all files + # that are on the same disk (not crossing mountpoints). + log.info(_("Creating filelist...")) + root = self.chrootPath() + for dir, subdirs, files in os.walk(root): + # Search for mountpoints and skip them. + if not dir == root and os.path.ismount(dir): + mountpoints.append(dir) + continue + + # Skip all directories under mountpoints. + if any([dir.startswith(m) for m in mountpoints]): + continue + + # Add all other files. + filelist.append(dir) + for file in files: + file = os.path.join(dir, file) + filelist.append(file) + + # Create a nice progressbar. + p = util.make_progress(_("Compressing files..."), len(filelist)) + i = 0 + + # Create tar file and add all files to it. + f = packages.file.InnerTarFile.open(filename, "w:gz") + for file in filelist: + i += 1 + if p: + p.update(i) + + f.add(file, os.path.relpath(file, root), recursive=False) + f.close() + + # Finish progressbar. + if p: + p.finish() + + filesize = os.path.getsize(filename) + + log.info(_("Cache file was successfully created at %s.") % filename) + log.info(_(" Containing %(files)s files, it has a size of %(size)s.") % \ + { "files" : len(filelist), "size" : util.format_size(filesize), }) + + def cache_extract(self): + root = self.chrootPath() + filename = self.cache_file + + f = packages.file.InnerTarFile.open(filename, "r:gz") + members = f.getmembers() + + # Make a nice progress bar as always. + p = util.make_progress(_("Extracting files..."), len(members)) + + # Extract all files from the cache. + i = 0 + for member in members: + if p: + i += 1 + p.update(i) + + f.extract(member, path=root) + f.close() + + # Finish progressbar. + if p: + p.finish() + + # Re-read local repository. + self.pakfire.repos.local.update(force=True) + + # Update all packages. + self.log.info(_("Updating packages from cache...")) + self.pakfire.update(interactive=False, logger=self.log, + allow_archchange=True, allow_vendorchange=True, allow_downgrade=True) + class Builder(object): def __init__(self, pakfire, filename, resultdir, **kwargs): diff --git a/python/pakfire/cli.py b/python/pakfire/cli.py index 6956b94a..7b82215e 100644 --- a/python/pakfire/cli.py +++ b/python/pakfire/cli.py @@ -395,6 +395,7 @@ class CliBuilder(Cli): self.parse_command_repolist() self.parse_command_clean() self.parse_command_resolvdep() + self.parse_command_cache() # Finally parse all arguments from the command line and save them. self.args = self.parser.parse_args() @@ -411,6 +412,8 @@ class CliBuilder(Cli): "repolist" : self.handle_repolist, "clean_all" : self.handle_clean_all, "resolvdep" : self.handle_resolvdep, + "cache_create": self.handle_cache_create, + "cache_cleanup": self.handle_cache_cleanup, } @property @@ -475,6 +478,27 @@ class CliBuilder(Cli): sub_dist.add_argument("--resultdir", nargs="?", help=_("Path were the output files should be copied to.")) + def parse_command_cache(self): + # Implement the "cache" command. + sub_cache = self.sub_commands.add_parser("cache", + help=_("Create a build environment cache.")) + + # Implement subcommands. + sub_cache_commands = sub_cache.add_subparsers() + + self.parse_command_cache_create(sub_cache_commands) + self.parse_command_cache_cleanup(sub_cache_commands) + + def parse_command_cache_create(self, sub_commands): + sub_create = sub_commands.add_parser("create", + help=_("Create a new build environment cache.")) + sub_create.add_argument("action", action="store_const", const="cache_create") + + def parse_command_cache_cleanup(self, sub_commands): + sub_cleanup = sub_commands.add_parser("cleanup", + help=_("Remove all cached build environments.")) + sub_cleanup.add_argument("action", action="store_const", const="cache_cleanup") + def handle_info(self): Cli.handle_info(self, long=True) @@ -542,6 +566,22 @@ class CliBuilder(Cli): for pkg in pkgs: print pkg.dump(long=True) + def handle_cache_create(self): + pakfire.cache_create(**self.pakfire_args) + + def handle_cache_cleanup(self): + for env in os.listdir(CACHE_ENVIRON_DIR): + if not env.endswith(".cache"): + continue + + print _("Removing environment cache file: %s..." % env) + env = os.path.join(CACHE_ENVIRON_DIR, env) + + try: + os.unlink(env) + except OSError: + print _("Could not remove file: %s") % env + class CliServer(Cli): def __init__(self): diff --git a/python/pakfire/constants.py b/python/pakfire/constants.py index 305da5fc..7e3867f6 100644 --- a/python/pakfire/constants.py +++ b/python/pakfire/constants.py @@ -36,6 +36,7 @@ CONFIG_FILE = os.path.join(SYSCONFDIR, "pakfire.conf") CACHE_DIR = "/var/cache/pakfire" CCACHE_CACHE_DIR = os.path.join(CACHE_DIR, "ccache") +CACHE_ENVIRON_DIR = os.path.join(CACHE_DIR, "environments") REPO_CACHE_DIR = os.path.join(CACHE_DIR, "repos") LOCAL_BUILD_REPO_PATH = "/var/lib/pakfire/local" -- 2.39.2