]>
git.ipfire.org Git - pakfire.git/blob - python/pakfire/builder.py
2 ###############################################################################
4 # Pakfire - The IPFire package management system #
5 # Copyright (C) 2011 Pakfire development team #
7 # This program is free software: you can redistribute it and/or modify #
8 # it under the terms of the GNU General Public License as published by #
9 # the Free Software Foundation, either version 3 of the License, or #
10 # (at your option) any later version. #
12 # This program is distributed in the hope that it will be useful, #
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
15 # GNU General Public License for more details. #
17 # You should have received a copy of the GNU General Public License #
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. #
20 ###############################################################################
36 import packages
.packager
41 log
= logging
.getLogger("pakfire")
43 from constants
import *
45 from errors
import BuildError
, BuildRootLocked
, Error
48 BUILD_LOG_HEADER
= """
50 | _ \ __ _| | __/ _(_)_ __ ___ | |__ _ _(_) | __| | ___ _ __
51 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
52 | __/ (_| | <| _| | | | __/ | |_) | |_| | | | (_| | __/ |
53 |_| \__,_|_|\_\_| |_|_| \___| |_.__/ \__,_|_|_|\__,_|\___|_|
61 class BuildEnviron(object):
62 # The version of the kernel this machine is running.
63 kernel_version
= os
.uname()[2]
65 def __init__(self
, filename
, distro_config
=None, build_id
=None, logfile
=None,
66 builder_mode
="release", **pakfire_args
):
68 assert builder_mode
in ("development", "release",)
69 self
.mode
= builder_mode
71 # Disable the build repository in release mode.
72 if self
.mode
== "release":
73 if pakfire_args
.has_key("disable_repos") and pakfire_args
["disable_repos"]:
74 pakfire_args
["disable_repos"] += ["build",]
76 pakfire_args
["disable_repos"] = ["build",]
78 # Save the build id and generate one if no build id was provided.
80 build_id
= "%s" % uuid
.uuid4()
82 self
.build_id
= build_id
86 self
.log
= logging
.getLogger(self
.build_id
)
87 # Propage everything to the root logger that we will see something
89 self
.log
.propagate
= 1
90 self
.log
.setLevel(logging
.INFO
)
92 # Add the given logfile to the logger.
93 h
= logging
.FileHandler(logfile
)
94 self
.log
.addHandler(h
)
96 # Format the log output for the file.
97 f
= logger
.BuildFormatter()
100 # If no logile was given, we use the root logger.
101 self
.log
= logging
.getLogger("pakfire")
103 # Log information about pakfire and some more information, when we
104 # are running in release mode.
105 if self
.mode
== "release":
107 "host" : socket
.gethostname(),
108 "time" : time
.strftime("%a, %d %b %Y %H:%M:%S +0000", time
.gmtime()),
109 "version" : "Pakfire %s" % PAKFIRE_VERSION
,
112 for line
in BUILD_LOG_HEADER
.splitlines():
113 self
.log
.info(line
% logdata
)
115 # Create pakfire instance.
116 if pakfire_args
.has_key("mode"):
117 del pakfire_args
["mode"]
118 self
.pakfire
= base
.Pakfire(mode
="builder", distro_config
=distro_config
, **pakfire_args
)
119 self
.distro
= self
.pakfire
.distro
120 self
.path
= self
.pakfire
.path
122 # Where do we put the result?
123 self
.resultdir
= os
.path
.join(self
.path
, "result")
126 # If we have a plain makefile, we first build a source package and go with that.
127 if filename
.endswith(".%s" % MAKEFILE_EXTENSION
):
128 pkg
= packages
.Makefile(self
.pakfire
, filename
)
129 pkg
.dist([self
.resultdir
,])
131 filename
= os
.path
.join(self
.resultdir
, "src", pkg
.package_filename
)
132 assert os
.path
.exists(filename
), filename
134 # Open source package.
135 self
.pkg
= packages
.SourcePackage(self
.pakfire
, None, filename
)
136 assert self
.pkg
, filename
138 # Log the package information.
139 self
.log
.info(_("Package information:"))
140 for line
in self
.pkg
.dump(long=True).splitlines():
141 self
.log
.info(" %s" % line
)
144 # Path where we extract the package and put all the source files.
145 self
.build_dir
= os
.path
.join(self
.path
, "usr/src", self
.pkg
.friendly_name
)
147 # XXX need to make this configureable
149 "enable_loop_devices" : True,
150 "enable_ccache" : True,
151 "enable_icecream" : False,
153 #self.settings.update(settings)
159 # Save the build time.
160 self
.build_time
= int(time
.time())
163 # Mount the directories.
169 # Setup domain name resolution in chroot.
172 # Extract all needed packages.
176 # Kill all still running processes.
177 util
.orphans_kill(self
.path
)
179 # Close pakfire instance.
182 # Umount the build environment.
191 Inherit architecture from distribution configuration.
193 return self
.distro
.arch
198 "build_date" : time
.strftime("%a, %d %b %Y %H:%M:%S +0000", time
.gmtime(self
.build_time
)),
199 "build_host" : socket
.gethostname(),
200 "build_id" : self
.build_id
,
201 "build_time" : self
.build_time
,
205 filename
= os
.path
.join(self
.path
, ".lock")
208 self
._lock
= open(filename
, "a+")
213 fcntl
.lockf(self
._lock
.fileno(), fcntl
.LOCK_EX | fcntl
.LOCK_NB
)
215 raise BuildRootLocked
, "Buildroot is locked"
224 def copyin(self
, file_out
, file_in
):
225 if file_in
.startswith("/"):
226 file_in
= file_in
[1:]
228 file_in
= self
.chrootPath(file_in
)
230 #if not os.path.exists(file_out):
233 dir_in
= os
.path
.dirname(file_in
)
234 if not os
.path
.exists(dir_in
):
237 self
.log
.debug("%s --> %s" % (file_out
, file_in
))
239 shutil
.copy2(file_out
, file_in
)
241 def copyout(self
, file_in
, file_out
):
242 if file_in
.startswith("/"):
243 file_in
= file_in
[1:]
245 file_in
= self
.chrootPath(file_in
)
247 #if not os.path.exists(file_in):
250 dir_out
= os
.path
.dirname(file_out
)
251 if not os
.path
.exists(dir_out
):
254 self
.log
.debug("%s --> %s" % (file_in
, file_out
))
256 shutil
.copy2(file_in
, file_out
)
258 def copy_result(self
, resultdir
):
259 dir_in
= self
.chrootPath("result")
261 for dir, subdirs
, files
in os
.walk(dir_in
):
262 basename
= os
.path
.basename(dir)
263 dir = dir[len(self
.chrootPath()):]
265 file_in
= os
.path
.join(dir, file)
267 file_out
= os
.path
.join(
273 self
.copyout(file_in
, file_out
)
275 def extract(self
, requires
=None, build_deps
=True):
277 Gets a dependency set and extracts all packages
283 # Add neccessary build dependencies.
284 requires
+= BUILD_PACKAGES
286 # If we have ccache enabled, we need to extract it
287 # to the build chroot.
288 if self
.settings
.get("enable_ccache"):
289 requires
.append("ccache")
291 # If we have icecream enabled, we need to extract it
292 # to the build chroot.
293 if self
.settings
.get("enable_icecream"):
294 requires
.append("icecream")
296 # Get build dependencies from source package.
297 for req
in self
.pkg
.requires
:
300 # Install all packages.
301 self
.install(requires
)
303 # Copy the makefile and load source tarballs.
304 self
.pkg
.extract(_("Extracting"), prefix
=self
.build_dir
)
306 def install(self
, requires
):
308 Install everything that is required in requires.
310 # If we got nothing to do, we quit immediately.
314 self
.pakfire
.install(requires
, interactive
=False,
315 allow_downgrade
=True, logger
=self
.log
)
317 def install_test(self
):
319 for dir, subdirs
, files
in os
.walk(self
.chrootPath("result")):
321 pkgs
.append(os
.path
.join(dir, file))
323 self
.pakfire
.localinstall(pkgs
, yes
=True, allow_uninstall
=True)
325 def chrootPath(self
, *args
):
326 # Remove all leading slashes
329 if arg
.startswith("/"):
334 ret
= os
.path
.join(self
.path
, *args
)
335 ret
= ret
.replace("//", "/")
337 assert ret
.startswith(self
.path
)
341 def populate_dev(self
):
355 # If we need loop devices (which are optional) we create them here.
356 if self
.settings
["enable_loop_devices"]:
357 for i
in range(0, 7):
358 nodes
.append("/dev/loop%d" % i
)
361 # Stat the original node of the host system and copy it to
363 node_stat
= os
.stat(node
)
365 self
._create
_node
(node
, node_stat
.st_mode
, node_stat
.st_rdev
)
367 os
.symlink("/proc/self/fd/0", self
.chrootPath("dev", "stdin"))
368 os
.symlink("/proc/self/fd/1", self
.chrootPath("dev", "stdout"))
369 os
.symlink("/proc/self/fd/2", self
.chrootPath("dev", "stderr"))
370 os
.symlink("/proc/self/fd", self
.chrootPath("dev", "fd"))
374 Add DNS resolution facility to chroot environment by copying
375 /etc/resolv.conf and /etc/hosts.
377 for i
in ("/etc/resolv.conf", "/etc/hosts"):
380 def _create_node(self
, filename
, mode
, device
):
381 self
.log
.debug("Create node: %s (%s)" % (filename
, mode
))
383 filename
= self
.chrootPath(filename
)
385 # Create parent directory if it is missing.
386 dirname
= os
.path
.dirname(filename
)
387 if not os
.path
.exists(dirname
):
390 os
.mknod(filename
, mode
, device
)
393 self
.log
.debug("Destroying environment %s" % self
.path
)
395 if os
.path
.exists(self
.path
):
399 self
.log
.debug("Cleaning environemnt.")
401 # Remove the build directory and buildroot.
402 dirs
= (self
.build_dir
, self
.chrootPath("result"),)
405 if not os
.path
.exists(d
):
412 self
.log
.debug("Mounting environment")
413 for src
, dest
, fs
, options
in self
.mountpoints
:
414 mountpoint
= self
.chrootPath(dest
)
416 options
= "-o %s" % options
418 # Eventually create mountpoint directory
419 if not os
.path
.exists(mountpoint
):
420 os
.makedirs(mountpoint
)
422 cmd
= "mount -n -t %s %s %s %s" % \
423 (fs
, options
, src
, mountpoint
)
424 chroot
.do(cmd
, shell
=True)
426 def _umountall(self
):
427 self
.log
.debug("Umounting environment")
430 for src
, dest
, fs
, options
in reversed(self
.mountpoints
):
431 if not dest
in mountpoints
:
432 mountpoints
.append(dest
)
434 for dest
in mountpoints
:
435 mountpoint
= self
.chrootPath(dest
)
437 chroot
.do("umount -n %s" % mountpoint
, raiseExc
=0, shell
=True)
440 def mountpoints(self
):
443 # Make root as a tmpfs.
445 # ("pakfire_root", "/", "tmpfs", "defaults"),
449 # src, dest, fs, options
450 ("pakfire_proc", "/proc", "proc", "nosuid,noexec,nodev"),
451 ("/proc/sys", "/proc/sys", "bind", "bind"),
452 ("/proc/sys", "/proc/sys", "bind", "bind,ro,remount"),
453 ("/sys", "/sys", "bind", "bind"),
454 ("/sys", "/sys", "bind", "bind,ro,remount"),
455 ("pakfire_tmpfs", "/dev", "tmpfs", "mode=755,nosuid"),
456 ("/dev/pts", "/dev/pts", "bind", "bind"),
457 ("pakfire_tmpfs", "/run", "tmpfs", "mode=755,nosuid,nodev"),
460 # If selinux is enabled.
461 if os
.path
.exists("/sys/fs/selinux"):
463 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
464 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
467 # If ccache support is requested, we bind mount the cache.
468 if self
.settings
.get("enable_ccache"):
469 # Create ccache cache directory if it does not exist.
470 if not os
.path
.exists(CCACHE_CACHE_DIR
):
471 os
.makedirs(CCACHE_CACHE_DIR
)
474 (CCACHE_CACHE_DIR
, "/var/cache/ccache", "bind", "bind"),
482 # Add HOME manually, because it is occasionally not set
483 # and some builds get in trouble then.
485 "TERM" : os
.environ
.get("TERM", "dumb"),
488 # Set the container that we can detect, if we are inside a
490 "container" : "pakfire-builder",
493 # Inherit environment from distro
494 env
.update(self
.pakfire
.distro
.environ
)
496 # Icecream environment settings
497 if self
.settings
.get("enable_icecream", False):
498 # Set the toolchain path
499 if self
.settings
.get("icecream_toolchain", None):
500 env
["ICECC_VERSION"] = self
.settings
.get("icecream_toolchain")
502 # Set preferred host if configured.
503 if self
.settings
.get("icecream_preferred_host", None):
504 env
["ICECC_PREFERRED_HOST"] = \
505 self
.settings
.get("icecream_preferred_host")
507 # XXX what do we need else?
511 def do(self
, command
, shell
=True, personality
=None, logger
=None, *args
, **kwargs
):
514 # Environment variables
517 if kwargs
.has_key("env"):
518 env
.update(kwargs
.pop("env"))
520 self
.log
.debug("Environment:")
521 for k
, v
in sorted(env
.items()):
522 self
.log
.debug(" %s=%s" % (k
, v
))
524 # Update personality it none was set
526 personality
= self
.distro
.personality
528 # Make every shell to a login shell because we set a lot of
529 # environment things there.
531 command
= ["bash", "--login", "-c", command
]
533 if not kwargs
.has_key("chrootPath"):
534 kwargs
["chrootPath"] = self
.chrootPath()
538 personality
=personality
,
548 def build(self
, install_test
=True):
551 # Search for the package file in build_dir and raise BuildError if it is not present.
552 pkgfile
= os
.path
.join(self
.build_dir
, "%s.%s" % (self
.pkg
.name
, MAKEFILE_EXTENSION
))
553 if not os
.path
.exists(pkgfile
):
554 raise BuildError
, _("Could not find makefile in build root: %s") % pkgfile
555 pkgfile
= os
.path
.relpath(pkgfile
, self
.chrootPath())
557 resultdir
= self
.chrootPath("/result")
559 # Create the build command, that is executed in the chroot.
560 build_command
= ["/usr/lib/pakfire/builder", "--offline", "build", pkgfile
,
561 "--nodeps", "--resultdir=/result",]
564 self
.do(" ".join(build_command
), logger
=self
.log
)
567 raise BuildError
, _("The build command failed. See logfile for details.")
569 # Perform install test.
573 # Copy the final packages and stuff.
576 def shell(self
, args
=[]):
577 if not util
.cli_is_interactive():
578 self
.log
.warning("Cannot run shell on non-interactive console.")
581 # Install all packages that are needed to run a shell.
582 self
.install(SHELL_PACKAGES
)
584 # XXX need to set CFLAGS here
585 command
= "/usr/sbin/chroot %s %s %s" % \
586 (self
.chrootPath(), SHELL_SCRIPT
, " ".join(args
))
588 # Add personality if we require one
589 if self
.pakfire
.distro
.personality
:
590 command
= "%s %s" % (self
.pakfire
.distro
.personality
, command
)
592 for key
, val
in self
.environ
.items():
593 command
= "%s=\"%s\" " % (key
, val
) + command
595 # Empty the environment
596 command
= "env -i - %s" % command
598 self
.log
.debug("Shell command: %s" % command
)
600 shell
= os
.system(command
)
601 return os
.WEXITSTATUS(shell
)
604 class Builder(object):
605 def __init__(self
, pakfire
, filename
, resultdir
, **kwargs
):
606 self
.pakfire
= pakfire
608 self
.filename
= filename
610 self
.resultdir
= resultdir
613 self
.pkg
= packages
.Makefile(self
.pakfire
, self
.filename
)
621 return self
.pkg
.buildroot
625 return self
.pakfire
.distro
631 # Get all definitions from the package.
632 environ
.update(self
.pkg
.exports
)
634 # Overwrite some definitions by default values.
635 environ
.update(self
._environ
)
639 def do(self
, command
, shell
=True, personality
=None, cwd
=None, *args
, **kwargs
):
640 # Environment variables
641 log
.debug("Environment:")
642 for k
, v
in sorted(self
.environ
.items()):
643 log
.debug(" %s=%s" % (k
, v
))
645 # Update personality it none was set
647 personality
= self
.distro
.personality
650 cwd
= "/%s" % LOCAL_TMP_PATH
652 # Make every shell to a login shell because we set a lot of
653 # environment things there.
655 command
= ["bash", "--login", "-c", command
]
659 personality
=personality
,
662 logger
=logging
.getLogger("pakfire"),
668 def create_icecream_toolchain(self
):
670 out
= self
.do("icecc --build-native 2>/dev/null", returnOutput
=True, cwd
="/tmp")
674 for line
in out
.splitlines():
675 m
= re
.match(r
"^creating ([a-z0-9]+\.tar\.gz)", line
)
677 self
._environ
["ICECC_VERSION"] = "/tmp/%s" % m
.group(1)
679 def create_buildscript(self
, stage
):
680 file = "/tmp/build_%s" % util
.random_string()
682 # Get buildscript from the package.
683 script
= self
.pkg
.get_buildscript(stage
)
685 # Write script to an empty file.
687 f
.write("#!/bin/sh\n\n")
690 f
.write("\n%s\n" % script
)
698 # Create buildroot and remove all content if it was existant.
699 util
.rm(self
.buildroot
)
700 os
.makedirs(self
.buildroot
)
702 # Build icecream toolchain if icecream is installed.
703 self
.create_icecream_toolchain()
705 for stage
in ("prepare", "build", "test", "install"):
706 self
.build_stage(stage
)
708 # Run post-build stuff.
709 self
.post_compress_man_pages()
710 self
.post_remove_static_libs()
712 # Package the result.
713 # Make all these little package from the build environment.
714 log
.info(_("Creating packages:"))
716 for pkg
in reversed(self
.pkg
.packages
):
717 packager
= packages
.packager
.BinaryPackager(self
.pakfire
, pkg
,
718 self
, self
.buildroot
)
719 pkg
= packager
.run(self
.resultdir
)
723 for pkg
in sorted(pkgs
):
724 for line
in pkg
.dump(long=True).splitlines():
729 def build_stage(self
, stage
):
730 # Get the buildscript for this stage.
731 buildscript
= self
.create_buildscript(stage
)
733 # Execute the buildscript of this stage.
734 log
.info(_("Running stage %s:") % stage
)
737 self
.do(buildscript
, shell
=False)
740 # Remove the buildscript.
741 if os
.path
.exists(buildscript
):
742 os
.unlink(buildscript
)
744 def post_remove_static_libs(self
):
745 keep_libs
= self
.pkg
.lexer
.build
.get_var("keep_libraries")
746 keep_libs
= keep_libs
.split()
749 self
.do("%s/remove-static-libs %s %s" % \
750 (SCRIPT_DIR
, self
.buildroot
, " ".join(keep_libs
)))
752 log
.warning(_("Could not remove static libraries: %s") % e
)
754 def post_compress_man_pages(self
):
756 self
.do("%s/compress-man-pages %s" % (SCRIPT_DIR
, self
.buildroot
))
758 log
.warning(_("Compressing man pages did not complete successfully."))
761 if os
.path
.exists(self
.buildroot
):
762 util
.rm(self
.buildroot
)