]>
git.ipfire.org Git - pakfire.git/blob - python/pakfire/builder.py
763d771ca626454ff331121040ced0c5d2c0dbfd
2 ###############################################################################
4 # Pakfire - The IPFire package management system #
5 # Copyright (C) 2011 Pakfire development team #
7 # This program is free software: you can redistribute it and/or modify #
8 # it under the terms of the GNU General Public License as published by #
9 # the Free Software Foundation, either version 3 of the License, or #
10 # (at your option) any later version. #
12 # This program is distributed in the hope that it will be useful, #
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
15 # GNU General Public License for more details. #
17 # You should have received a copy of the GNU General Public License #
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. #
20 ###############################################################################
37 import packages
.packager
41 from constants
import *
43 from errors
import BuildError
, BuildRootLocked
, Error
46 BUILD_LOG_HEADER
= """
48 | _ \ __ _| | __/ _(_)_ __ ___ | |__ _ _(_) | __| | ___ _ __
49 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
50 | __/ (_| | <| _| | | | __/ | |_) | |_| | | | (_| | __/ |
51 |_| \__,_|_|\_\_| |_|_| \___| |_.__/ \__,_|_|_|\__,_|\___|_|
59 class BuildEnviron(object):
60 # The version of the kernel this machine is running.
61 kernel_version
= os
.uname()[2]
63 def __init__(self
, pkg
=None, distro_config
=None, build_id
=None, logfile
=None,
64 builder_mode
="release", **pakfire_args
):
66 assert builder_mode
in ("development", "release",)
67 self
.mode
= builder_mode
69 # Disable the build repository in release mode.
70 if self
.mode
== "release":
71 if pakfire_args
.has_key("disable_repos") and pakfire_args
["disable_repos"]:
72 pakfire_args
["disable_repos"] += ["build",]
74 pakfire_args
["disable_repos"] = ["build",]
76 # Save the build id and generate one if no build id was provided.
78 build_id
= "%s" % uuid
.uuid4()
80 self
.build_id
= build_id
84 self
.log
= logging
.getLogger(self
.build_id
)
85 # Propage everything to the root logger that we will see something
87 self
.log
.propagate
= 1
88 self
.log
.setLevel(logging
.INFO
)
90 # Add the given logfile to the logger.
91 h
= logging
.FileHandler(logfile
)
92 self
.log
.addHandler(h
)
94 # Format the log output for the file.
95 f
= logger
.BuildFormatter()
98 # If no logile was given, we use the root logger.
99 self
.log
= logging
.getLogger()
101 # Log information about pakfire and some more information, when we
102 # are running in release mode.
103 if self
.mode
== "release":
105 "host" : socket
.gethostname(),
106 "time" : time
.strftime("%a, %d %b %Y %H:%M:%S +0000", time
.gmtime()),
107 "version" : "Pakfire %s" % PAKFIRE_VERSION
,
110 for line
in BUILD_LOG_HEADER
.splitlines():
111 self
.log
.info(line
% logdata
)
113 # Create pakfire instance.
114 if pakfire_args
.has_key("mode"):
115 del pakfire_args
["mode"]
116 self
.pakfire
= base
.Pakfire(mode
="builder", distro_config
=distro_config
, **pakfire_args
)
117 self
.distro
= self
.pakfire
.distro
118 self
.path
= self
.pakfire
.path
120 # Log the package information.
121 self
.pkg
= packages
.Makefile(self
.pakfire
, pkg
)
122 self
.log
.info(_("Package information:"))
123 for line
in self
.pkg
.dump(long=True).splitlines():
124 self
.log
.info(" %s" % line
)
127 # Download all package files.
130 # XXX need to make this configureable
132 "enable_loop_devices" : True,
133 "enable_ccache" : True,
134 "enable_icecream" : False,
136 #self.settings.update(settings)
142 # Save the build time.
143 self
.build_time
= int(time
.time())
146 # Mount the directories.
152 # Setup domain name resolution in chroot.
155 # Extract all needed packages.
159 # Kill all still running processes.
160 util
.orphans_kill(self
.path
)
162 # Close pakfire instance.
165 # Umount the build environment.
174 Inherit architecture from distribution configuration.
176 return self
.distro
.arch
181 "build_date" : time
.strftime("%a, %d %b %Y %H:%M:%S +0000", time
.gmtime(self
.build_time
)),
182 "build_host" : socket
.gethostname(),
183 "build_id" : self
.build_id
,
184 "build_time" : self
.build_time
,
188 filename
= os
.path
.join(self
.path
, ".lock")
191 self
._lock
= open(filename
, "a+")
196 fcntl
.lockf(self
._lock
.fileno(), fcntl
.LOCK_EX | fcntl
.LOCK_NB
)
198 raise BuildRootLocked
, "Buildroot is locked"
207 def copyin(self
, file_out
, file_in
):
208 if file_in
.startswith("/"):
209 file_in
= file_in
[1:]
211 file_in
= self
.chrootPath(file_in
)
213 #if not os.path.exists(file_out):
216 dir_in
= os
.path
.dirname(file_in
)
217 if not os
.path
.exists(dir_in
):
220 logging
.debug("%s --> %s" % (file_out
, file_in
))
222 shutil
.copy2(file_out
, file_in
)
224 def copyout(self
, file_in
, file_out
):
225 if file_in
.startswith("/"):
226 file_in
= file_in
[1:]
228 file_in
= self
.chrootPath(file_in
)
230 #if not os.path.exists(file_in):
233 dir_out
= os
.path
.dirname(file_out
)
234 if not os
.path
.exists(dir_out
):
237 logging
.debug("%s --> %s" % (file_in
, file_out
))
239 shutil
.copy2(file_in
, file_out
)
241 def copy_result(self
, resultdir
):
242 dir_in
= self
.chrootPath("result")
244 for dir, subdirs
, files
in os
.walk(dir_in
):
245 basename
= os
.path
.basename(dir)
246 dir = dir[len(self
.chrootPath()):]
248 file_in
= os
.path
.join(dir, file)
250 file_out
= os
.path
.join(
256 self
.copyout(file_in
, file_out
)
258 def extract(self
, requires
=None, build_deps
=True):
260 Gets a dependency set and extracts all packages
266 # Add neccessary build dependencies.
267 requires
+= BUILD_PACKAGES
269 # If we have ccache enabled, we need to extract it
270 # to the build chroot.
271 if self
.settings
.get("enable_ccache"):
272 requires
.append("ccache")
274 # If we have icecream enabled, we need to extract it
275 # to the build chroot.
276 if self
.settings
.get("enable_icecream"):
277 requires
.append("icecream")
279 # Get build dependencies from source package.
280 for req
in self
.pkg
.requires
:
283 # Install all packages.
284 self
.install(requires
)
286 # Copy the makefile and load source tarballs.
287 self
.pkg
.extract(_("Extracting"),
288 prefix
=os
.path
.join(self
.path
, "build"))
290 def install(self
, requires
):
292 Install everything that is required in requires.
294 # If we got nothing to do, we quit immediately.
298 self
.pakfire
.install(requires
, interactive
=False,
299 allow_downgrade
=True, logger
=self
.log
)
301 def install_test(self
):
303 for dir, subdirs
, files
in os
.walk(self
.chrootPath("result")):
305 pkgs
.append(os
.path
.join(dir, file))
307 self
.pakfire
.localinstall(pkgs
, yes
=True, allow_uninstall
=True)
309 def chrootPath(self
, *args
):
310 # Remove all leading slashes
313 if arg
.startswith("/"):
318 ret
= os
.path
.join(self
.path
, *args
)
319 ret
= ret
.replace("//", "/")
321 assert ret
.startswith(self
.path
)
325 def populate_dev(self
):
339 # If we need loop devices (which are optional) we create them here.
340 if self
.settings
["enable_loop_devices"]:
341 for i
in range(0, 7):
342 nodes
.append("/dev/loop%d" % i
)
345 # Stat the original node of the host system and copy it to
347 node_stat
= os
.stat(node
)
349 self
._create
_node
(node
, node_stat
.st_mode
, node_stat
.st_rdev
)
351 os
.symlink("/proc/self/fd/0", self
.chrootPath("dev", "stdin"))
352 os
.symlink("/proc/self/fd/1", self
.chrootPath("dev", "stdout"))
353 os
.symlink("/proc/self/fd/2", self
.chrootPath("dev", "stderr"))
354 os
.symlink("/proc/self/fd", self
.chrootPath("dev", "fd"))
358 Add DNS resolution facility to chroot environment by copying
359 /etc/resolv.conf and /etc/hosts.
361 for i
in ("/etc/resolv.conf", "/etc/hosts"):
364 def _create_node(self
, filename
, mode
, device
):
365 logging
.debug("Create node: %s (%s)" % (filename
, mode
))
367 filename
= self
.chrootPath(filename
)
369 # Create parent directory if it is missing.
370 dirname
= os
.path
.dirname(filename
)
371 if not os
.path
.exists(dirname
):
374 os
.mknod(filename
, mode
, device
)
377 logging
.debug("Destroying environment %s" % self
.path
)
379 if os
.path
.exists(self
.path
):
383 logging
.debug("Cleaning environemnt.")
385 # Remove the build directory and buildroot.
386 dirs
= ("build", "result")
389 d
= self
.chrootPath(d
)
390 if not os
.path
.exists(d
):
397 self
.log
.debug("Mounting environment")
398 for src
, dest
, fs
, options
in self
.mountpoints
:
399 mountpoint
= self
.chrootPath(dest
)
401 options
= "-o %s" % options
403 # Eventually create mountpoint directory
404 if not os
.path
.exists(mountpoint
):
405 os
.makedirs(mountpoint
)
407 cmd
= "mount -n -t %s %s %s %s" % \
408 (fs
, options
, src
, mountpoint
)
409 chroot
.do(cmd
, shell
=True)
411 def _umountall(self
):
412 self
.log
.debug("Umounting environment")
415 for src
, dest
, fs
, options
in reversed(self
.mountpoints
):
416 if not dest
in mountpoints
:
417 mountpoints
.append(dest
)
419 for dest
in mountpoints
:
420 mountpoint
= self
.chrootPath(dest
)
422 chroot
.do("umount -n %s" % mountpoint
, raiseExc
=0, shell
=True)
425 def mountpoints(self
):
428 # Make root as a tmpfs.
430 # ("pakfire_root", "/", "tmpfs", "defaults"),
434 # src, dest, fs, options
435 ("pakfire_proc", "/proc", "proc", "nosuid,noexec,nodev"),
436 ("/proc/sys", "/proc/sys", "bind", "bind"),
437 ("/proc/sys", "/proc/sys", "bind", "bind,ro,remount"),
438 ("/sys", "/sys", "bind", "bind"),
439 ("/sys", "/sys", "bind", "bind,ro,remount"),
440 ("pakfire_tmpfs", "/dev", "tmpfs", "mode=755,nosuid"),
441 ("/dev/pts", "/dev/pts", "bind", "bind"),
442 ("pakfire_tmpfs", "/run", "tmpfs", "mode=755,nosuid,nodev"),
445 # If selinux is enabled.
446 if os
.path
.exists("/sys/fs/selinux"):
448 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
449 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
452 # If ccache support is requested, we bind mount the cache.
453 if self
.settings
.get("enable_ccache"):
454 # Create ccache cache directory if it does not exist.
455 if not os
.path
.exists(CCACHE_CACHE_DIR
):
456 os
.makedirs(CCACHE_CACHE_DIR
)
459 (CCACHE_CACHE_DIR
, "/var/cache/ccache", "bind", "bind"),
467 # Add HOME manually, because it is occasionally not set
468 # and some builds get in trouble then.
470 "TERM" : os
.environ
.get("TERM", "dumb"),
473 # Set the container that we can detect, if we are inside a
475 "container" : "pakfire-builder",
478 # Inherit environment from distro
479 env
.update(self
.pakfire
.distro
.environ
)
481 # Icecream environment settings
482 if self
.settings
.get("enable_icecream", False):
483 # Set the toolchain path
484 if self
.settings
.get("icecream_toolchain", None):
485 env
["ICECC_VERSION"] = self
.settings
.get("icecream_toolchain")
487 # Set preferred host if configured.
488 if self
.settings
.get("icecream_preferred_host", None):
489 env
["ICECC_PREFERRED_HOST"] = \
490 self
.settings
.get("icecream_preferred_host")
492 # XXX what do we need else?
496 def do(self
, command
, shell
=True, personality
=None, logger
=None, *args
, **kwargs
):
499 # Environment variables
502 if kwargs
.has_key("env"):
503 env
.update(kwargs
.pop("env"))
505 logging
.debug("Environment:")
506 for k
, v
in sorted(env
.items()):
507 logging
.debug(" %s=%s" % (k
, v
))
509 # Update personality it none was set
511 personality
= self
.distro
.personality
513 # Make every shell to a login shell because we set a lot of
514 # environment things there.
516 command
= ["bash", "--login", "-c", command
]
518 if not kwargs
.has_key("chrootPath"):
519 kwargs
["chrootPath"] = self
.chrootPath()
523 personality
=personality
,
533 def build(self
, install_test
=True):
536 pkgfile
= os
.path
.join("/build", os
.path
.basename(self
.pkg
.filename
))
537 resultdir
= self
.chrootPath("/result")
539 # Create the build command, that is executed in the chroot.
540 build_command
= ["/usr/lib/pakfire/builder", "--offline", "build", pkgfile
,
541 "--nodeps", "--resultdir=/result",]
544 self
.do(" ".join(build_command
), logger
=self
.log
)
547 raise BuildError
, _("The build command failed. See logfile for details.")
549 # Perform install test.
553 # Copy the final packages and stuff.
556 def shell(self
, args
=[]):
557 if not util
.cli_is_interactive():
558 logging
.warning("Cannot run shell on non-interactive console.")
561 # Install all packages that are needed to run a shell.
562 self
.install(SHELL_PACKAGES
)
564 # XXX need to set CFLAGS here
565 command
= "/usr/sbin/chroot %s %s %s" % \
566 (self
.chrootPath(), SHELL_SCRIPT
, " ".join(args
))
568 # Add personality if we require one
569 if self
.pakfire
.distro
.personality
:
570 command
= "%s %s" % (self
.pakfire
.distro
.personality
, command
)
572 for key
, val
in self
.environ
.items():
573 command
= "%s=\"%s\" " % (key
, val
) + command
575 # Empty the environment
576 command
= "env -i - %s" % command
578 logging
.debug("Shell command: %s" % command
)
580 shell
= os
.system(command
)
581 return os
.WEXITSTATUS(shell
)
584 class Builder(object):
585 def __init__(self
, pakfire
, filename
, resultdir
, **kwargs
):
586 self
.pakfire
= pakfire
588 self
.filename
= filename
590 self
.resultdir
= resultdir
593 self
.pkg
= packages
.Makefile(self
.pakfire
, self
.filename
)
601 return self
.pkg
.buildroot
605 return self
.pakfire
.distro
611 # Get all definitions from the package.
612 environ
.update(self
.pkg
.exports
)
614 # Overwrite some definitions by default values.
615 environ
.update(self
._environ
)
619 def do(self
, command
, shell
=True, personality
=None, cwd
=None, *args
, **kwargs
):
620 # Environment variables
621 logging
.debug("Environment:")
622 for k
, v
in sorted(self
.environ
.items()):
623 logging
.debug(" %s=%s" % (k
, v
))
625 # Update personality it none was set
627 personality
= self
.distro
.personality
630 cwd
= "/%s" % LOCAL_TMP_PATH
632 # Make every shell to a login shell because we set a lot of
633 # environment things there.
635 command
= ["bash", "--login", "-c", command
]
639 personality
=personality
,
642 logger
=logging
.getLogger(),
648 def create_icecream_toolchain(self
):
650 out
= self
.do("icecc --build-native 2>/dev/null", returnOutput
=True)
654 for line
in out
.splitlines():
655 m
= re
.match(r
"^creating ([a-z0-9]+\.tar\.gz)", line
)
657 self
._environ
["icecream_toolchain"] = "/%s" % m
.group(1)
659 def create_buildscript(self
, stage
):
660 file = "/tmp/build_%s" % util
.random_string()
662 # Get buildscript from the package.
663 script
= self
.pkg
.get_buildscript(stage
)
665 # Write script to an empty file.
667 f
.write("#!/bin/sh\n\n")
670 f
.write("\n%s\n" % script
)
679 if not os
.path
.exists(self
.buildroot
):
680 os
.makedirs(self
.buildroot
)
682 # Build icecream toolchain if icecream is installed.
683 self
.create_icecream_toolchain()
685 for stage
in ("prepare", "build", "test", "install"):
686 self
.build_stage(stage
)
688 # Package the result.
689 # Make all these little package from the build environment.
690 logging
.info(_("Creating packages:"))
692 for pkg
in reversed(self
.pkg
.packages
):
693 packager
= packages
.packager
.BinaryPackager(self
.pakfire
, pkg
,
694 self
, self
.buildroot
)
695 pkg
= packager
.run(self
.resultdir
)
699 for pkg
in sorted(pkgs
):
700 for line
in pkg
.dump(long=True).splitlines():
705 def build_stage(self
, stage
):
706 # Get the buildscript for this stage.
707 buildscript
= self
.create_buildscript(stage
)
709 # Execute the buildscript of this stage.
710 logging
.info(_("Running stage %s:") % stage
)
713 self
.do(buildscript
, shell
=False)
716 # Remove the buildscript.
717 if os
.path
.exists(buildscript
):
718 os
.unlink(buildscript
)
721 if os
.path
.exists(self
.buildroot
):
722 util
.rm(self
.buildroot
)