]>
git.ipfire.org Git - pakfire.git/blob - pakfire/builder.py
2 ###############################################################################
4 # Pakfire - The IPFire package management system #
5 # Copyright (C) 2011 Pakfire development team #
7 # This program is free software: you can redistribute it and/or modify #
8 # it under the terms of the GNU General Public License as published by #
9 # the Free Software Foundation, either version 3 of the License, or #
10 # (at your option) any later version. #
12 # This program is distributed in the hope that it will be useful, #
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
15 # GNU General Public License for more details. #
17 # You should have received a copy of the GNU General Public License #
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. #
20 ###############################################################################
38 import packages
.packager
42 from constants
import *
44 from errors
import BuildError
, BuildRootLocked
, Error
47 BUILD_LOG_HEADER
= """
49 | _ \ __ _| | __/ _(_)_ __ ___ | |__ _ _(_) | __| | ___ _ __
50 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
51 | __/ (_| | <| _| | | | __/ | |_) | |_| | | | (_| | __/ |
52 |_| \__,_|_|\_\_| |_|_| \___| |_.__/ \__,_|_|_|\__,_|\___|_|
60 class BuildEnviron(object):
61 # The version of the kernel this machine is running.
62 kernel_version
= os
.uname()[2]
64 def __init__(self
, pkg
=None, distro_config
=None, build_id
=None, logfile
=None,
65 builder_mode
="release", **pakfire_args
):
67 assert builder_mode
in ("development", "release",)
68 self
.mode
= builder_mode
70 # Disable the build repository in release mode.
71 if self
.mode
== "release":
72 if pakfire_args
.has_key("disable_repos") and pakfire_args
["disable_repos"]:
73 pakfire_args
["disable_repos"] += ["build",]
75 pakfire_args
["disable_repos"] = ["build",]
77 # Save the build id and generate one if no build id was provided.
79 build_id
= "%s" % uuid
.uuid4()
81 self
.build_id
= build_id
85 self
.log
= logging
.getLogger(self
.build_id
)
86 # Propage everything to the root logger that we will see something
88 self
.log
.propagate
= 1
89 self
.log
.setLevel(logging
.INFO
)
91 # Add the given logfile to the logger.
92 h
= logging
.FileHandler(logfile
)
93 self
.log
.addHandler(h
)
95 # Format the log output for the file.
96 f
= logger
.BuildFormatter()
99 # If no logile was given, we use the root logger.
100 self
.log
= logging
.getLogger()
102 # Log information about pakfire and some more information, when we
103 # are running in release mode.
104 if self
.mode
== "release":
106 "host" : socket
.gethostname(),
107 "time" : time
.strftime("%a, %d %b %Y %H:%M:%S +0000", time
.gmtime()),
108 "version" : "Pakfire %s" % PAKFIRE_VERSION
,
111 for line
in BUILD_LOG_HEADER
.splitlines():
112 self
.log
.info(line
% logdata
)
114 # Create pakfire instance.
115 if pakfire_args
.has_key("mode"):
116 del pakfire_args
["mode"]
117 self
.pakfire
= base
.Pakfire(mode
="builder", distro_config
=distro_config
, **pakfire_args
)
118 self
.distro
= self
.pakfire
.distro
119 self
.path
= self
.pakfire
.path
121 # Log the package information.
122 self
.pkg
= packages
.Makefile(self
.pakfire
, pkg
)
123 self
.log
.info(_("Package information:"))
124 for line
in self
.pkg
.dump(long=True).splitlines():
125 self
.log
.info(" %s" % line
)
128 # XXX need to make this configureable
130 "enable_loop_devices" : True,
131 "enable_ccache" : True,
132 "enable_icecream" : False,
134 #self.settings.update(settings)
136 self
.buildroot
= "/buildroot"
142 # Save the build time.
143 self
.build_time
= int(time
.time())
148 Inherit architecture from distribution configuration.
150 return self
.distro
.arch
155 "build_date" : time
.strftime("%a, %d %b %Y %H:%M:%S +0000", time
.gmtime(self
.build_time
)),
156 "build_host" : socket
.gethostname(),
157 "build_id" : self
.build_id
,
158 "build_time" : self
.build_time
,
162 filename
= os
.path
.join(self
.path
, ".lock")
165 self
._lock
= open(filename
, "a+")
170 fcntl
.lockf(self
._lock
.fileno(), fcntl
.LOCK_EX | fcntl
.LOCK_NB
)
172 raise BuildRootLocked
, "Buildroot is locked"
181 def copyin(self
, file_out
, file_in
):
182 if file_in
.startswith("/"):
183 file_in
= file_in
[1:]
185 file_in
= self
.chrootPath(file_in
)
187 #if not os.path.exists(file_out):
190 dir_in
= os
.path
.dirname(file_in
)
191 if not os
.path
.exists(dir_in
):
194 logging
.debug("%s --> %s" % (file_out
, file_in
))
196 shutil
.copy2(file_out
, file_in
)
198 def copyout(self
, file_in
, file_out
):
199 if file_in
.startswith("/"):
200 file_in
= file_in
[1:]
202 file_in
= self
.chrootPath(file_in
)
204 #if not os.path.exists(file_in):
207 dir_out
= os
.path
.dirname(file_out
)
208 if not os
.path
.exists(dir_out
):
211 logging
.debug("%s --> %s" % (file_in
, file_out
))
213 shutil
.copy2(file_in
, file_out
)
215 def copy_result(self
, resultdir
):
216 dir_in
= self
.chrootPath("result")
218 for dir, subdirs
, files
in os
.walk(dir_in
):
219 basename
= os
.path
.basename(dir)
220 dir = dir[len(self
.chrootPath()):]
222 file_in
= os
.path
.join(dir, file)
224 file_out
= os
.path
.join(
230 self
.copyout(file_in
, file_out
)
232 def extract(self
, requires
=None, build_deps
=True):
234 Gets a dependency set and extracts all packages
240 # Add neccessary build dependencies.
241 requires
+= BUILD_PACKAGES
243 # If we have ccache enabled, we need to extract it
244 # to the build chroot.
245 if self
.settings
.get("enable_ccache"):
246 requires
.append("ccache")
248 # If we have icecream enabled, we need to extract it
249 # to the build chroot.
250 if self
.settings
.get("enable_icecream"):
251 requires
.append("icecream")
253 # Get build dependencies from source package.
254 for req
in self
.pkg
.requires
:
257 # Install all packages.
258 self
.install(requires
)
260 # Copy the makefile and load source tarballs.
261 self
.pkg
.extract(_("Extracting"),
262 prefix
=os
.path
.join(self
.path
, "build"))
264 def install(self
, requires
):
266 Install everything that is required in requires.
268 # If we got nothing to do, we quit immediately.
272 self
.pakfire
.install(requires
, interactive
=False,
273 allow_downgrade
=True, logger
=self
.log
)
275 def install_test(self
):
277 for dir, subdirs
, files
in os
.walk(self
.chrootPath("result")):
279 pkgs
.append(os
.path
.join(dir, file))
281 self
.pakfire
.localinstall(pkgs
, yes
=True)
283 def chrootPath(self
, *args
):
284 # Remove all leading slashes
287 if arg
.startswith("/"):
292 ret
= os
.path
.join(self
.path
, *args
)
293 ret
= ret
.replace("//", "/")
295 assert ret
.startswith(self
.path
)
300 prepared_tag
= ".prepared"
302 if os
.path
.exists(self
.chrootPath(prepared_tag
)):
306 if not os
.path
.exists(self
.path
):
307 os
.makedirs(self
.path
)
309 # Create important directories.
324 # Create cache dir if ccache is enabled.
325 if self
.settings
.get("enable_ccache"):
326 dirs
.append("var/cache/ccache")
328 if not os
.path
.exists(CCACHE_CACHE_DIR
):
329 os
.makedirs(CCACHE_CACHE_DIR
)
332 dir = self
.chrootPath(dir)
333 if not os
.path
.exists(dir):
336 # Create neccessary files like /etc/fstab and /etc/mtab.
344 file = self
.chrootPath(file)
345 dir = os
.path
.dirname(file)
346 if not os
.path
.exists(dir):
354 def _prepare_dev(self
):
355 prevMask
= os
.umask(0000)
358 ("dev/null", stat
.S_IFCHR |
0666, os
.makedev(1, 3)),
359 ("dev/full", stat
.S_IFCHR |
0666, os
.makedev(1, 7)),
360 ("dev/zero", stat
.S_IFCHR |
0666, os
.makedev(1, 5)),
361 ("dev/random", stat
.S_IFCHR |
0666, os
.makedev(1, 8)),
362 ("dev/urandom", stat
.S_IFCHR |
0444, os
.makedev(1, 9)),
363 ("dev/tty", stat
.S_IFCHR |
0666, os
.makedev(5, 0)),
364 ("dev/console", stat
.S_IFCHR |
0600, os
.makedev(5, 1)),
367 # If we need loop devices (which are optional) we create them here.
368 if self
.settings
["enable_loop_devices"]:
369 for i
in range(0, 7):
370 nodes
.append(("dev/loop%d" % i
, stat
.S_IFBLK |
0660, os
.makedev(7, i
)))
372 # Create all the nodes.
374 self
._create
_node
(*node
)
376 os
.symlink("/proc/self/fd/0", self
.chrootPath("dev", "stdin"))
377 os
.symlink("/proc/self/fd/1", self
.chrootPath("dev", "stdout"))
378 os
.symlink("/proc/self/fd/2", self
.chrootPath("dev", "stderr"))
379 os
.symlink("/proc/self/fd", self
.chrootPath("dev", "fd"))
381 # make device node for el4 and el5
382 if self
.kernel_version
< "2.6.19":
383 self
._make
_node
("dev/ptmx", stat
.S_IFCHR |
0666, os
.makedev(5, 2))
385 os
.symlink("/dev/pts/ptmx", self
.chrootPath("dev", "ptmx"))
389 def _prepare_dns(self
):
391 Add DNS resolution facility to chroot environment by copying
392 /etc/resolv.conf and /etc/hosts.
394 for i
in ("/etc/resolv.conf", "/etc/hosts"):
397 def _create_node(self
, filename
, mode
, device
):
398 logging
.debug("Create node: %s (%s)" % (filename
, mode
))
400 filename
= self
.chrootPath(filename
)
402 # Create parent directory if it is missing.
403 dirname
= os
.path
.dirname(filename
)
404 if not os
.path
.exists(dirname
):
407 os
.mknod(filename
, mode
, device
)
410 util
.orphans_kill(self
.path
)
412 logging
.debug("Destroying environment %s" % self
.path
)
414 if os
.path
.exists(self
.path
):
418 logging
.debug("Cleaning environemnt.")
420 # Remove the build directory and buildroot.
421 dirs
= ("build", self
.buildroot
, "result")
424 d
= self
.chrootPath(d
)
425 if not os
.path
.exists(d
):
432 self
.log
.debug("Mounting environment")
433 for cmd
, mountpoint
in self
.mountpoints
:
434 cmd
= "%s %s" % (cmd
, self
.chrootPath(mountpoint
))
435 chroot
.do(cmd
, shell
=True)
437 def _umountall(self
):
438 self
.log
.debug("Umounting environment")
439 for cmd
, mountpoint
in self
.mountpoints
:
440 cmd
= "umount -n %s" % self
.chrootPath(mountpoint
)
441 chroot
.do(cmd
, raiseExc
=0, shell
=True)
444 def mountpoints(self
):
446 ("mount -n -t proc pakfire_chroot_proc", "proc"),
447 ("mount -n -t sysfs pakfire_chroot_sysfs", "sys"),
450 mountopt
= "gid=%d,mode=0620,ptmxmode=0666" % grp
.getgrnam("tty").gr_gid
451 if self
.kernel_version
>= "2.6.29":
452 mountopt
+= ",newinstance"
455 ("mount -n -t devpts -o %s pakfire_chroot_devpts" % mountopt
, "dev/pts"),
456 ("mount -n -t tmpfs pakfire_chroot_shmfs", "dev/shm"),
459 if self
.settings
.get("enable_ccache"):
460 ret
.append(("mount -n --bind %s" % CCACHE_CACHE_DIR
, "var/cache/ccache"))
467 # Add HOME manually, because it is occasionally not set
468 # and some builds get in trouble then.
470 "TERM" : os
.environ
.get("TERM", "dumb"),
473 "BUILDROOT" : self
.buildroot
,
474 "PARALLELISMFLAGS" : "-j%s" % util
.calc_parallelism(),
477 # Inherit environment from distro
478 env
.update(self
.pakfire
.distro
.environ
)
480 # Icecream environment settings
481 if self
.settings
.get("enable_icecream", False):
482 # Set the toolchain path
483 if self
.settings
.get("icecream_toolchain", None):
484 env
["ICECC_VERSION"] = self
.settings
.get("icecream_toolchain")
486 # Set preferred host if configured.
487 if self
.settings
.get("icecream_preferred_host", None):
488 env
["ICECC_PREFERRED_HOST"] = \
489 self
.settings
.get("icecream_preferred_host")
491 # XXX what do we need else?
495 def do(self
, command
, shell
=True, personality
=None, logger
=None, *args
, **kwargs
):
498 # Environment variables
501 if kwargs
.has_key("env"):
502 env
.update(kwargs
.pop("env"))
504 logging
.debug("Environment:")
505 for k
, v
in sorted(env
.items()):
506 logging
.debug(" %s=%s" % (k
, v
))
508 # Update personality it none was set
510 personality
= self
.distro
.personality
512 # Make every shell to a login shell because we set a lot of
513 # environment things there.
515 command
= ["bash", "--login", "-c", command
]
519 if not kwargs
.has_key("chrootPath"):
520 kwargs
["chrootPath"] = self
.chrootPath()
524 personality
=personality
,
537 def build(self
, install_test
=True):
540 pkgfile
= os
.path
.join("/build", os
.path
.basename(self
.pkg
.filename
))
541 resultdir
= self
.chrootPath("/result")
543 # Create the build command, that is executed in the chroot.
544 build_command
= ["/usr/lib/pakfire/builder", "--offline", "build", pkgfile
,
545 "--nodeps", "--resultdir=/result",]
548 self
.do(" ".join(build_command
), logger
=self
.log
)
551 raise BuildError
, _("The build command failed. See logfile for details.")
553 # Perform install test.
557 # Copy the final packages and stuff.
560 def shell(self
, args
=[]):
561 if not util
.cli_is_interactive():
562 logging
.warning("Cannot run shell on non-interactive console.")
565 # Install all packages that are needed to run a shell.
566 self
.install(SHELL_PACKAGES
)
568 # XXX need to set CFLAGS here
569 command
= "/usr/sbin/chroot %s /usr/bin/chroot-shell %s" % \
570 (self
.chrootPath(), " ".join(args
))
572 # Add personality if we require one
573 if self
.pakfire
.distro
.personality
:
574 command
= "%s %s" % (self
.pakfire
.distro
.personality
, command
)
576 for key
, val
in self
.environ
.items():
577 command
= "%s=\"%s\" " % (key
, val
) + command
579 # Empty the environment
580 command
= "env -i - %s" % command
582 logging
.debug("Shell command: %s" % command
)
587 shell
= os
.system(command
)
588 return os
.WEXITSTATUS(shell
)
594 class Builder(object):
595 def __init__(self
, pakfire
, filename
, resultdir
, **kwargs
):
596 self
.pakfire
= pakfire
598 self
.filename
= filename
600 self
.resultdir
= resultdir
603 self
.pkg
= packages
.Makefile(self
.pakfire
, self
.filename
)
605 #self.buildroot = "/tmp/pakfire_buildroot/%s" % util.random_string(20)
606 self
.buildroot
= "/buildroot"
609 "BUILDROOT" : self
.buildroot
,
615 return self
.pakfire
.distro
621 # Get all definitions from the package.
622 environ
.update(self
.pkg
.exports
)
624 # Overwrite some definitions by default values.
625 environ
.update(self
._environ
)
629 def do(self
, command
, shell
=True, personality
=None, cwd
=None, *args
, **kwargs
):
630 # Environment variables
631 logging
.debug("Environment:")
632 for k
, v
in sorted(self
.environ
.items()):
633 logging
.debug(" %s=%s" % (k
, v
))
635 # Update personality it none was set
637 personality
= self
.distro
.personality
640 cwd
= "/%s" % LOCAL_TMP_PATH
642 # Make every shell to a login shell because we set a lot of
643 # environment things there.
645 command
= ["bash", "--login", "-c", command
]
649 personality
=personality
,
652 logger
=logging
.getLogger(),
658 def create_icecream_toolchain(self
):
660 out
= self
.do("icecc --build-native 2>/dev/null", returnOutput
=True)
664 for line
in out
.splitlines():
665 m
= re
.match(r
"^creating ([a-z0-9]+\.tar\.gz)", line
)
667 self
._environ
["icecream_toolchain"] = "/%s" % m
.group(1)
669 def create_buildscript(self
, stage
):
670 file = "/tmp/build_%s" % util
.random_string()
672 # Get buildscript from the package.
673 script
= self
.pkg
.get_buildscript(stage
)
675 # Write script to an empty file.
677 f
.write("#!/bin/sh\n\n")
680 f
.write("\n%s\n" % script
)
689 if not os
.path
.exists(self
.buildroot
):
690 os
.makedirs(self
.buildroot
)
692 # Build icecream toolchain if icecream is installed.
693 self
.create_icecream_toolchain()
695 for stage
in ("prepare", "build", "test", "install"):
696 self
.build_stage(stage
)
698 # Package the result.
699 # Make all these little package from the build environment.
700 logging
.info(_("Creating packages:"))
702 for pkg
in reversed(self
.pkg
.packages
):
703 packager
= packages
.packager
.BinaryPackager(self
.pakfire
, pkg
,
704 self
, self
.buildroot
)
705 pkg
= packager
.run(self
.resultdir
)
709 for pkg
in sorted(pkgs
):
710 for line
in pkg
.dump(long=True).splitlines():
715 def build_stage(self
, stage
):
716 # Get the buildscript for this stage.
717 buildscript
= self
.create_buildscript(stage
)
719 # Execute the buildscript of this stage.
720 logging
.info(_("Running stage %s:") % stage
)
723 self
.do(buildscript
, shell
=False)
726 # Remove the buildscript.
727 if os
.path
.exists(buildscript
):
728 os
.unlink(buildscript
)
731 if os
.path
.exists(self
.buildroot
):
732 util
.rm(self
.buildroot
)