]> git.ipfire.org Git - pakfire.git/blob - python/pakfire/builder.py
Fix generation of package file path name.
[pakfire.git] / python / pakfire / builder.py
1 #!/usr/bin/python
2 ###############################################################################
3 # #
4 # Pakfire - The IPFire package management system #
5 # Copyright (C) 2011 Pakfire development team #
6 # #
7 # This program is free software: you can redistribute it and/or modify #
8 # it under the terms of the GNU General Public License as published by #
9 # the Free Software Foundation, either version 3 of the License, or #
10 # (at your option) any later version. #
11 # #
12 # This program is distributed in the hope that it will be useful, #
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
15 # GNU General Public License for more details. #
16 # #
17 # You should have received a copy of the GNU General Public License #
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. #
19 # #
20 ###############################################################################
21
22 import fcntl
23 import grp
24 import math
25 import os
26 import re
27 import shutil
28 import socket
29 import time
30 import uuid
31
32 import base
33 import chroot
34 import logger
35 import packages
36 import packages.packager
37 import repository
38 import util
39
40 import logging
41 log = logging.getLogger("pakfire")
42
43 from constants import *
44 from i18n import _
45 from errors import BuildError, BuildRootLocked, Error
46
47
48 BUILD_LOG_HEADER = """
49 ____ _ __ _ _ _ _ _
50 | _ \ __ _| | __/ _(_)_ __ ___ | |__ _ _(_) | __| | ___ _ __
51 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
52 | __/ (_| | <| _| | | | __/ | |_) | |_| | | | (_| | __/ |
53 |_| \__,_|_|\_\_| |_|_| \___| |_.__/ \__,_|_|_|\__,_|\___|_|
54
55 Time : %(time)s
56 Host : %(host)s
57 Version : %(version)s
58
59 """
60
61 class BuildEnviron(object):
62 # The version of the kernel this machine is running.
63 kernel_version = os.uname()[2]
64
65 def __init__(self, filename, distro_config=None, build_id=None, logfile=None,
66 builder_mode="release", **pakfire_args):
67 # Set mode.
68 assert builder_mode in ("development", "release",)
69 self.mode = builder_mode
70
71 # Disable the build repository in release mode.
72 if self.mode == "release":
73 if pakfire_args.has_key("disable_repos") and pakfire_args["disable_repos"]:
74 pakfire_args["disable_repos"] += ["build",]
75 else:
76 pakfire_args["disable_repos"] = ["build",]
77
78 # Save the build id and generate one if no build id was provided.
79 if not build_id:
80 build_id = "%s" % uuid.uuid4()
81
82 self.build_id = build_id
83
84 # Setup the logging.
85 if logfile:
86 self.log = logging.getLogger(self.build_id)
87 # Propage everything to the root logger that we will see something
88 # on the terminal.
89 self.log.propagate = 1
90 self.log.setLevel(logging.INFO)
91
92 # Add the given logfile to the logger.
93 h = logging.FileHandler(logfile)
94 self.log.addHandler(h)
95
96 # Format the log output for the file.
97 f = logger.BuildFormatter()
98 h.setFormatter(f)
99 else:
100 # If no logile was given, we use the root logger.
101 self.log = logging.getLogger("pakfire")
102
103 # Log information about pakfire and some more information, when we
104 # are running in release mode.
105 if self.mode == "release":
106 logdata = {
107 "host" : socket.gethostname(),
108 "time" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
109 "version" : "Pakfire %s" % PAKFIRE_VERSION,
110 }
111
112 for line in BUILD_LOG_HEADER.splitlines():
113 self.log.info(line % logdata)
114
115 # Create pakfire instance.
116 if pakfire_args.has_key("mode"):
117 del pakfire_args["mode"]
118 self.pakfire = base.Pakfire(mode="builder", distro_config=distro_config, **pakfire_args)
119 self.distro = self.pakfire.distro
120 self.path = self.pakfire.path
121
122 # Where do we put the result?
123 self.resultdir = os.path.join(self.path, "result")
124
125 # Open package.
126 # If we have a plain makefile, we first build a source package and go with that.
127 if filename.endswith(".%s" % MAKEFILE_EXTENSION):
128 pkg = packages.Makefile(self.pakfire, filename)
129 pkg.dist([self.resultdir,])
130
131 filename = os.path.join(self.resultdir, "src", pkg.package_filename)
132 assert os.path.exists(filename), filename
133
134 # Open source package.
135 self.pkg = packages.SourcePackage(self.pakfire, None, filename)
136 assert self.pkg, filename
137
138 # Log the package information.
139 self.log.info(_("Package information:"))
140 for line in self.pkg.dump(long=True).splitlines():
141 self.log.info(" %s" % line)
142 self.log.info("")
143
144 # Path where we extract the package and put all the source files.
145 self.build_dir = os.path.join(self.path, "usr/src", self.pkg.friendly_name)
146
147 # XXX need to make this configureable
148 self.settings = {
149 "enable_loop_devices" : True,
150 "enable_ccache" : True,
151 "enable_icecream" : False,
152 }
153 #self.settings.update(settings)
154
155 # Lock the buildroot
156 self._lock = None
157 self.lock()
158
159 # Save the build time.
160 self.build_time = int(time.time())
161
162 def start(self):
163 # Mount the directories.
164 self._mountall()
165
166 # Populate /dev.
167 self.populate_dev()
168
169 # Setup domain name resolution in chroot.
170 self.setup_dns()
171
172 # Extract all needed packages.
173 self.extract()
174
175 def stop(self):
176 # Kill all still running processes.
177 util.orphans_kill(self.path)
178
179 # Close pakfire instance.
180 del self.pakfire
181
182 # Umount the build environment.
183 self._umountall()
184
185 # Remove all files.
186 self.destroy()
187
188 @property
189 def arch(self):
190 """
191 Inherit architecture from distribution configuration.
192 """
193 return self.distro.arch
194
195 @property
196 def info(self):
197 return {
198 "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
199 "build_host" : socket.gethostname(),
200 "build_id" : self.build_id,
201 "build_time" : self.build_time,
202 }
203
204 def lock(self):
205 filename = os.path.join(self.path, ".lock")
206
207 try:
208 self._lock = open(filename, "a+")
209 except IOError, e:
210 return 0
211
212 try:
213 fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
214 except IOError, e:
215 raise BuildRootLocked, "Buildroot is locked"
216
217 return 1
218
219 def unlock(self):
220 if self._lock:
221 self._lock.close()
222 self._lock = None
223
224 def copyin(self, file_out, file_in):
225 if file_in.startswith("/"):
226 file_in = file_in[1:]
227
228 file_in = self.chrootPath(file_in)
229
230 #if not os.path.exists(file_out):
231 # return
232
233 dir_in = os.path.dirname(file_in)
234 if not os.path.exists(dir_in):
235 os.makedirs(dir_in)
236
237 self.log.debug("%s --> %s" % (file_out, file_in))
238
239 shutil.copy2(file_out, file_in)
240
241 def copyout(self, file_in, file_out):
242 if file_in.startswith("/"):
243 file_in = file_in[1:]
244
245 file_in = self.chrootPath(file_in)
246
247 #if not os.path.exists(file_in):
248 # return
249
250 dir_out = os.path.dirname(file_out)
251 if not os.path.exists(dir_out):
252 os.makedirs(dir_out)
253
254 self.log.debug("%s --> %s" % (file_in, file_out))
255
256 shutil.copy2(file_in, file_out)
257
258 def copy_result(self, resultdir):
259 dir_in = self.chrootPath("result")
260
261 for dir, subdirs, files in os.walk(dir_in):
262 basename = os.path.basename(dir)
263 dir = dir[len(self.chrootPath()):]
264 for file in files:
265 file_in = os.path.join(dir, file)
266
267 file_out = os.path.join(
268 resultdir,
269 basename,
270 file,
271 )
272
273 self.copyout(file_in, file_out)
274
275 def extract(self, requires=None, build_deps=True):
276 """
277 Gets a dependency set and extracts all packages
278 to the environment.
279 """
280 if not requires:
281 requires = []
282
283 # Add neccessary build dependencies.
284 requires += BUILD_PACKAGES
285
286 # If we have ccache enabled, we need to extract it
287 # to the build chroot.
288 if self.settings.get("enable_ccache"):
289 requires.append("ccache")
290
291 # If we have icecream enabled, we need to extract it
292 # to the build chroot.
293 if self.settings.get("enable_icecream"):
294 requires.append("icecream")
295
296 # Get build dependencies from source package.
297 for req in self.pkg.requires:
298 requires.append(req)
299
300 # Install all packages.
301 self.install(requires)
302
303 # Copy the makefile and load source tarballs.
304 self.pkg.extract(_("Extracting"), prefix=self.build_dir)
305
306 def install(self, requires):
307 """
308 Install everything that is required in requires.
309 """
310 # If we got nothing to do, we quit immediately.
311 if not requires:
312 return
313
314 self.pakfire.install(requires, interactive=False,
315 allow_downgrade=True, logger=self.log)
316
317 def install_test(self):
318 pkgs = []
319 for dir, subdirs, files in os.walk(self.chrootPath("result")):
320 for file in files:
321 pkgs.append(os.path.join(dir, file))
322
323 self.pakfire.localinstall(pkgs, yes=True, allow_uninstall=True)
324
325 def chrootPath(self, *args):
326 # Remove all leading slashes
327 _args = []
328 for arg in args:
329 if arg.startswith("/"):
330 arg = arg[1:]
331 _args.append(arg)
332 args = _args
333
334 ret = os.path.join(self.path, *args)
335 ret = ret.replace("//", "/")
336
337 assert ret.startswith(self.path)
338
339 return ret
340
341 def populate_dev(self):
342 nodes = [
343 "/dev/null",
344 "/dev/zero",
345 "/dev/full",
346 "/dev/random",
347 "/dev/urandom",
348 "/dev/tty",
349 "/dev/ptmx",
350 "/dev/kmsg",
351 "/dev/rtc0",
352 "/dev/console",
353 ]
354
355 # If we need loop devices (which are optional) we create them here.
356 if self.settings["enable_loop_devices"]:
357 for i in range(0, 7):
358 nodes.append("/dev/loop%d" % i)
359
360 for node in nodes:
361 # Stat the original node of the host system and copy it to
362 # the build chroot.
363 node_stat = os.stat(node)
364
365 self._create_node(node, node_stat.st_mode, node_stat.st_rdev)
366
367 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
368 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
369 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
370 os.symlink("/proc/self/fd", self.chrootPath("dev", "fd"))
371
372 def setup_dns(self):
373 """
374 Add DNS resolution facility to chroot environment by copying
375 /etc/resolv.conf and /etc/hosts.
376 """
377 for i in ("/etc/resolv.conf", "/etc/hosts"):
378 self.copyin(i, i)
379
380 def _create_node(self, filename, mode, device):
381 self.log.debug("Create node: %s (%s)" % (filename, mode))
382
383 filename = self.chrootPath(filename)
384
385 # Create parent directory if it is missing.
386 dirname = os.path.dirname(filename)
387 if not os.path.exists(dirname):
388 os.makedirs(dirname)
389
390 os.mknod(filename, mode, device)
391
392 def destroy(self):
393 self.log.debug("Destroying environment %s" % self.path)
394
395 if os.path.exists(self.path):
396 util.rm(self.path)
397
398 def cleanup(self):
399 self.log.debug("Cleaning environemnt.")
400
401 # Remove the build directory and buildroot.
402 dirs = (self.build_dir, self.chrootPath("result"),)
403
404 for d in dirs:
405 if not os.path.exists(d):
406 continue
407
408 util.rm(d)
409 os.makedirs(d)
410
411 def _mountall(self):
412 self.log.debug("Mounting environment")
413 for src, dest, fs, options in self.mountpoints:
414 mountpoint = self.chrootPath(dest)
415 if options:
416 options = "-o %s" % options
417
418 # Eventually create mountpoint directory
419 if not os.path.exists(mountpoint):
420 os.makedirs(mountpoint)
421
422 cmd = "mount -n -t %s %s %s %s" % \
423 (fs, options, src, mountpoint)
424 chroot.do(cmd, shell=True)
425
426 def _umountall(self):
427 self.log.debug("Umounting environment")
428
429 mountpoints = []
430 for src, dest, fs, options in reversed(self.mountpoints):
431 if not dest in mountpoints:
432 mountpoints.append(dest)
433
434 for dest in mountpoints:
435 mountpoint = self.chrootPath(dest)
436
437 chroot.do("umount -n %s" % mountpoint, raiseExc=0, shell=True)
438
439 @property
440 def mountpoints(self):
441 mountpoints = []
442
443 # Make root as a tmpfs.
444 #mountpoints += [
445 # ("pakfire_root", "/", "tmpfs", "defaults"),
446 #]
447
448 mountpoints += [
449 # src, dest, fs, options
450 ("pakfire_proc", "/proc", "proc", "nosuid,noexec,nodev"),
451 ("/proc/sys", "/proc/sys", "bind", "bind"),
452 ("/proc/sys", "/proc/sys", "bind", "bind,ro,remount"),
453 ("/sys", "/sys", "bind", "bind"),
454 ("/sys", "/sys", "bind", "bind,ro,remount"),
455 ("pakfire_tmpfs", "/dev", "tmpfs", "mode=755,nosuid"),
456 ("/dev/pts", "/dev/pts", "bind", "bind"),
457 ("pakfire_tmpfs", "/run", "tmpfs", "mode=755,nosuid,nodev"),
458 ]
459
460 # If selinux is enabled.
461 if os.path.exists("/sys/fs/selinux"):
462 mountpoints += [
463 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind"),
464 ("/sys/fs/selinux", "/sys/fs/selinux", "bind", "bind,ro,remount"),
465 ]
466
467 # If ccache support is requested, we bind mount the cache.
468 if self.settings.get("enable_ccache"):
469 # Create ccache cache directory if it does not exist.
470 if not os.path.exists(CCACHE_CACHE_DIR):
471 os.makedirs(CCACHE_CACHE_DIR)
472
473 mountpoints += [
474 (CCACHE_CACHE_DIR, "/var/cache/ccache", "bind", "bind"),
475 ]
476
477 return mountpoints
478
479 @property
480 def environ(self):
481 env = {
482 # Add HOME manually, because it is occasionally not set
483 # and some builds get in trouble then.
484 "HOME" : "/root",
485 "TERM" : os.environ.get("TERM", "dumb"),
486 "PS1" : "\u:\w\$ ",
487
488 # Set the container that we can detect, if we are inside a
489 # chroot.
490 "container" : "pakfire-builder",
491 }
492
493 # Inherit environment from distro
494 env.update(self.pakfire.distro.environ)
495
496 # Icecream environment settings
497 if self.settings.get("enable_icecream", False):
498 # Set the toolchain path
499 if self.settings.get("icecream_toolchain", None):
500 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
501
502 # Set preferred host if configured.
503 if self.settings.get("icecream_preferred_host", None):
504 env["ICECC_PREFERRED_HOST"] = \
505 self.settings.get("icecream_preferred_host")
506
507 # XXX what do we need else?
508
509 return env
510
511 def do(self, command, shell=True, personality=None, logger=None, *args, **kwargs):
512 ret = None
513
514 # Environment variables
515 env = self.environ
516
517 if kwargs.has_key("env"):
518 env.update(kwargs.pop("env"))
519
520 self.log.debug("Environment:")
521 for k, v in sorted(env.items()):
522 self.log.debug(" %s=%s" % (k, v))
523
524 # Update personality it none was set
525 if not personality:
526 personality = self.distro.personality
527
528 # Make every shell to a login shell because we set a lot of
529 # environment things there.
530 if shell:
531 command = ["bash", "--login", "-c", command]
532
533 if not kwargs.has_key("chrootPath"):
534 kwargs["chrootPath"] = self.chrootPath()
535
536 ret = chroot.do(
537 command,
538 personality=personality,
539 shell=False,
540 env=env,
541 logger=logger,
542 *args,
543 **kwargs
544 )
545
546 return ret
547
548 def build(self, install_test=True):
549 assert self.pkg
550
551 # Search for the package file in build_dir and raise BuildError if it is not present.
552 pkgfile = os.path.join(self.build_dir, "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION))
553 if not os.path.exists(pkgfile):
554 raise BuildError, _("Could not find makefile in build root: %s") % pkgfile
555 pkgfile = "/%s" % os.path.relpath(pkgfile, self.chrootPath())
556
557 resultdir = self.chrootPath("/result")
558
559 # Create the build command, that is executed in the chroot.
560 build_command = ["/usr/lib/pakfire/builder", "--offline", "build", pkgfile,
561 "--nodeps", "--resultdir=/result",]
562
563 try:
564 self.do(" ".join(build_command), logger=self.log)
565
566 except Error:
567 raise BuildError, _("The build command failed. See logfile for details.")
568
569 # Perform install test.
570 if install_test:
571 self.install_test()
572
573 # Copy the final packages and stuff.
574 # XXX TODO resultdir
575
576 def shell(self, args=[]):
577 if not util.cli_is_interactive():
578 self.log.warning("Cannot run shell on non-interactive console.")
579 return
580
581 # Install all packages that are needed to run a shell.
582 self.install(SHELL_PACKAGES)
583
584 # XXX need to set CFLAGS here
585 command = "/usr/sbin/chroot %s %s %s" % \
586 (self.chrootPath(), SHELL_SCRIPT, " ".join(args))
587
588 # Add personality if we require one
589 if self.pakfire.distro.personality:
590 command = "%s %s" % (self.pakfire.distro.personality, command)
591
592 for key, val in self.environ.items():
593 command = "%s=\"%s\" " % (key, val) + command
594
595 # Empty the environment
596 command = "env -i - %s" % command
597
598 self.log.debug("Shell command: %s" % command)
599
600 shell = os.system(command)
601 return os.WEXITSTATUS(shell)
602
603
604 class Builder(object):
605 def __init__(self, pakfire, filename, resultdir, **kwargs):
606 self.pakfire = pakfire
607
608 self.filename = filename
609
610 self.resultdir = resultdir
611
612 # Open package file.
613 self.pkg = packages.Makefile(self.pakfire, self.filename)
614
615 self._environ = {
616 "LANG" : "C",
617 }
618
619 @property
620 def buildroot(self):
621 return self.pkg.buildroot
622
623 @property
624 def distro(self):
625 return self.pakfire.distro
626
627 @property
628 def environ(self):
629 environ = os.environ
630
631 # Get all definitions from the package.
632 environ.update(self.pkg.exports)
633
634 # Overwrite some definitions by default values.
635 environ.update(self._environ)
636
637 return environ
638
639 def do(self, command, shell=True, personality=None, cwd=None, *args, **kwargs):
640 # Environment variables
641 log.debug("Environment:")
642 for k, v in sorted(self.environ.items()):
643 log.debug(" %s=%s" % (k, v))
644
645 # Update personality it none was set
646 if not personality:
647 personality = self.distro.personality
648
649 if not cwd:
650 cwd = "/%s" % LOCAL_TMP_PATH
651
652 # Make every shell to a login shell because we set a lot of
653 # environment things there.
654 if shell:
655 command = ["bash", "--login", "-c", command]
656
657 return chroot.do(
658 command,
659 personality=personality,
660 shell=False,
661 env=self.environ,
662 logger=logging.getLogger("pakfire"),
663 cwd=cwd,
664 *args,
665 **kwargs
666 )
667
668 def create_icecream_toolchain(self):
669 try:
670 out = self.do("icecc --build-native 2>/dev/null", returnOutput=True, cwd="/tmp")
671 except Error:
672 return
673
674 for line in out.splitlines():
675 m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
676 if m:
677 self._environ["ICECC_VERSION"] = "/tmp/%s" % m.group(1)
678
679 def create_buildscript(self, stage):
680 file = "/tmp/build_%s" % util.random_string()
681
682 # Get buildscript from the package.
683 script = self.pkg.get_buildscript(stage)
684
685 # Write script to an empty file.
686 f = open(file, "w")
687 f.write("#!/bin/sh\n\n")
688 f.write("set -e\n")
689 f.write("set -x\n")
690 f.write("\n%s\n" % script)
691 f.write("exit 0\n")
692 f.close()
693 os.chmod(file, 700)
694
695 return file
696
697 def build(self):
698 # Create buildroot and remove all content if it was existant.
699 util.rm(self.buildroot)
700 os.makedirs(self.buildroot)
701
702 # Build icecream toolchain if icecream is installed.
703 self.create_icecream_toolchain()
704
705 for stage in ("prepare", "build", "test", "install"):
706 self.build_stage(stage)
707
708 # Run post-build stuff.
709 self.post_compress_man_pages()
710 self.post_remove_static_libs()
711
712 # Package the result.
713 # Make all these little package from the build environment.
714 log.info(_("Creating packages:"))
715 pkgs = []
716 for pkg in reversed(self.pkg.packages):
717 packager = packages.packager.BinaryPackager(self.pakfire, pkg,
718 self, self.buildroot)
719 pkg = packager.run(self.resultdir)
720 pkgs.append(pkg)
721 log.info("")
722
723 for pkg in sorted(pkgs):
724 for line in pkg.dump(long=True).splitlines():
725 log.info(line)
726 log.info("")
727 log.info("")
728
729 def build_stage(self, stage):
730 # Get the buildscript for this stage.
731 buildscript = self.create_buildscript(stage)
732
733 # Execute the buildscript of this stage.
734 log.info(_("Running stage %s:") % stage)
735
736 try:
737 self.do(buildscript, shell=False)
738
739 finally:
740 # Remove the buildscript.
741 if os.path.exists(buildscript):
742 os.unlink(buildscript)
743
744 def post_remove_static_libs(self):
745 keep_libs = self.pkg.lexer.build.get_var("keep_libraries")
746 keep_libs = keep_libs.split()
747
748 try:
749 self.do("%s/remove-static-libs %s %s" % \
750 (SCRIPT_DIR, self.buildroot, " ".join(keep_libs)))
751 except Error, e:
752 log.warning(_("Could not remove static libraries: %s") % e)
753
754 def post_compress_man_pages(self):
755 try:
756 self.do("%s/compress-man-pages %s" % (SCRIPT_DIR, self.buildroot))
757 except Error, e:
758 log.warning(_("Compressing man pages did not complete successfully."))
759
760 def cleanup(self):
761 if os.path.exists(self.buildroot):
762 util.rm(self.buildroot)