]> git.ipfire.org Git - pakfire.git/blob - pakfire/builder.py
Bump version 0.9.9.
[pakfire.git] / pakfire / builder.py
1 #!/usr/bin/python
2 ###############################################################################
3 # #
4 # Pakfire - The IPFire package management system #
5 # Copyright (C) 2011 Pakfire development team #
6 # #
7 # This program is free software: you can redistribute it and/or modify #
8 # it under the terms of the GNU General Public License as published by #
9 # the Free Software Foundation, either version 3 of the License, or #
10 # (at your option) any later version. #
11 # #
12 # This program is distributed in the hope that it will be useful, #
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
15 # GNU General Public License for more details. #
16 # #
17 # You should have received a copy of the GNU General Public License #
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. #
19 # #
20 ###############################################################################
21
22 import fcntl
23 import grp
24 import logging
25 import math
26 import os
27 import re
28 import shutil
29 import socket
30 import stat
31 import time
32 import uuid
33
34 import base
35 import chroot
36 import logger
37 import packages
38 import packages.packager
39 import repository
40 import util
41
42 from constants import *
43 from i18n import _
44 from errors import BuildError, BuildRootLocked, Error
45
46
47 BUILD_LOG_HEADER = """
48 ____ _ __ _ _ _ _ _
49 | _ \ __ _| | __/ _(_)_ __ ___ | |__ _ _(_) | __| | ___ _ __
50 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
51 | __/ (_| | <| _| | | | __/ | |_) | |_| | | | (_| | __/ |
52 |_| \__,_|_|\_\_| |_|_| \___| |_.__/ \__,_|_|_|\__,_|\___|_|
53
54 Time : %(time)s
55 Host : %(host)s
56 Version : %(version)s
57
58 """
59
60 class BuildEnviron(object):
61 # The version of the kernel this machine is running.
62 kernel_version = os.uname()[2]
63
64 def __init__(self, pkg=None, distro_config=None, build_id=None, logfile=None,
65 builder_mode="release", **pakfire_args):
66 # Set mode.
67 assert builder_mode in ("development", "release",)
68 self.mode = builder_mode
69
70 # Disable the build repository in release mode.
71 if self.mode == "release":
72 if pakfire_args.has_key("disable_repos") and pakfire_args["disable_repos"]:
73 pakfire_args["disable_repos"] += ["build",]
74 else:
75 pakfire_args["disable_repos"] = ["build",]
76
77 # Save the build id and generate one if no build id was provided.
78 if not build_id:
79 build_id = "%s" % uuid.uuid4()
80
81 self.build_id = build_id
82
83 # Setup the logging.
84 if logfile:
85 self.log = logging.getLogger(self.build_id)
86 # Propage everything to the root logger that we will see something
87 # on the terminal.
88 self.log.propagate = 1
89 self.log.setLevel(logging.INFO)
90
91 # Add the given logfile to the logger.
92 h = logging.FileHandler(logfile)
93 self.log.addHandler(h)
94
95 # Format the log output for the file.
96 f = logger.BuildFormatter()
97 h.setFormatter(f)
98 else:
99 # If no logile was given, we use the root logger.
100 self.log = logging.getLogger()
101
102 # Log information about pakfire and some more information, when we
103 # are running in release mode.
104 if self.mode == "release":
105 logdata = {
106 "host" : socket.gethostname(),
107 "time" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
108 "version" : "Pakfire %s" % PAKFIRE_VERSION,
109 }
110
111 for line in BUILD_LOG_HEADER.splitlines():
112 self.log.info(line % logdata)
113
114 # Create pakfire instance.
115 if pakfire_args.has_key("mode"):
116 del pakfire_args["mode"]
117 self.pakfire = base.Pakfire(mode="builder", distro_config=distro_config, **pakfire_args)
118 self.distro = self.pakfire.distro
119 self.path = self.pakfire.path
120
121 # Log the package information.
122 self.pkg = packages.Makefile(self.pakfire, pkg)
123 self.log.info(_("Package information:"))
124 for line in self.pkg.dump(long=True).splitlines():
125 self.log.info(" %s" % line)
126 self.log.info("")
127
128 # XXX need to make this configureable
129 self.settings = {
130 "enable_loop_devices" : True,
131 "enable_ccache" : True,
132 "enable_icecream" : False,
133 }
134 #self.settings.update(settings)
135
136 self.buildroot = "/buildroot"
137
138 # Lock the buildroot
139 self._lock = None
140 self.lock()
141
142 # Save the build time.
143 self.build_time = int(time.time())
144
145 @property
146 def arch(self):
147 """
148 Inherit architecture from distribution configuration.
149 """
150 return self.distro.arch
151
152 @property
153 def info(self):
154 return {
155 "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
156 "build_host" : socket.gethostname(),
157 "build_id" : self.build_id,
158 "build_time" : self.build_time,
159 }
160
161 def lock(self):
162 filename = os.path.join(self.path, ".lock")
163
164 try:
165 self._lock = open(filename, "a+")
166 except IOError, e:
167 return 0
168
169 try:
170 fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
171 except IOError, e:
172 raise BuildRootLocked, "Buildroot is locked"
173
174 return 1
175
176 def unlock(self):
177 if self._lock:
178 self._lock.close()
179 self._lock = None
180
181 def copyin(self, file_out, file_in):
182 if file_in.startswith("/"):
183 file_in = file_in[1:]
184
185 file_in = self.chrootPath(file_in)
186
187 #if not os.path.exists(file_out):
188 # return
189
190 dir_in = os.path.dirname(file_in)
191 if not os.path.exists(dir_in):
192 os.makedirs(dir_in)
193
194 logging.debug("%s --> %s" % (file_out, file_in))
195
196 shutil.copy2(file_out, file_in)
197
198 def copyout(self, file_in, file_out):
199 if file_in.startswith("/"):
200 file_in = file_in[1:]
201
202 file_in = self.chrootPath(file_in)
203
204 #if not os.path.exists(file_in):
205 # return
206
207 dir_out = os.path.dirname(file_out)
208 if not os.path.exists(dir_out):
209 os.makedirs(dir_out)
210
211 logging.debug("%s --> %s" % (file_in, file_out))
212
213 shutil.copy2(file_in, file_out)
214
215 def copy_result(self, resultdir):
216 dir_in = self.chrootPath("result")
217
218 for dir, subdirs, files in os.walk(dir_in):
219 basename = os.path.basename(dir)
220 dir = dir[len(self.chrootPath()):]
221 for file in files:
222 file_in = os.path.join(dir, file)
223
224 file_out = os.path.join(
225 resultdir,
226 basename,
227 file,
228 )
229
230 self.copyout(file_in, file_out)
231
232 def extract(self, requires=None, build_deps=True):
233 """
234 Gets a dependency set and extracts all packages
235 to the environment.
236 """
237 if not requires:
238 requires = []
239
240 # Add neccessary build dependencies.
241 requires += BUILD_PACKAGES
242
243 # If we have ccache enabled, we need to extract it
244 # to the build chroot.
245 if self.settings.get("enable_ccache"):
246 requires.append("ccache")
247
248 # If we have icecream enabled, we need to extract it
249 # to the build chroot.
250 if self.settings.get("enable_icecream"):
251 requires.append("icecream")
252
253 # Get build dependencies from source package.
254 for req in self.pkg.requires:
255 requires.append(req)
256
257 # Install all packages.
258 self.install(requires)
259
260 # Copy the makefile and load source tarballs.
261 self.pkg.extract(_("Extracting"),
262 prefix=os.path.join(self.path, "build"))
263
264 def install(self, requires):
265 """
266 Install everything that is required in requires.
267 """
268 # If we got nothing to do, we quit immediately.
269 if not requires:
270 return
271
272 self.pakfire.install(requires, interactive=False,
273 allow_downgrade=True, logger=self.log)
274
275 def install_test(self):
276 pkgs = []
277 for dir, subdirs, files in os.walk(self.chrootPath("result")):
278 for file in files:
279 pkgs.append(os.path.join(dir, file))
280
281 self.pakfire.localinstall(pkgs, yes=True)
282
283 def chrootPath(self, *args):
284 # Remove all leading slashes
285 _args = []
286 for arg in args:
287 if arg.startswith("/"):
288 arg = arg[1:]
289 _args.append(arg)
290 args = _args
291
292 ret = os.path.join(self.path, *args)
293 ret = ret.replace("//", "/")
294
295 assert ret.startswith(self.path)
296
297 return ret
298
299 def prepare(self):
300 prepared_tag = ".prepared"
301
302 if os.path.exists(self.chrootPath(prepared_tag)):
303 return
304
305 # Create directory.
306 if not os.path.exists(self.path):
307 os.makedirs(self.path)
308
309 # Create important directories.
310 dirs = [
311 "build",
312 self.buildroot,
313 "dev",
314 "dev/pts",
315 "dev/shm",
316 "etc",
317 "proc",
318 "result",
319 "sys",
320 "tmp",
321 "usr/src",
322 ]
323
324 # Create cache dir if ccache is enabled.
325 if self.settings.get("enable_ccache"):
326 dirs.append("var/cache/ccache")
327
328 if not os.path.exists(CCACHE_CACHE_DIR):
329 os.makedirs(CCACHE_CACHE_DIR)
330
331 for dir in dirs:
332 dir = self.chrootPath(dir)
333 if not os.path.exists(dir):
334 os.makedirs(dir)
335
336 # Create neccessary files like /etc/fstab and /etc/mtab.
337 files = (
338 "etc/fstab",
339 "etc/mtab",
340 prepared_tag,
341 )
342
343 for file in files:
344 file = self.chrootPath(file)
345 dir = os.path.dirname(file)
346 if not os.path.exists(dir):
347 os.makedirs(dir)
348 f = open(file, "w")
349 f.close()
350
351 self._prepare_dev()
352 self._prepare_dns()
353
354 def _prepare_dev(self):
355 prevMask = os.umask(0000)
356
357 nodes = [
358 ("dev/null", stat.S_IFCHR | 0666, os.makedev(1, 3)),
359 ("dev/full", stat.S_IFCHR | 0666, os.makedev(1, 7)),
360 ("dev/zero", stat.S_IFCHR | 0666, os.makedev(1, 5)),
361 ("dev/random", stat.S_IFCHR | 0666, os.makedev(1, 8)),
362 ("dev/urandom", stat.S_IFCHR | 0444, os.makedev(1, 9)),
363 ("dev/tty", stat.S_IFCHR | 0666, os.makedev(5, 0)),
364 ("dev/console", stat.S_IFCHR | 0600, os.makedev(5, 1)),
365 ]
366
367 # If we need loop devices (which are optional) we create them here.
368 if self.settings["enable_loop_devices"]:
369 for i in range(0, 7):
370 nodes.append(("dev/loop%d" % i, stat.S_IFBLK | 0660, os.makedev(7, i)))
371
372 # Create all the nodes.
373 for node in nodes:
374 self._create_node(*node)
375
376 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
377 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
378 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
379 os.symlink("/proc/self/fd", self.chrootPath("dev", "fd"))
380
381 # make device node for el4 and el5
382 if self.kernel_version < "2.6.19":
383 self._make_node("dev/ptmx", stat.S_IFCHR | 0666, os.makedev(5, 2))
384 else:
385 os.symlink("/dev/pts/ptmx", self.chrootPath("dev", "ptmx"))
386
387 os.umask(prevMask)
388
389 def _prepare_dns(self):
390 """
391 Add DNS resolution facility to chroot environment by copying
392 /etc/resolv.conf and /etc/hosts.
393 """
394 for i in ("/etc/resolv.conf", "/etc/hosts"):
395 self.copyin(i, i)
396
397 def _create_node(self, filename, mode, device):
398 logging.debug("Create node: %s (%s)" % (filename, mode))
399
400 filename = self.chrootPath(filename)
401
402 # Create parent directory if it is missing.
403 dirname = os.path.dirname(filename)
404 if not os.path.exists(dirname):
405 os.makedirs(dirname)
406
407 os.mknod(filename, mode, device)
408
409 def destroy(self):
410 util.orphans_kill(self.path)
411
412 logging.debug("Destroying environment %s" % self.path)
413
414 if os.path.exists(self.path):
415 util.rm(self.path)
416
417 def cleanup(self):
418 logging.debug("Cleaning environemnt.")
419
420 # Remove the build directory and buildroot.
421 dirs = ("build", self.buildroot, "result")
422
423 for d in dirs:
424 d = self.chrootPath(d)
425 if not os.path.exists(d):
426 continue
427
428 util.rm(d)
429 os.makedirs(d)
430
431 def _mountall(self):
432 self.log.debug("Mounting environment")
433 for cmd, mountpoint in self.mountpoints:
434 cmd = "%s %s" % (cmd, self.chrootPath(mountpoint))
435 chroot.do(cmd, shell=True)
436
437 def _umountall(self):
438 self.log.debug("Umounting environment")
439 for cmd, mountpoint in self.mountpoints:
440 cmd = "umount -n %s" % self.chrootPath(mountpoint)
441 chroot.do(cmd, raiseExc=0, shell=True)
442
443 @property
444 def mountpoints(self):
445 ret = [
446 ("mount -n -t proc pakfire_chroot_proc", "proc"),
447 ("mount -n -t sysfs pakfire_chroot_sysfs", "sys"),
448 ]
449
450 mountopt = "gid=%d,mode=0620,ptmxmode=0666" % grp.getgrnam("tty").gr_gid
451 if self.kernel_version >= "2.6.29":
452 mountopt += ",newinstance"
453
454 ret.extend([
455 ("mount -n -t devpts -o %s pakfire_chroot_devpts" % mountopt, "dev/pts"),
456 ("mount -n -t tmpfs pakfire_chroot_shmfs", "dev/shm"),
457 ])
458
459 if self.settings.get("enable_ccache"):
460 ret.append(("mount -n --bind %s" % CCACHE_CACHE_DIR, "var/cache/ccache"))
461
462 return ret
463
464 @property
465 def environ(self):
466 env = {
467 # Add HOME manually, because it is occasionally not set
468 # and some builds get in trouble then.
469 "HOME" : "/root",
470 "TERM" : os.environ.get("TERM", "dumb"),
471 "PS1" : "\u:\w\$ ",
472
473 "BUILDROOT" : self.buildroot,
474 "PARALLELISMFLAGS" : "-j%s" % util.calc_parallelism(),
475 }
476
477 # Inherit environment from distro
478 env.update(self.pakfire.distro.environ)
479
480 # Icecream environment settings
481 if self.settings.get("enable_icecream", False):
482 # Set the toolchain path
483 if self.settings.get("icecream_toolchain", None):
484 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
485
486 # Set preferred host if configured.
487 if self.settings.get("icecream_preferred_host", None):
488 env["ICECC_PREFERRED_HOST"] = \
489 self.settings.get("icecream_preferred_host")
490
491 # XXX what do we need else?
492
493 return env
494
495 def do(self, command, shell=True, personality=None, logger=None, *args, **kwargs):
496 ret = None
497 try:
498 # Environment variables
499 env = self.environ
500
501 if kwargs.has_key("env"):
502 env.update(kwargs.pop("env"))
503
504 logging.debug("Environment:")
505 for k, v in sorted(env.items()):
506 logging.debug(" %s=%s" % (k, v))
507
508 # Update personality it none was set
509 if not personality:
510 personality = self.distro.personality
511
512 # Make every shell to a login shell because we set a lot of
513 # environment things there.
514 if shell:
515 command = ["bash", "--login", "-c", command]
516
517 self._mountall()
518
519 if not kwargs.has_key("chrootPath"):
520 kwargs["chrootPath"] = self.chrootPath()
521
522 ret = chroot.do(
523 command,
524 personality=personality,
525 shell=False,
526 env=env,
527 logger=logger,
528 *args,
529 **kwargs
530 )
531
532 finally:
533 self._umountall()
534
535 return ret
536
537 def build(self, install_test=True):
538 assert self.pkg
539
540 pkgfile = os.path.join("/build", os.path.basename(self.pkg.filename))
541 resultdir = self.chrootPath("/result")
542
543 # Create the build command, that is executed in the chroot.
544 build_command = ["/usr/lib/pakfire/builder", "--offline", "build", pkgfile,
545 "--nodeps", "--resultdir=/result",]
546
547 try:
548 self.do(" ".join(build_command), logger=self.log)
549
550 except Error:
551 raise BuildError, _("The build command failed. See logfile for details.")
552
553 # Perform install test.
554 if install_test:
555 self.install_test()
556
557 # Copy the final packages and stuff.
558 # XXX TODO resultdir
559
560 def shell(self, args=[]):
561 if not util.cli_is_interactive():
562 logging.warning("Cannot run shell on non-interactive console.")
563 return
564
565 # Install all packages that are needed to run a shell.
566 self.install(SHELL_PACKAGES)
567
568 # XXX need to set CFLAGS here
569 command = "/usr/sbin/chroot %s /usr/bin/chroot-shell %s" % \
570 (self.chrootPath(), " ".join(args))
571
572 # Add personality if we require one
573 if self.pakfire.distro.personality:
574 command = "%s %s" % (self.pakfire.distro.personality, command)
575
576 for key, val in self.environ.items():
577 command = "%s=\"%s\" " % (key, val) + command
578
579 # Empty the environment
580 command = "env -i - %s" % command
581
582 logging.debug("Shell command: %s" % command)
583
584 try:
585 self._mountall()
586
587 shell = os.system(command)
588 return os.WEXITSTATUS(shell)
589
590 finally:
591 self._umountall()
592
593
594 class Builder(object):
595 def __init__(self, pakfire, filename, resultdir, **kwargs):
596 self.pakfire = pakfire
597
598 self.filename = filename
599
600 self.resultdir = resultdir
601
602 # Open package file.
603 self.pkg = packages.Makefile(self.pakfire, self.filename)
604
605 #self.buildroot = "/tmp/pakfire_buildroot/%s" % util.random_string(20)
606 self.buildroot = "/buildroot"
607
608 self._environ = {
609 "BUILDROOT" : self.buildroot,
610 "LANG" : "C",
611 }
612
613 @property
614 def distro(self):
615 return self.pakfire.distro
616
617 @property
618 def environ(self):
619 environ = os.environ
620
621 # Get all definitions from the package.
622 environ.update(self.pkg.exports)
623
624 # Overwrite some definitions by default values.
625 environ.update(self._environ)
626
627 return environ
628
629 def do(self, command, shell=True, personality=None, cwd=None, *args, **kwargs):
630 # Environment variables
631 logging.debug("Environment:")
632 for k, v in sorted(self.environ.items()):
633 logging.debug(" %s=%s" % (k, v))
634
635 # Update personality it none was set
636 if not personality:
637 personality = self.distro.personality
638
639 if not cwd:
640 cwd = "/%s" % LOCAL_TMP_PATH
641
642 # Make every shell to a login shell because we set a lot of
643 # environment things there.
644 if shell:
645 command = ["bash", "--login", "-c", command]
646
647 return chroot.do(
648 command,
649 personality=personality,
650 shell=False,
651 env=self.environ,
652 logger=logging.getLogger(),
653 cwd=cwd,
654 *args,
655 **kwargs
656 )
657
658 def create_icecream_toolchain(self):
659 try:
660 out = self.do("icecc --build-native 2>/dev/null", returnOutput=True)
661 except Error:
662 return
663
664 for line in out.splitlines():
665 m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
666 if m:
667 self._environ["icecream_toolchain"] = "/%s" % m.group(1)
668
669 def create_buildscript(self, stage):
670 file = "/tmp/build_%s" % util.random_string()
671
672 # Get buildscript from the package.
673 script = self.pkg.get_buildscript(stage)
674
675 # Write script to an empty file.
676 f = open(file, "w")
677 f.write("#!/bin/sh\n\n")
678 f.write("set -e\n")
679 f.write("set -x\n")
680 f.write("\n%s\n" % script)
681 f.write("exit 0\n")
682 f.close()
683 os.chmod(file, 700)
684
685 return file
686
687 def build(self):
688 # Create buildroot.
689 if not os.path.exists(self.buildroot):
690 os.makedirs(self.buildroot)
691
692 # Build icecream toolchain if icecream is installed.
693 self.create_icecream_toolchain()
694
695 for stage in ("prepare", "build", "test", "install"):
696 self.build_stage(stage)
697
698 # Package the result.
699 # Make all these little package from the build environment.
700 logging.info(_("Creating packages:"))
701 pkgs = []
702 for pkg in reversed(self.pkg.packages):
703 packager = packages.packager.BinaryPackager(self.pakfire, pkg,
704 self, self.buildroot)
705 pkg = packager.run(self.resultdir)
706 pkgs.append(pkg)
707 logging.info("")
708
709 for pkg in sorted(pkgs):
710 for line in pkg.dump(long=True).splitlines():
711 logging.info(line)
712 logging.info("")
713 logging.info("")
714
715 def build_stage(self, stage):
716 # Get the buildscript for this stage.
717 buildscript = self.create_buildscript(stage)
718
719 # Execute the buildscript of this stage.
720 logging.info(_("Running stage %s:") % stage)
721
722 try:
723 self.do(buildscript, shell=False)
724
725 finally:
726 # Remove the buildscript.
727 if os.path.exists(buildscript):
728 os.unlink(buildscript)
729
730 def cleanup(self):
731 if os.path.exists(self.buildroot):
732 util.rm(self.buildroot)