]>
git.ipfire.org Git - people/stevee/pakfire.git/blob - pakfire/builder.py
23 from constants
import *
25 from errors
import BuildError
, BuildRootLocked
, Error
28 class Builder(object):
29 # The version of the kernel this machine is running.
30 kernel_version
= os
.uname()[2]
32 def __init__(self
, pkg
=None, distro_config
=None, build_id
=None, **pakfire_args
):
37 # Create pakfire instance.
38 self
.pakfire
= base
.Pakfire(distro_config
=distro_config
, **pakfire_args
)
39 self
.distro
= self
.pakfire
.distro
40 self
.path
= self
.pakfire
.path
45 # XXX need to make this configureable
47 "enable_loop_devices" : True,
48 "enable_ccache" : True,
49 "enable_icecream" : False,
51 #self.settings.update(settings)
53 self
.buildroot
= "/buildroot"
59 # Save the build time.
60 self
.build_time
= int(time
.time())
62 # Save the build id and generate one if no build id was provided.
64 build_id
= uuid
.uuid4()
66 self
.build_id
= build_id
69 return getattr(self
, "_pkg", None)
71 def set_pkg(self
, pkg
):
76 self
._pkg
= packages
.open(self
.pakfire
, None, pkg
)
78 # Log the package information.
79 if not isinstance(self
._pkg
, packages
.Makefile
):
80 dump
= self
._pkg
.dump(long=True)
85 pkg
= property(get_pkg
, set_pkg
)
90 Inherit architecture from distribution configuration.
92 return self
.distro
.arch
97 "build_date" : time
.strftime("%a, %d %b %Y %H:%M:%S +0000", time
.gmtime(self
.build_time
)),
98 "build_host" : socket
.gethostname(),
99 "build_id" : self
.build_id
,
100 "build_time" : self
.build_time
,
104 filename
= os
.path
.join(self
.path
, ".lock")
107 self
._lock
= open(filename
, "a+")
112 fcntl
.lockf(self
._lock
.fileno(), fcntl
.LOCK_EX | fcntl
.LOCK_NB
)
114 raise BuildRootLocked
, "Buildroot is locked"
123 def copyin(self
, file_out
, file_in
):
124 if file_in
.startswith("/"):
125 file_in
= file_in
[1:]
127 file_in
= self
.chrootPath(file_in
)
129 #if not os.path.exists(file_out):
132 dir_in
= os
.path
.dirname(file_in
)
133 if not os
.path
.exists(dir_in
):
136 logging
.debug("%s --> %s" % (file_out
, file_in
))
138 shutil
.copy2(file_out
, file_in
)
140 def copyout(self
, file_in
, file_out
):
141 if file_in
.startswith("/"):
142 file_in
= file_in
[1:]
144 file_in
= self
.chrootPath(file_in
)
146 #if not os.path.exists(file_in):
149 dir_out
= os
.path
.dirname(file_out
)
150 if not os
.path
.exists(dir_out
):
153 logging
.debug("%s --> %s" % (file_in
, file_out
))
155 shutil
.copy2(file_in
, file_out
)
157 def copy_result(self
, resultdir
):
158 dir_in
= self
.chrootPath("result")
160 for dir, subdirs
, files
in os
.walk(dir_in
):
161 basename
= os
.path
.basename(dir)
162 dir = dir[len(self
.chrootPath()):]
164 file_in
= os
.path
.join(dir, file)
166 file_out
= os
.path
.join(
172 self
.copyout(file_in
, file_out
)
174 def extract(self
, requires
=None, build_deps
=True):
176 Gets a dependency set and extracts all packages
182 # Add neccessary build dependencies.
183 requires
+= BUILD_PACKAGES
185 # If we have ccache enabled, we need to extract it
186 # to the build chroot.
187 if self
.settings
.get("enable_ccache"):
188 requires
.append("ccache")
190 # If we have icecream enabled, we need to extract it
191 # to the build chroot.
192 if self
.settings
.get("enable_icecream"):
193 requires
.append("icecream")
195 # Get build dependencies from source package.
196 if isinstance(self
.pkg
, packages
.SourcePackage
):
197 for req
in self
.pkg
.requires
:
200 # Install all packages.
201 self
.install(requires
)
203 # Copy the makefile and load source tarballs.
204 if isinstance(self
.pkg
, packages
.Makefile
):
205 self
.pkg
.extract(self
)
207 elif isinstance(self
.pkg
, packages
.SourcePackage
):
208 self
.pkg
.extract(_("Extracting: %s (source)") % self
.pkg
.name
,
209 prefix
=os
.path
.join(self
.path
, "build"))
211 # If we have a makefile, we can only get the build dependencies
212 # after we have extracted all the rest.
213 if build_deps
and isinstance(self
.pkg
, packages
.Makefile
):
214 requires
= self
.make_requires()
218 self
.install(requires
)
220 def install(self
, requires
):
222 Install everything that is required in requires.
224 # If we got nothing to do, we quit immediately.
228 ds
= depsolve
.DependencySet(self
.pakfire
)
230 if isinstance(r
, packages
.BinaryPackage
):
237 ts
= transaction
.Transaction(self
.pakfire
, ds
)
240 def install_test(self
):
243 # Connect packages to the FS repository.
244 r
= repository
.FileSystemRepository(self
.pakfire
)
246 for dir, subdirs
, files
in os
.walk(self
.chrootPath("result")):
248 file = os
.path
.join(dir, file)
250 if not file.endswith(PACKAGE_EXTENSION
):
253 p
= packages
.BinaryPackage(self
.pakfire
, r
, file)
260 # XXX for now, return the root logger
261 return logging
.getLogger()
263 def chrootPath(self
, *args
):
264 # Remove all leading slashes
267 if arg
.startswith("/"):
272 ret
= os
.path
.join(self
.path
, *args
)
273 ret
= ret
.replace("//", "/")
275 assert ret
.startswith(self
.path
)
280 prepared_tag
= ".prepared"
282 if os
.path
.exists(self
.chrootPath(prepared_tag
)):
286 if not os
.path
.exists(self
.path
):
287 os
.makedirs(self
.path
)
289 # Create important directories.
304 # Create cache dir if ccache is enabled.
305 if self
.settings
.get("enable_ccache"):
306 dirs
.append("var/cache/ccache")
308 if not os
.path
.exists(CCACHE_CACHE_DIR
):
309 os
.makedirs(CCACHE_CACHE_DIR
)
312 dir = self
.chrootPath(dir)
313 if not os
.path
.exists(dir):
316 # Create neccessary files like /etc/fstab and /etc/mtab.
324 file = self
.chrootPath(file)
325 dir = os
.path
.dirname(file)
326 if not os
.path
.exists(dir):
334 def _prepare_dev(self
):
335 prevMask
= os
.umask(0000)
338 ("dev/null", stat
.S_IFCHR |
0666, os
.makedev(1, 3)),
339 ("dev/full", stat
.S_IFCHR |
0666, os
.makedev(1, 7)),
340 ("dev/zero", stat
.S_IFCHR |
0666, os
.makedev(1, 5)),
341 ("dev/random", stat
.S_IFCHR |
0666, os
.makedev(1, 8)),
342 ("dev/urandom", stat
.S_IFCHR |
0444, os
.makedev(1, 9)),
343 ("dev/tty", stat
.S_IFCHR |
0666, os
.makedev(5, 0)),
344 ("dev/console", stat
.S_IFCHR |
0600, os
.makedev(5, 1)),
347 # If we need loop devices (which are optional) we create them here.
348 if self
.settings
["enable_loop_devices"]:
349 for i
in range(0, 7):
350 nodes
.append(("dev/loop%d" % i
, stat
.S_IFBLK |
0660, os
.makedev(7, i
)))
352 # Create all the nodes.
354 self
._create
_node
(*node
)
356 os
.symlink("/proc/self/fd/0", self
.chrootPath("dev", "stdin"))
357 os
.symlink("/proc/self/fd/1", self
.chrootPath("dev", "stdout"))
358 os
.symlink("/proc/self/fd/2", self
.chrootPath("dev", "stderr"))
359 os
.symlink("/proc/self/fd", self
.chrootPath("dev", "fd"))
361 # make device node for el4 and el5
362 if self
.kernel_version
< "2.6.19":
363 self
._make
_node
("dev/ptmx", stat
.S_IFCHR |
0666, os
.makedev(5, 2))
365 os
.symlink("/dev/pts/ptmx", self
.chrootPath("dev", "ptmx"))
369 def _prepare_dns(self
):
371 Add DNS resolution facility to chroot environment by copying
372 /etc/resolv.conf and /etc/hosts.
374 for i
in ("/etc/resolv.conf", "/etc/hosts"):
377 def _create_node(self
, filename
, mode
, device
):
378 logging
.debug("Create node: %s (%s)" % (filename
, mode
))
380 filename
= self
.chrootPath(filename
)
382 # Create parent directory if it is missing.
383 dirname
= os
.path
.dirname(filename
)
384 if not os
.path
.exists(dirname
):
387 os
.mknod(filename
, mode
, device
)
390 logging
.debug("Destroying environment %s" % self
.path
)
392 if os
.path
.exists(self
.path
):
396 logging
.debug("Cleaning environemnt.")
398 # Run make clean and let it cleanup its stuff.
401 # Remove the build directory and buildroot.
402 dirs
= ("build", self
.buildroot
, "result")
405 d
= self
.chrootPath(d
)
406 if not os
.path
.exists(d
):
412 # Clear make_info cache.
413 if hasattr(self
, "_make_info"):
417 self
.log
.debug("Mounting environment")
418 for cmd
, mountpoint
in self
.mountpoints
:
419 cmd
= "%s %s" % (cmd
, self
.chrootPath(mountpoint
))
420 chroot
.do(cmd
, shell
=True)
422 def _umountall(self
):
423 self
.log
.debug("Umounting environment")
424 for cmd
, mountpoint
in self
.mountpoints
:
425 cmd
= "umount -n %s" % self
.chrootPath(mountpoint
)
426 chroot
.do(cmd
, raiseExc
=0, shell
=True)
429 def mountpoints(self
):
431 ("mount -n -t proc pakfire_chroot_proc", "proc"),
432 ("mount -n -t sysfs pakfire_chroot_sysfs", "sys"),
435 mountopt
= "gid=%d,mode=0620,ptmxmode=0666" % grp
.getgrnam("tty").gr_gid
436 if self
.kernel_version
>= "2.6.29":
437 mountopt
+= ",newinstance"
440 ("mount -n -t devpts -o %s pakfire_chroot_devpts" % mountopt
, "dev/pts"),
441 ("mount -n -t tmpfs pakfire_chroot_shmfs", "dev/shm"),
444 if self
.settings
.get("enable_ccache"):
445 ret
.append(("mount -n --bind %s" % CCACHE_CACHE_DIR
, "var/cache/ccache"))
450 def calc_parallelism():
452 Calculate how many processes to run
455 We take the log10(number of processors) * factor
457 num
= os
.sysconf("SC_NPROCESSORS_CONF")
461 return int(round(math
.log10(num
) * 26))
466 # Add HOME manually, because it is occasionally not set
467 # and some builds get in trouble then.
469 "TERM" : os
.environ
.get("TERM", "dumb"),
472 "BUILDROOT" : self
.buildroot
,
473 "PARALLELISMFLAGS" : "-j%s" % self
.calc_parallelism(),
476 # Inherit environment from distro
477 env
.update(self
.pakfire
.distro
.environ
)
479 # Icecream environment settings
480 if self
.settings
.get("enable_icecream", None):
481 # Set the toolchain path
482 if self
.settings
.get("icecream_toolchain", None):
483 env
["ICECC_VERSION"] = self
.settings
.get("icecream_toolchain")
485 # Set preferred host if configured.
486 if self
.settings
.get("icecream_preferred_host", None):
487 env
["ICECC_PREFERRED_HOST"] = \
488 self
.settings
.get("icecream_preferred_host")
490 # XXX what do we need else?
494 def do(self
, command
, shell
=True, personality
=None, *args
, **kwargs
):
497 # Environment variables
500 if kwargs
.has_key("env"):
501 env
.update(kwargs
.pop("env"))
503 logging
.debug("Environment:")
504 for k
, v
in sorted(env
.items()):
505 logging
.debug(" %s=%s" % (k
, v
))
507 # Update personality it none was set
509 personality
= self
.distro
.personality
511 # Make every shell to a login shell because we set a lot of
512 # environment things there.
514 command
= ["bash", "--login", "-c", command
]
518 if not kwargs
.has_key("chrootPath"):
519 kwargs
["chrootPath"] = self
.chrootPath()
523 personality
=personality
,
536 def make(self
, *args
, **kwargs
):
537 if isinstance(self
.pkg
, packages
.Makefile
):
538 filename
= os
.path
.basename(self
.pkg
.filename
)
539 elif isinstance(self
.pkg
, packages
.SourcePackage
):
540 filename
= "%s.%s" % (self
.pkg
.name
, MAKEFILE_EXTENSION
)
542 return self
.do("make -f /build/%s %s" % (filename
, " ".join(args
)),
547 if not hasattr(self
, "_make_info"):
550 output
= self
.make("buildinfo", returnOutput
=True)
552 for line
in output
.splitlines():
557 m
= re
.match(r
"^(\w+)=(.*)$", line
)
561 info
[m
.group(1)] = m
.group(2).strip("\"")
563 self
._make
_info
= info
565 return self
._make
_info
569 if hasattr(self
, "_packages"):
570 return self
._packages
573 output
= self
.make("packageinfo", returnOutput
=True)
576 for line
in output
.splitlines():
581 m
= re
.match(r
"^(\w+)=(.*)$", line
)
586 pkg
[k
] = v
.strip("\"")
588 # Create a dummy repository to link the virtual packages to
589 repo
= repository
.DummyRepository(self
.pakfire
)
593 pkg
= packages
.VirtualPackage(self
.pakfire
, pkg
) # XXX had to remove repo here?!
594 self
._packages
.append(pkg
)
596 return self
._packages
598 def make_requires(self
):
599 return self
.make_info
.get("PKG_BUILD_DEPS", "").split()
601 def make_sources(self
):
602 return self
.make_info
.get("PKG_FILES", "").split()
604 def create_icecream_toolchain(self
):
605 if not self
.settings
.get("enable_icecream", None):
608 out
= self
.do("icecc --build-native", returnOutput
=True)
610 for line
in out
.splitlines():
611 m
= re
.match(r
"^creating ([a-z0-9]+\.tar\.gz)", line
)
613 self
.settings
["icecream_toolchain"] = "/%s" % m
.group(1)
616 self
.create_icecream_toolchain()
622 raise BuildError
, "The build command failed."
624 for pkg
in reversed(self
.packages
):
625 packager
= packages
.BinaryPackager(self
.pakfire
, pkg
, self
)
631 def shell(self
, args
=[]):
632 if not util
.cli_is_interactive():
633 logging
.warning("Cannot run shell on non-interactive console.")
636 # Install all packages that are needed to run a shell.
637 self
.install(SHELL_PACKAGES
)
639 # XXX need to set CFLAGS here
640 command
= "/usr/sbin/chroot %s /usr/bin/chroot-shell %s" % \
641 (self
.chrootPath(), " ".join(args
))
643 # Add personality if we require one
644 if self
.pakfire
.distro
.personality
:
645 command
= "%s %s" % (self
.pakfire
.distro
.personality
, command
)
647 for key
, val
in self
.environ
.items():
648 command
= "%s=\"%s\" " % (key
, val
) + command
650 # Empty the environment
651 command
= "env -i - %s" % command
653 logging
.debug("Shell command: %s" % command
)
658 shell
= os
.system(command
)
659 return os
.WEXITSTATUS(shell
)