]>
Commit | Line | Data |
---|---|---|
47a4cb89 MT |
1 | #!/usr/bin/python |
2 | ||
3 | import fcntl | |
4 | import grp | |
5 | import logging | |
d59bde4c | 6 | import math |
47a4cb89 MT |
7 | import os |
8 | import re | |
9 | import shutil | |
fc4d4177 | 10 | import socket |
47a4cb89 MT |
11 | import stat |
12 | import time | |
f02283bb | 13 | import uuid |
47a4cb89 MT |
14 | |
15 | import depsolve | |
16 | import packages | |
fa6d335b | 17 | import repository |
47a4cb89 MT |
18 | import transaction |
19 | import util | |
20 | ||
21 | from constants import * | |
22 | from errors import BuildRootLocked | |
23 | ||
24 | ||
25 | class Builder(object): | |
26 | # The version of the kernel this machine is running. | |
27 | kernel_version = os.uname()[2] | |
28 | ||
f02283bb | 29 | def __init__(self, pakfire, pkg, build_id=None, **settings): |
47a4cb89 MT |
30 | self.pakfire = pakfire |
31 | self.pkg = pkg | |
32 | ||
33 | self.settings = { | |
34 | "enable_loop_devices" : True, | |
33f4679b | 35 | "enable_ccache" : True, |
8c716255 | 36 | "enable_icecream" : False, |
47a4cb89 | 37 | } |
5be98997 | 38 | self.settings.update(settings) |
47a4cb89 MT |
39 | |
40 | self.buildroot = "/buildroot" | |
41 | ||
42 | # Lock the buildroot | |
43 | self._lock = None | |
44 | self.lock() | |
45 | ||
fc4d4177 MT |
46 | # Save the build time. |
47 | self.build_time = int(time.time()) | |
f02283bb MT |
48 | |
49 | # Save the build id and generate one if no build id was provided. | |
50 | if not build_id: | |
51 | build_id = uuid.uuid4() | |
52 | ||
fc4d4177 MT |
53 | self.build_id = build_id |
54 | ||
fc4d4177 MT |
55 | @property |
56 | def info(self): | |
57 | return { | |
58 | "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)), | |
59 | "build_host" : socket.gethostname(), | |
60 | "build_id" : self.build_id, | |
61 | "build_time" : self.build_time, | |
62 | } | |
63 | ||
47a4cb89 MT |
64 | @property |
65 | def path(self): | |
66 | return self.pakfire.path | |
67 | ||
68 | def lock(self): | |
69 | filename = os.path.join(self.path, ".lock") | |
70 | ||
71 | try: | |
72 | self._lock = open(filename, "a+") | |
73 | except IOError, e: | |
74 | return 0 | |
75 | ||
76 | try: | |
77 | fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) | |
78 | except IOError, e: | |
79 | raise BuildRootLocked, "Buildroot is locked" | |
80 | ||
81 | return 1 | |
82 | ||
83 | def unlock(self): | |
84 | if self._lock: | |
85 | self._lock.close() | |
86 | self._lock = None | |
87 | ||
88 | def copyin(self, file_out, file_in): | |
89 | if file_in.startswith("/"): | |
90 | file_in = file_in[1:] | |
91 | ||
92 | file_in = self.chrootPath(file_in) | |
93 | ||
94 | #if not os.path.exists(file_out): | |
95 | # return | |
96 | ||
97 | dir_in = os.path.dirname(file_in) | |
98 | if not os.path.exists(dir_in): | |
99 | os.makedirs(dir_in) | |
100 | ||
101 | logging.debug("%s --> %s" % (file_out, file_in)) | |
102 | ||
103 | shutil.copy2(file_out, file_in) | |
104 | ||
105 | def copyout(self, file_in, file_out): | |
106 | if file_in.startswith("/"): | |
107 | file_in = file_in[1:] | |
108 | ||
109 | file_in = self.chrootPath(file_in) | |
110 | ||
111 | #if not os.path.exists(file_in): | |
112 | # return | |
113 | ||
114 | dir_out = os.path.dirname(file_out) | |
115 | if not os.path.exists(dir_out): | |
116 | os.makedirs(dir_out) | |
117 | ||
118 | logging.debug("%s --> %s" % (file_in, file_out)) | |
119 | ||
120 | shutil.copy2(file_in, file_out) | |
121 | ||
122 | def copy_result(self, resultdir): | |
123 | dir_in = self.chrootPath("result") | |
124 | ||
125 | for dir, subdirs, files in os.walk(dir_in): | |
126 | basename = os.path.basename(dir) | |
127 | dir = dir[len(self.chrootPath()):] | |
128 | for file in files: | |
129 | file_in = os.path.join(dir, file) | |
130 | ||
131 | file_out = os.path.join( | |
132 | resultdir, | |
133 | basename, | |
134 | file, | |
135 | ) | |
136 | ||
137 | self.copyout(file_in, file_out) | |
138 | ||
139 | def extract(self, requires=[], build_deps=True): | |
140 | """ | |
141 | Gets a dependency set and extracts all packages | |
142 | to the environment. | |
143 | """ | |
144 | ds = depsolve.DependencySet(self.pakfire) | |
145 | for p in BUILD_PACKAGES + requires: | |
146 | ds.add_requires(p) | |
5be98997 | 147 | |
3d960a21 MT |
148 | # XXX just because I screwed things up |
149 | # Adds all packages to the chroot environment | |
150 | #for p in self.pakfire.repos.get_all(): | |
151 | # ds.add_requires(p.name) | |
152 | ||
33f4679b MT |
153 | # If we have ccache enabled, we need to extract it |
154 | # to the build chroot. | |
155 | if self.settings.get("enable_ccache"): | |
156 | ds.add_requires("ccache") | |
157 | ||
5be98997 MT |
158 | # If we have icecream enabled, we need to extract it |
159 | # to the build chroot. | |
160 | if self.settings.get("enable_icecream"): | |
161 | ds.add_requires("icecream") | |
162 | ||
47a4cb89 | 163 | ds.resolve() |
1de8761d | 164 | ds.dump() |
47a4cb89 MT |
165 | |
166 | # Get build dependencies from source package. | |
167 | if isinstance(self.pkg, packages.SourcePackage): | |
168 | for req in self.pkg.requires: | |
169 | ds.add_requires(req) | |
170 | ||
1de8761d | 171 | ts = transaction.Transaction(self.pakfire, ds) |
47a4cb89 MT |
172 | ts.run() |
173 | ||
174 | # Copy the makefile and load source tarballs. | |
175 | if isinstance(self.pkg, packages.Makefile): | |
176 | self.pkg.extract(self) | |
177 | ||
178 | # If we have a makefile, we can only get the build dependencies | |
179 | # after we have extracted all the rest. | |
180 | if build_deps and isinstance(self.pkg, packages.Makefile): | |
181 | requires = self.make_requires() | |
182 | if not requires: | |
183 | return | |
184 | ||
185 | ds = depsolve.DependencySet(self.pakfire) | |
186 | for r in requires: | |
187 | ds.add_requires(r) | |
188 | ds.resolve() | |
1de8761d | 189 | ds.dump() |
47a4cb89 | 190 | |
1de8761d | 191 | ts = transaction.Transaction(self.pakfire, ds) |
47a4cb89 MT |
192 | ts.run() |
193 | ||
194 | @property | |
195 | def log(self): | |
196 | # XXX for now, return the root logger | |
197 | return logging.getLogger() | |
198 | ||
199 | def chrootPath(self, *args): | |
200 | # Remove all leading slashes | |
201 | _args = [] | |
202 | for arg in args: | |
203 | if arg.startswith("/"): | |
204 | arg = arg[1:] | |
205 | _args.append(arg) | |
206 | args = _args | |
207 | ||
208 | ret = os.path.join(self.path, *args) | |
209 | ret = ret.replace("//", "/") | |
210 | ||
211 | assert ret.startswith(self.path) | |
212 | ||
213 | return ret | |
214 | ||
215 | def prepare(self): | |
216 | # Create directory. | |
217 | if not os.path.exists(self.path): | |
218 | os.makedirs(self.path) | |
219 | ||
220 | # Create important directories. | |
221 | dirs = [ | |
222 | "build", | |
223 | self.buildroot, | |
224 | "dev", | |
225 | "dev/pts", | |
226 | "dev/shm", | |
227 | "etc", | |
228 | "proc", | |
229 | "result", | |
230 | "sys", | |
231 | "tmp", | |
232 | "usr/src", | |
233 | ] | |
33f4679b MT |
234 | |
235 | # Create cache dir if ccache is enabled. | |
236 | if self.settings.get("enable_ccache"): | |
237 | dirs.append("var/cache/ccache") | |
238 | ||
239 | if not os.path.exists(CCACHE_CACHE_DIR): | |
240 | os.makedirs(CCACHE_CACHE_DIR) | |
241 | ||
47a4cb89 MT |
242 | for dir in dirs: |
243 | dir = self.chrootPath(dir) | |
244 | if not os.path.exists(dir): | |
245 | os.makedirs(dir) | |
246 | ||
6378690e MT |
247 | # Create neccessary files like /etc/fstab and /etc/mtab. |
248 | files = ( | |
249 | "etc/fstab", | |
250 | "etc/mtab" | |
251 | ) | |
252 | ||
253 | for file in files: | |
254 | file = self.chrootPath(file) | |
255 | dir = os.path.dirname(file) | |
256 | if not os.path.exists(dir): | |
257 | os.makedirs(dir) | |
258 | f = open(file, "w") | |
259 | f.close() | |
260 | ||
47a4cb89 MT |
261 | self._prepare_dev() |
262 | self._prepare_users() | |
263 | self._prepare_dns() | |
264 | ||
265 | def _prepare_dev(self): | |
266 | prevMask = os.umask(0000) | |
267 | ||
268 | nodes = [ | |
269 | ("dev/null", stat.S_IFCHR | 0666, os.makedev(1, 3)), | |
270 | ("dev/full", stat.S_IFCHR | 0666, os.makedev(1, 7)), | |
271 | ("dev/zero", stat.S_IFCHR | 0666, os.makedev(1, 5)), | |
272 | ("dev/random", stat.S_IFCHR | 0666, os.makedev(1, 8)), | |
273 | ("dev/urandom", stat.S_IFCHR | 0444, os.makedev(1, 9)), | |
274 | ("dev/tty", stat.S_IFCHR | 0666, os.makedev(5, 0)), | |
275 | ("dev/console", stat.S_IFCHR | 0600, os.makedev(5, 1)), | |
276 | ] | |
277 | ||
278 | # If we need loop devices (which are optional) we create them here. | |
279 | if self.settings["enable_loop_devices"]: | |
280 | for i in range(0, 7): | |
281 | nodes.append(("dev/loop%d" % i, stat.S_IFBLK | 0660, os.makedev(7, i))) | |
282 | ||
283 | # Create all the nodes. | |
284 | for node in nodes: | |
285 | self._create_node(*node) | |
286 | ||
287 | os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin")) | |
288 | os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout")) | |
289 | os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr")) | |
290 | os.symlink("/proc/self/fd", self.chrootPath("dev", "fd")) | |
291 | ||
292 | # make device node for el4 and el5 | |
293 | if self.kernel_version < "2.6.19": | |
294 | self._make_node("dev/ptmx", stat.S_IFCHR | 0666, os.makedev(5, 2)) | |
295 | else: | |
296 | os.symlink("/dev/pts/ptmx", self.chrootPath("dev", "ptmx")) | |
297 | ||
298 | os.umask(prevMask) | |
299 | ||
300 | def _prepare_users(self): | |
301 | f = open(self.chrootPath("etc", "passwd"), "w") | |
302 | f.write("root:x:0:0:root:/root:/bin/bash\n") | |
303 | f.write("nobody:x:99:99:Nobody:/:/sbin/nologin\n") | |
304 | f.close() | |
305 | ||
306 | f = open(self.chrootPath("etc", "group"), "w") | |
307 | f.write("root:x:0:root\n") | |
308 | f.write("nobody:x:99:\n") | |
309 | f.close() | |
310 | ||
311 | def _prepare_dns(self): | |
18973967 MT |
312 | """ |
313 | Add DNS resolution facility to chroot environment by copying | |
314 | /etc/resolv.conf and /etc/hosts. | |
315 | """ | |
316 | for i in ("/etc/resolv.conf", "/etc/hosts"): | |
317 | self.copyin(i, i) | |
47a4cb89 MT |
318 | |
319 | def _create_node(self, filename, mode, device): | |
320 | logging.debug("Create node: %s (%s)" % (filename, mode)) | |
321 | ||
322 | filename = self.chrootPath(filename) | |
323 | ||
324 | # Create parent directory if it is missing. | |
325 | dirname = os.path.dirname(filename) | |
326 | if not os.path.exists(dirname): | |
327 | os.makedirs(dirname) | |
328 | ||
329 | os.mknod(filename, mode, device) | |
330 | ||
331 | def cleanup(self): | |
332 | logging.debug("Cleanup environment %s" % self.path) | |
333 | ||
334 | if os.path.exists(self.path): | |
335 | util.rm(self.path) | |
336 | ||
337 | def _mountall(self): | |
338 | self.log.debug("Mounting environment") | |
339 | for cmd, mountpoint in self.mountpoints: | |
340 | cmd = "%s %s" % (cmd, self.chrootPath(mountpoint)) | |
341 | util.do(cmd, shell=True) | |
342 | ||
343 | def _umountall(self): | |
344 | self.log.debug("Umounting environment") | |
345 | for cmd, mountpoint in self.mountpoints: | |
346 | cmd = "umount -n %s" % self.chrootPath(mountpoint) | |
347 | util.do(cmd, raiseExc=0, shell=True) | |
348 | ||
349 | @property | |
350 | def mountpoints(self): | |
351 | ret = [ | |
352 | ("mount -n -t proc pakfire_chroot_proc", "proc"), | |
353 | ("mount -n -t sysfs pakfire_chroot_sysfs", "sys"), | |
354 | ] | |
355 | ||
356 | mountopt = "gid=%d,mode=0620,ptmxmode=0666" % grp.getgrnam("tty").gr_gid | |
357 | if self.kernel_version >= "2.6.29": | |
358 | mountopt += ",newinstance" | |
359 | ||
360 | ret.extend([ | |
361 | ("mount -n -t devpts -o %s pakfire_chroot_devpts" % mountopt, "dev/pts"), | |
362 | ("mount -n -t tmpfs pakfire_chroot_shmfs", "dev/shm"), | |
363 | ]) | |
364 | ||
33f4679b MT |
365 | if self.settings.get("enable_ccache"): |
366 | ret.append(("mount -n --bind %s" % CCACHE_CACHE_DIR, "var/cache/ccache")) | |
367 | ||
47a4cb89 MT |
368 | return ret |
369 | ||
d59bde4c MT |
370 | @staticmethod |
371 | def calc_parallelism(): | |
372 | """ | |
373 | Calculate how many processes to run | |
374 | at the same time. | |
375 | ||
376 | We take the log10(number of processors) * factor | |
377 | """ | |
378 | num = os.sysconf("SC_NPROCESSORS_CONF") | |
379 | if num == 1: | |
380 | return 2 | |
381 | else: | |
382 | return int(round(math.log10(num) * 26)) | |
383 | ||
47a4cb89 MT |
384 | @property |
385 | def environ(self): | |
386 | env = { | |
0cf10c2d MT |
387 | # Add HOME manually, because it is occasionally not set |
388 | # and some builds get in trouble then. | |
389 | "HOME" : "/root", | |
89ebac67 MT |
390 | "TERM" : os.environ.get("TERM", "dumb"), |
391 | "PS1" : "\u:\w\$ ", | |
0cf10c2d | 392 | |
47a4cb89 | 393 | "BUILDROOT" : self.buildroot, |
d59bde4c | 394 | "PARALLELISMFLAGS" : "-j%s" % self.calc_parallelism(), |
47a4cb89 MT |
395 | } |
396 | ||
397 | # Inherit environment from distro | |
398 | env.update(self.pakfire.distro.environ) | |
399 | ||
5be98997 MT |
400 | # Icecream environment settings |
401 | if self.settings.get("enable_icecream", None): | |
402 | # Set the toolchain path | |
403 | if self.settings.get("icecream_toolchain", None): | |
404 | env["ICECC_VERSION"] = self.settings.get("icecream_toolchain") | |
405 | ||
406 | # Set preferred host if configured. | |
407 | if self.settings.get("icecream_preferred_host", None): | |
408 | env["ICECC_PREFERRED_HOST"] = \ | |
409 | self.settings.get("icecream_preferred_host") | |
410 | ||
47a4cb89 MT |
411 | # XXX what do we need else? |
412 | ||
413 | return env | |
414 | ||
415 | def do(self, command, shell=True, personality=None, *args, **kwargs): | |
416 | ret = None | |
417 | try: | |
418 | # Environment variables | |
419 | env = self.environ | |
420 | ||
421 | if kwargs.has_key("env"): | |
422 | env.update(kwargs.pop("env")) | |
423 | ||
15398910 MT |
424 | logging.debug("Environment:") |
425 | for k, v in sorted(env.items()): | |
426 | logging.debug(" %s=%s" % (k, v)) | |
427 | ||
e360ea59 MT |
428 | # Update personality it none was set |
429 | if not personality: | |
430 | personality = self.pakfire.distro.personality | |
431 | ||
5be98997 MT |
432 | # Make every shell to a login shell because we set a lot of |
433 | # environment things there. | |
434 | if shell: | |
435 | command = ["bash", "--login", "-c", command] | |
436 | ||
47a4cb89 MT |
437 | self._mountall() |
438 | ||
439 | if not kwargs.has_key("chrootPath"): | |
440 | kwargs["chrootPath"] = self.chrootPath() | |
441 | ||
442 | ret = util.do( | |
443 | command, | |
444 | personality=personality, | |
5be98997 | 445 | shell=False, |
47a4cb89 MT |
446 | env=env, |
447 | logger=self.log, | |
448 | *args, | |
449 | **kwargs | |
450 | ) | |
451 | ||
452 | finally: | |
453 | self._umountall() | |
454 | ||
455 | return ret | |
456 | ||
457 | def make(self, *args, **kwargs): | |
5be98997 MT |
458 | return self.do("make -f /build/%s %s" % \ |
459 | (os.path.basename(self.pkg.filename), " ".join(args)), | |
460 | **kwargs) | |
47a4cb89 MT |
461 | |
462 | @property | |
463 | def make_info(self): | |
464 | if not hasattr(self, "_make_info"): | |
465 | info = {} | |
466 | ||
467 | output = self.make("buildinfo", returnOutput=True) | |
468 | ||
469 | for line in output.splitlines(): | |
470 | # XXX temporarily | |
471 | if not line: | |
472 | break | |
473 | ||
474 | m = re.match(r"^(\w+)=(.*)$", line) | |
475 | if not m: | |
476 | continue | |
477 | ||
478 | info[m.group(1)] = m.group(2).strip("\"") | |
479 | ||
480 | self._make_info = info | |
481 | ||
482 | return self._make_info | |
483 | ||
484 | @property | |
485 | def packages(self): | |
486 | if hasattr(self, "_packages"): | |
487 | return self._packages | |
488 | ||
489 | pkgs = [] | |
490 | output = self.make("packageinfo", returnOutput=True) | |
491 | ||
492 | pkg = {} | |
493 | for line in output.splitlines(): | |
494 | if not line: | |
495 | pkgs.append(pkg) | |
496 | pkg = {} | |
497 | ||
498 | m = re.match(r"^(\w+)=(.*)$", line) | |
499 | if not m: | |
500 | continue | |
501 | ||
502 | k, v = m.groups() | |
503 | pkg[k] = v.strip("\"") | |
504 | ||
3723913b MT |
505 | # Create a dummy repository to link the virtual packages to |
506 | repo = repository.DummyRepository(self.pakfire) | |
507 | ||
47a4cb89 MT |
508 | self._packages = [] |
509 | for pkg in pkgs: | |
fa6d335b | 510 | pkg = packages.VirtualPackage(self.pakfire, pkg) # XXX had to remove repo here?! |
47a4cb89 MT |
511 | self._packages.append(pkg) |
512 | ||
513 | return self._packages | |
514 | ||
515 | def make_requires(self): | |
516 | return self.make_info.get("PKG_BUILD_DEPS", "").split() | |
517 | ||
518 | def make_sources(self): | |
519 | return self.make_info.get("PKG_FILES", "").split() | |
520 | ||
5be98997 MT |
521 | def create_icecream_toolchain(self): |
522 | if not self.settings.get("enable_icecream", None): | |
523 | return | |
524 | ||
525 | out = self.do("icecc --build-native", returnOutput=True) | |
526 | ||
527 | for line in out.splitlines(): | |
528 | m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line) | |
529 | if m: | |
530 | self.settings["icecream_toolchain"] = "/%s" % m.group(1) | |
531 | ||
47a4cb89 | 532 | def build(self): |
5be98997 MT |
533 | self.create_icecream_toolchain() |
534 | ||
47a4cb89 MT |
535 | self.make("build") |
536 | ||
537 | for pkg in reversed(self.packages): | |
538 | packager = packages.Packager(self.pakfire, pkg, self) | |
539 | packager() | |
540 | ||
541 | def dist(self): | |
542 | self.pkg.dist(self) | |
543 | ||
544 | def shell(self, args=[]): | |
47a4cb89 | 545 | # XXX need to set CFLAGS here |
a7596ccf | 546 | command = "/usr/sbin/chroot %s /usr/bin/chroot-shell %s" % \ |
47a4cb89 MT |
547 | (self.chrootPath(), " ".join(args)) |
548 | ||
e360ea59 MT |
549 | # Add personality if we require one |
550 | if self.pakfire.distro.personality: | |
a7596ccf MT |
551 | command = "%s %s" % (self.pakfire.distro.personality, command) |
552 | ||
553 | for key, val in self.environ.items(): | |
554 | command = "%s=\"%s\" " % (key, val) + command | |
e360ea59 | 555 | |
47a4cb89 | 556 | # Empty the environment |
a7596ccf | 557 | command = "env -i - %s" % command |
47a4cb89 MT |
558 | |
559 | logging.debug("Shell command: %s" % command) | |
560 | ||
561 | try: | |
562 | self._mountall() | |
563 | ||
564 | shell = os.system(command) | |
565 | return os.WEXITSTATUS(shell) | |
566 | ||
567 | finally: | |
568 | self._umountall() |