]>
Commit | Line | Data |
---|---|---|
47a4cb89 MT |
1 | #!/usr/bin/python |
2 | ||
3 | import fcntl | |
4 | import grp | |
5 | import logging | |
d59bde4c | 6 | import math |
47a4cb89 MT |
7 | import os |
8 | import re | |
9 | import shutil | |
10 | import stat | |
11 | import time | |
12 | ||
13 | import depsolve | |
14 | import packages | |
fa6d335b | 15 | import repository |
47a4cb89 MT |
16 | import transaction |
17 | import util | |
18 | ||
19 | from constants import * | |
20 | from errors import BuildRootLocked | |
21 | ||
22 | ||
23 | class Builder(object): | |
24 | # The version of the kernel this machine is running. | |
25 | kernel_version = os.uname()[2] | |
26 | ||
5be98997 | 27 | def __init__(self, pakfire, pkg, **settings): |
47a4cb89 MT |
28 | self.pakfire = pakfire |
29 | self.pkg = pkg | |
30 | ||
31 | self.settings = { | |
32 | "enable_loop_devices" : True, | |
33f4679b | 33 | "enable_ccache" : True, |
fa6d335b | 34 | "enable_icecream" : True, |
47a4cb89 | 35 | } |
5be98997 | 36 | self.settings.update(settings) |
47a4cb89 MT |
37 | |
38 | self.buildroot = "/buildroot" | |
39 | ||
40 | # Lock the buildroot | |
41 | self._lock = None | |
42 | self.lock() | |
43 | ||
44 | # Initialize the environment | |
45 | self.prepare() | |
46 | ||
47 | @property | |
48 | def path(self): | |
49 | return self.pakfire.path | |
50 | ||
51 | def lock(self): | |
52 | filename = os.path.join(self.path, ".lock") | |
53 | ||
54 | try: | |
55 | self._lock = open(filename, "a+") | |
56 | except IOError, e: | |
57 | return 0 | |
58 | ||
59 | try: | |
60 | fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) | |
61 | except IOError, e: | |
62 | raise BuildRootLocked, "Buildroot is locked" | |
63 | ||
64 | return 1 | |
65 | ||
66 | def unlock(self): | |
67 | if self._lock: | |
68 | self._lock.close() | |
69 | self._lock = None | |
70 | ||
71 | def copyin(self, file_out, file_in): | |
72 | if file_in.startswith("/"): | |
73 | file_in = file_in[1:] | |
74 | ||
75 | file_in = self.chrootPath(file_in) | |
76 | ||
77 | #if not os.path.exists(file_out): | |
78 | # return | |
79 | ||
80 | dir_in = os.path.dirname(file_in) | |
81 | if not os.path.exists(dir_in): | |
82 | os.makedirs(dir_in) | |
83 | ||
84 | logging.debug("%s --> %s" % (file_out, file_in)) | |
85 | ||
86 | shutil.copy2(file_out, file_in) | |
87 | ||
88 | def copyout(self, file_in, file_out): | |
89 | if file_in.startswith("/"): | |
90 | file_in = file_in[1:] | |
91 | ||
92 | file_in = self.chrootPath(file_in) | |
93 | ||
94 | #if not os.path.exists(file_in): | |
95 | # return | |
96 | ||
97 | dir_out = os.path.dirname(file_out) | |
98 | if not os.path.exists(dir_out): | |
99 | os.makedirs(dir_out) | |
100 | ||
101 | logging.debug("%s --> %s" % (file_in, file_out)) | |
102 | ||
103 | shutil.copy2(file_in, file_out) | |
104 | ||
105 | def copy_result(self, resultdir): | |
106 | dir_in = self.chrootPath("result") | |
107 | ||
108 | for dir, subdirs, files in os.walk(dir_in): | |
109 | basename = os.path.basename(dir) | |
110 | dir = dir[len(self.chrootPath()):] | |
111 | for file in files: | |
112 | file_in = os.path.join(dir, file) | |
113 | ||
114 | file_out = os.path.join( | |
115 | resultdir, | |
116 | basename, | |
117 | file, | |
118 | ) | |
119 | ||
120 | self.copyout(file_in, file_out) | |
121 | ||
122 | def extract(self, requires=[], build_deps=True): | |
123 | """ | |
124 | Gets a dependency set and extracts all packages | |
125 | to the environment. | |
126 | """ | |
127 | ds = depsolve.DependencySet(self.pakfire) | |
128 | for p in BUILD_PACKAGES + requires: | |
129 | ds.add_requires(p) | |
5be98997 | 130 | |
3d960a21 MT |
131 | # XXX just because I screwed things up |
132 | # Adds all packages to the chroot environment | |
133 | #for p in self.pakfire.repos.get_all(): | |
134 | # ds.add_requires(p.name) | |
135 | ||
33f4679b MT |
136 | # If we have ccache enabled, we need to extract it |
137 | # to the build chroot. | |
138 | if self.settings.get("enable_ccache"): | |
139 | ds.add_requires("ccache") | |
140 | ||
5be98997 MT |
141 | # If we have icecream enabled, we need to extract it |
142 | # to the build chroot. | |
143 | if self.settings.get("enable_icecream"): | |
144 | ds.add_requires("icecream") | |
145 | ||
47a4cb89 MT |
146 | ds.resolve() |
147 | ||
148 | # Get build dependencies from source package. | |
149 | if isinstance(self.pkg, packages.SourcePackage): | |
150 | for req in self.pkg.requires: | |
151 | ds.add_requires(req) | |
152 | ||
153 | ts = transaction.TransactionSet(self.pakfire, ds) | |
154 | ts.dump() | |
155 | ts.run() | |
156 | ||
157 | # Copy the makefile and load source tarballs. | |
158 | if isinstance(self.pkg, packages.Makefile): | |
159 | self.pkg.extract(self) | |
160 | ||
161 | # If we have a makefile, we can only get the build dependencies | |
162 | # after we have extracted all the rest. | |
163 | if build_deps and isinstance(self.pkg, packages.Makefile): | |
164 | requires = self.make_requires() | |
165 | if not requires: | |
166 | return | |
167 | ||
168 | ds = depsolve.DependencySet(self.pakfire) | |
169 | for r in requires: | |
170 | ds.add_requires(r) | |
171 | ds.resolve() | |
172 | ||
173 | ts = transaction.TransactionSet(self.pakfire, ds) | |
174 | ts.dump() | |
175 | ts.run() | |
176 | ||
177 | @property | |
178 | def log(self): | |
179 | # XXX for now, return the root logger | |
180 | return logging.getLogger() | |
181 | ||
182 | def chrootPath(self, *args): | |
183 | # Remove all leading slashes | |
184 | _args = [] | |
185 | for arg in args: | |
186 | if arg.startswith("/"): | |
187 | arg = arg[1:] | |
188 | _args.append(arg) | |
189 | args = _args | |
190 | ||
191 | ret = os.path.join(self.path, *args) | |
192 | ret = ret.replace("//", "/") | |
193 | ||
194 | assert ret.startswith(self.path) | |
195 | ||
196 | return ret | |
197 | ||
198 | def prepare(self): | |
199 | # Create directory. | |
200 | if not os.path.exists(self.path): | |
201 | os.makedirs(self.path) | |
202 | ||
203 | # Create important directories. | |
204 | dirs = [ | |
205 | "build", | |
206 | self.buildroot, | |
207 | "dev", | |
208 | "dev/pts", | |
209 | "dev/shm", | |
210 | "etc", | |
211 | "proc", | |
212 | "result", | |
213 | "sys", | |
214 | "tmp", | |
215 | "usr/src", | |
216 | ] | |
33f4679b MT |
217 | |
218 | # Create cache dir if ccache is enabled. | |
219 | if self.settings.get("enable_ccache"): | |
220 | dirs.append("var/cache/ccache") | |
221 | ||
222 | if not os.path.exists(CCACHE_CACHE_DIR): | |
223 | os.makedirs(CCACHE_CACHE_DIR) | |
224 | ||
47a4cb89 MT |
225 | for dir in dirs: |
226 | dir = self.chrootPath(dir) | |
227 | if not os.path.exists(dir): | |
228 | os.makedirs(dir) | |
229 | ||
230 | self._prepare_dev() | |
231 | self._prepare_users() | |
232 | self._prepare_dns() | |
233 | ||
234 | def _prepare_dev(self): | |
235 | prevMask = os.umask(0000) | |
236 | ||
237 | nodes = [ | |
238 | ("dev/null", stat.S_IFCHR | 0666, os.makedev(1, 3)), | |
239 | ("dev/full", stat.S_IFCHR | 0666, os.makedev(1, 7)), | |
240 | ("dev/zero", stat.S_IFCHR | 0666, os.makedev(1, 5)), | |
241 | ("dev/random", stat.S_IFCHR | 0666, os.makedev(1, 8)), | |
242 | ("dev/urandom", stat.S_IFCHR | 0444, os.makedev(1, 9)), | |
243 | ("dev/tty", stat.S_IFCHR | 0666, os.makedev(5, 0)), | |
244 | ("dev/console", stat.S_IFCHR | 0600, os.makedev(5, 1)), | |
245 | ] | |
246 | ||
247 | # If we need loop devices (which are optional) we create them here. | |
248 | if self.settings["enable_loop_devices"]: | |
249 | for i in range(0, 7): | |
250 | nodes.append(("dev/loop%d" % i, stat.S_IFBLK | 0660, os.makedev(7, i))) | |
251 | ||
252 | # Create all the nodes. | |
253 | for node in nodes: | |
254 | self._create_node(*node) | |
255 | ||
256 | os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin")) | |
257 | os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout")) | |
258 | os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr")) | |
259 | os.symlink("/proc/self/fd", self.chrootPath("dev", "fd")) | |
260 | ||
261 | # make device node for el4 and el5 | |
262 | if self.kernel_version < "2.6.19": | |
263 | self._make_node("dev/ptmx", stat.S_IFCHR | 0666, os.makedev(5, 2)) | |
264 | else: | |
265 | os.symlink("/dev/pts/ptmx", self.chrootPath("dev", "ptmx")) | |
266 | ||
267 | os.umask(prevMask) | |
268 | ||
269 | def _prepare_users(self): | |
270 | f = open(self.chrootPath("etc", "passwd"), "w") | |
271 | f.write("root:x:0:0:root:/root:/bin/bash\n") | |
272 | f.write("nobody:x:99:99:Nobody:/:/sbin/nologin\n") | |
273 | f.close() | |
274 | ||
275 | f = open(self.chrootPath("etc", "group"), "w") | |
276 | f.write("root:x:0:root\n") | |
277 | f.write("nobody:x:99:\n") | |
278 | f.close() | |
279 | ||
280 | def _prepare_dns(self): | |
18973967 MT |
281 | """ |
282 | Add DNS resolution facility to chroot environment by copying | |
283 | /etc/resolv.conf and /etc/hosts. | |
284 | """ | |
285 | for i in ("/etc/resolv.conf", "/etc/hosts"): | |
286 | self.copyin(i, i) | |
47a4cb89 MT |
287 | |
288 | def _create_node(self, filename, mode, device): | |
289 | logging.debug("Create node: %s (%s)" % (filename, mode)) | |
290 | ||
291 | filename = self.chrootPath(filename) | |
292 | ||
293 | # Create parent directory if it is missing. | |
294 | dirname = os.path.dirname(filename) | |
295 | if not os.path.exists(dirname): | |
296 | os.makedirs(dirname) | |
297 | ||
298 | os.mknod(filename, mode, device) | |
299 | ||
300 | def cleanup(self): | |
301 | logging.debug("Cleanup environment %s" % self.path) | |
302 | ||
303 | if os.path.exists(self.path): | |
304 | util.rm(self.path) | |
305 | ||
306 | def _mountall(self): | |
307 | self.log.debug("Mounting environment") | |
308 | for cmd, mountpoint in self.mountpoints: | |
309 | cmd = "%s %s" % (cmd, self.chrootPath(mountpoint)) | |
310 | util.do(cmd, shell=True) | |
311 | ||
312 | def _umountall(self): | |
313 | self.log.debug("Umounting environment") | |
314 | for cmd, mountpoint in self.mountpoints: | |
315 | cmd = "umount -n %s" % self.chrootPath(mountpoint) | |
316 | util.do(cmd, raiseExc=0, shell=True) | |
317 | ||
318 | @property | |
319 | def mountpoints(self): | |
320 | ret = [ | |
321 | ("mount -n -t proc pakfire_chroot_proc", "proc"), | |
322 | ("mount -n -t sysfs pakfire_chroot_sysfs", "sys"), | |
323 | ] | |
324 | ||
325 | mountopt = "gid=%d,mode=0620,ptmxmode=0666" % grp.getgrnam("tty").gr_gid | |
326 | if self.kernel_version >= "2.6.29": | |
327 | mountopt += ",newinstance" | |
328 | ||
329 | ret.extend([ | |
330 | ("mount -n -t devpts -o %s pakfire_chroot_devpts" % mountopt, "dev/pts"), | |
331 | ("mount -n -t tmpfs pakfire_chroot_shmfs", "dev/shm"), | |
332 | ]) | |
333 | ||
33f4679b MT |
334 | if self.settings.get("enable_ccache"): |
335 | ret.append(("mount -n --bind %s" % CCACHE_CACHE_DIR, "var/cache/ccache")) | |
336 | ||
47a4cb89 MT |
337 | return ret |
338 | ||
d59bde4c MT |
339 | @staticmethod |
340 | def calc_parallelism(): | |
341 | """ | |
342 | Calculate how many processes to run | |
343 | at the same time. | |
344 | ||
345 | We take the log10(number of processors) * factor | |
346 | """ | |
347 | num = os.sysconf("SC_NPROCESSORS_CONF") | |
348 | if num == 1: | |
349 | return 2 | |
350 | else: | |
351 | return int(round(math.log10(num) * 26)) | |
352 | ||
47a4cb89 MT |
353 | @property |
354 | def environ(self): | |
355 | env = { | |
0cf10c2d MT |
356 | # Add HOME manually, because it is occasionally not set |
357 | # and some builds get in trouble then. | |
358 | "HOME" : "/root", | |
89ebac67 MT |
359 | "TERM" : os.environ.get("TERM", "dumb"), |
360 | "PS1" : "\u:\w\$ ", | |
0cf10c2d | 361 | |
47a4cb89 | 362 | "BUILDROOT" : self.buildroot, |
d59bde4c | 363 | "PARALLELISMFLAGS" : "-j%s" % self.calc_parallelism(), |
47a4cb89 MT |
364 | } |
365 | ||
366 | # Inherit environment from distro | |
367 | env.update(self.pakfire.distro.environ) | |
368 | ||
5be98997 MT |
369 | # Icecream environment settings |
370 | if self.settings.get("enable_icecream", None): | |
371 | # Set the toolchain path | |
372 | if self.settings.get("icecream_toolchain", None): | |
373 | env["ICECC_VERSION"] = self.settings.get("icecream_toolchain") | |
374 | ||
375 | # Set preferred host if configured. | |
376 | if self.settings.get("icecream_preferred_host", None): | |
377 | env["ICECC_PREFERRED_HOST"] = \ | |
378 | self.settings.get("icecream_preferred_host") | |
379 | ||
47a4cb89 MT |
380 | # XXX what do we need else? |
381 | ||
382 | return env | |
383 | ||
384 | def do(self, command, shell=True, personality=None, *args, **kwargs): | |
385 | ret = None | |
386 | try: | |
387 | # Environment variables | |
388 | env = self.environ | |
389 | ||
390 | if kwargs.has_key("env"): | |
391 | env.update(kwargs.pop("env")) | |
392 | ||
15398910 MT |
393 | logging.debug("Environment:") |
394 | for k, v in sorted(env.items()): | |
395 | logging.debug(" %s=%s" % (k, v)) | |
396 | ||
e360ea59 MT |
397 | # Update personality it none was set |
398 | if not personality: | |
399 | personality = self.pakfire.distro.personality | |
400 | ||
5be98997 MT |
401 | # Make every shell to a login shell because we set a lot of |
402 | # environment things there. | |
403 | if shell: | |
404 | command = ["bash", "--login", "-c", command] | |
405 | ||
47a4cb89 MT |
406 | self._mountall() |
407 | ||
408 | if not kwargs.has_key("chrootPath"): | |
409 | kwargs["chrootPath"] = self.chrootPath() | |
410 | ||
411 | ret = util.do( | |
412 | command, | |
413 | personality=personality, | |
5be98997 | 414 | shell=False, |
47a4cb89 MT |
415 | env=env, |
416 | logger=self.log, | |
417 | *args, | |
418 | **kwargs | |
419 | ) | |
420 | ||
421 | finally: | |
422 | self._umountall() | |
423 | ||
424 | return ret | |
425 | ||
426 | def make(self, *args, **kwargs): | |
5be98997 MT |
427 | return self.do("make -f /build/%s %s" % \ |
428 | (os.path.basename(self.pkg.filename), " ".join(args)), | |
429 | **kwargs) | |
47a4cb89 MT |
430 | |
431 | @property | |
432 | def make_info(self): | |
433 | if not hasattr(self, "_make_info"): | |
434 | info = {} | |
435 | ||
436 | output = self.make("buildinfo", returnOutput=True) | |
437 | ||
438 | for line in output.splitlines(): | |
439 | # XXX temporarily | |
440 | if not line: | |
441 | break | |
442 | ||
443 | m = re.match(r"^(\w+)=(.*)$", line) | |
444 | if not m: | |
445 | continue | |
446 | ||
447 | info[m.group(1)] = m.group(2).strip("\"") | |
448 | ||
449 | self._make_info = info | |
450 | ||
451 | return self._make_info | |
452 | ||
453 | @property | |
454 | def packages(self): | |
455 | if hasattr(self, "_packages"): | |
456 | return self._packages | |
457 | ||
458 | pkgs = [] | |
459 | output = self.make("packageinfo", returnOutput=True) | |
460 | ||
461 | pkg = {} | |
462 | for line in output.splitlines(): | |
463 | if not line: | |
464 | pkgs.append(pkg) | |
465 | pkg = {} | |
466 | ||
467 | m = re.match(r"^(\w+)=(.*)$", line) | |
468 | if not m: | |
469 | continue | |
470 | ||
471 | k, v = m.groups() | |
472 | pkg[k] = v.strip("\"") | |
473 | ||
3723913b MT |
474 | # Create a dummy repository to link the virtual packages to |
475 | repo = repository.DummyRepository(self.pakfire) | |
476 | ||
47a4cb89 MT |
477 | self._packages = [] |
478 | for pkg in pkgs: | |
fa6d335b | 479 | pkg = packages.VirtualPackage(self.pakfire, pkg) # XXX had to remove repo here?! |
47a4cb89 MT |
480 | self._packages.append(pkg) |
481 | ||
482 | return self._packages | |
483 | ||
484 | def make_requires(self): | |
485 | return self.make_info.get("PKG_BUILD_DEPS", "").split() | |
486 | ||
487 | def make_sources(self): | |
488 | return self.make_info.get("PKG_FILES", "").split() | |
489 | ||
5be98997 MT |
490 | def create_icecream_toolchain(self): |
491 | if not self.settings.get("enable_icecream", None): | |
492 | return | |
493 | ||
494 | out = self.do("icecc --build-native", returnOutput=True) | |
495 | ||
496 | for line in out.splitlines(): | |
497 | m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line) | |
498 | if m: | |
499 | self.settings["icecream_toolchain"] = "/%s" % m.group(1) | |
500 | ||
47a4cb89 | 501 | def build(self): |
5be98997 MT |
502 | self.create_icecream_toolchain() |
503 | ||
47a4cb89 MT |
504 | self.make("build") |
505 | ||
506 | for pkg in reversed(self.packages): | |
507 | packager = packages.Packager(self.pakfire, pkg, self) | |
508 | packager() | |
509 | ||
510 | def dist(self): | |
511 | self.pkg.dist(self) | |
512 | ||
513 | def shell(self, args=[]): | |
514 | # XXX need to add linux32 or linux64 to the command line | |
515 | # XXX need to set CFLAGS here | |
516 | command = "chroot %s /usr/bin/chroot-shell %s" % \ | |
517 | (self.chrootPath(), " ".join(args)) | |
518 | ||
519 | for key, val in self.environ.items(): | |
520 | command = "%s=\"%s\" " % (key, val) + command | |
521 | ||
e360ea59 MT |
522 | # Add personality if we require one |
523 | if self.pakfire.distro.personality: | |
524 | command = "%s %s" % (self.pakfire.disto.personality, command) | |
525 | ||
47a4cb89 | 526 | # Empty the environment |
19979f20 | 527 | #command = "env -i - %s" % command |
47a4cb89 MT |
528 | |
529 | logging.debug("Shell command: %s" % command) | |
530 | ||
531 | try: | |
532 | self._mountall() | |
533 | ||
534 | shell = os.system(command) | |
535 | return os.WEXITSTATUS(shell) | |
536 | ||
537 | finally: | |
538 | self._umountall() |