]> git.ipfire.org Git - people/stevee/pakfire.git/blame - pakfire/builder.py
Initial import.
[people/stevee/pakfire.git] / pakfire / builder.py
CommitLineData
47a4cb89
MT
1#!/usr/bin/python
2
3import fcntl
4import grp
5import logging
6import os
7import re
8import shutil
9import stat
10import time
11
12import depsolve
13import packages
14import transaction
15import util
16
17from constants import *
18from errors import BuildRootLocked
19
20
21class Builder(object):
22 # The version of the kernel this machine is running.
23 kernel_version = os.uname()[2]
24
25 def __init__(self, pakfire, pkg):
26 self.pakfire = pakfire
27 self.pkg = pkg
28
29 self.settings = {
30 "enable_loop_devices" : True,
31 }
32
33 self.buildroot = "/buildroot"
34
35 # Lock the buildroot
36 self._lock = None
37 self.lock()
38
39 # Initialize the environment
40 self.prepare()
41
42 @property
43 def path(self):
44 return self.pakfire.path
45
46 def lock(self):
47 filename = os.path.join(self.path, ".lock")
48
49 try:
50 self._lock = open(filename, "a+")
51 except IOError, e:
52 return 0
53
54 try:
55 fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
56 except IOError, e:
57 raise BuildRootLocked, "Buildroot is locked"
58
59 return 1
60
61 def unlock(self):
62 if self._lock:
63 self._lock.close()
64 self._lock = None
65
66 def copyin(self, file_out, file_in):
67 if file_in.startswith("/"):
68 file_in = file_in[1:]
69
70 file_in = self.chrootPath(file_in)
71
72 #if not os.path.exists(file_out):
73 # return
74
75 dir_in = os.path.dirname(file_in)
76 if not os.path.exists(dir_in):
77 os.makedirs(dir_in)
78
79 logging.debug("%s --> %s" % (file_out, file_in))
80
81 shutil.copy2(file_out, file_in)
82
83 def copyout(self, file_in, file_out):
84 if file_in.startswith("/"):
85 file_in = file_in[1:]
86
87 file_in = self.chrootPath(file_in)
88
89 #if not os.path.exists(file_in):
90 # return
91
92 dir_out = os.path.dirname(file_out)
93 if not os.path.exists(dir_out):
94 os.makedirs(dir_out)
95
96 logging.debug("%s --> %s" % (file_in, file_out))
97
98 shutil.copy2(file_in, file_out)
99
100 def copy_result(self, resultdir):
101 dir_in = self.chrootPath("result")
102
103 for dir, subdirs, files in os.walk(dir_in):
104 basename = os.path.basename(dir)
105 dir = dir[len(self.chrootPath()):]
106 for file in files:
107 file_in = os.path.join(dir, file)
108
109 file_out = os.path.join(
110 resultdir,
111 basename,
112 file,
113 )
114
115 self.copyout(file_in, file_out)
116
117 def extract(self, requires=[], build_deps=True):
118 """
119 Gets a dependency set and extracts all packages
120 to the environment.
121 """
122 ds = depsolve.DependencySet(self.pakfire)
123 for p in BUILD_PACKAGES + requires:
124 ds.add_requires(p)
125 ds.resolve()
126
127 # Get build dependencies from source package.
128 if isinstance(self.pkg, packages.SourcePackage):
129 for req in self.pkg.requires:
130 ds.add_requires(req)
131
132 ts = transaction.TransactionSet(self.pakfire, ds)
133 ts.dump()
134 ts.run()
135
136 # Copy the makefile and load source tarballs.
137 if isinstance(self.pkg, packages.Makefile):
138 self.pkg.extract(self)
139
140 # If we have a makefile, we can only get the build dependencies
141 # after we have extracted all the rest.
142 if build_deps and isinstance(self.pkg, packages.Makefile):
143 requires = self.make_requires()
144 if not requires:
145 return
146
147 ds = depsolve.DependencySet(self.pakfire)
148 for r in requires:
149 ds.add_requires(r)
150 ds.resolve()
151
152 ts = transaction.TransactionSet(self.pakfire, ds)
153 ts.dump()
154 ts.run()
155
156 @property
157 def log(self):
158 # XXX for now, return the root logger
159 return logging.getLogger()
160
161 def chrootPath(self, *args):
162 # Remove all leading slashes
163 _args = []
164 for arg in args:
165 if arg.startswith("/"):
166 arg = arg[1:]
167 _args.append(arg)
168 args = _args
169
170 ret = os.path.join(self.path, *args)
171 ret = ret.replace("//", "/")
172
173 assert ret.startswith(self.path)
174
175 return ret
176
177 def prepare(self):
178 # Create directory.
179 if not os.path.exists(self.path):
180 os.makedirs(self.path)
181
182 # Create important directories.
183 dirs = [
184 "build",
185 self.buildroot,
186 "dev",
187 "dev/pts",
188 "dev/shm",
189 "etc",
190 "proc",
191 "result",
192 "sys",
193 "tmp",
194 "usr/src",
195 ]
196 for dir in dirs:
197 dir = self.chrootPath(dir)
198 if not os.path.exists(dir):
199 os.makedirs(dir)
200
201 self._prepare_dev()
202 self._prepare_users()
203 self._prepare_dns()
204
205 def _prepare_dev(self):
206 prevMask = os.umask(0000)
207
208 nodes = [
209 ("dev/null", stat.S_IFCHR | 0666, os.makedev(1, 3)),
210 ("dev/full", stat.S_IFCHR | 0666, os.makedev(1, 7)),
211 ("dev/zero", stat.S_IFCHR | 0666, os.makedev(1, 5)),
212 ("dev/random", stat.S_IFCHR | 0666, os.makedev(1, 8)),
213 ("dev/urandom", stat.S_IFCHR | 0444, os.makedev(1, 9)),
214 ("dev/tty", stat.S_IFCHR | 0666, os.makedev(5, 0)),
215 ("dev/console", stat.S_IFCHR | 0600, os.makedev(5, 1)),
216 ]
217
218 # If we need loop devices (which are optional) we create them here.
219 if self.settings["enable_loop_devices"]:
220 for i in range(0, 7):
221 nodes.append(("dev/loop%d" % i, stat.S_IFBLK | 0660, os.makedev(7, i)))
222
223 # Create all the nodes.
224 for node in nodes:
225 self._create_node(*node)
226
227 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
228 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
229 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
230 os.symlink("/proc/self/fd", self.chrootPath("dev", "fd"))
231
232 # make device node for el4 and el5
233 if self.kernel_version < "2.6.19":
234 self._make_node("dev/ptmx", stat.S_IFCHR | 0666, os.makedev(5, 2))
235 else:
236 os.symlink("/dev/pts/ptmx", self.chrootPath("dev", "ptmx"))
237
238 os.umask(prevMask)
239
240 def _prepare_users(self):
241 f = open(self.chrootPath("etc", "passwd"), "w")
242 f.write("root:x:0:0:root:/root:/bin/bash\n")
243 f.write("nobody:x:99:99:Nobody:/:/sbin/nologin\n")
244 f.close()
245
246 f = open(self.chrootPath("etc", "group"), "w")
247 f.write("root:x:0:root\n")
248 f.write("nobody:x:99:\n")
249 f.close()
250
251 def _prepare_dns(self):
252 # XXX to be replaced
253 # maybe we can copyin /etc/resolv.conf and /etc/hosts
254 nameservers = []
255 f = open("/etc/resolv.conf")
256 for line in f.readlines():
257 if line.startswith("nameserver"):
258 nameservers.append(line.split(" ")[-1].strip())
259 f.close()
260
261 logging.debug("Using nameservers: %s" % nameservers)
262
263 f = open(self.chrootPath("etc", "resolv.conf"), "w")
264 for nameserver in nameservers:
265 f.write("nameserver %s" % nameserver)
266 f.close()
267
268 logging.debug("Creating record for localhost")
269 f = open(self.chrootPath("etc", "hosts"), "w")
270 f.write("127.0.0.1 localhost\n")
271 f.close()
272
273 def _create_node(self, filename, mode, device):
274 logging.debug("Create node: %s (%s)" % (filename, mode))
275
276 filename = self.chrootPath(filename)
277
278 # Create parent directory if it is missing.
279 dirname = os.path.dirname(filename)
280 if not os.path.exists(dirname):
281 os.makedirs(dirname)
282
283 os.mknod(filename, mode, device)
284
285 def cleanup(self):
286 logging.debug("Cleanup environment %s" % self.path)
287
288 if os.path.exists(self.path):
289 util.rm(self.path)
290
291 def _mountall(self):
292 self.log.debug("Mounting environment")
293 for cmd, mountpoint in self.mountpoints:
294 cmd = "%s %s" % (cmd, self.chrootPath(mountpoint))
295 util.do(cmd, shell=True)
296
297 def _umountall(self):
298 self.log.debug("Umounting environment")
299 for cmd, mountpoint in self.mountpoints:
300 cmd = "umount -n %s" % self.chrootPath(mountpoint)
301 util.do(cmd, raiseExc=0, shell=True)
302
303 @property
304 def mountpoints(self):
305 ret = [
306 ("mount -n -t proc pakfire_chroot_proc", "proc"),
307 ("mount -n -t sysfs pakfire_chroot_sysfs", "sys"),
308 ]
309
310 mountopt = "gid=%d,mode=0620,ptmxmode=0666" % grp.getgrnam("tty").gr_gid
311 if self.kernel_version >= "2.6.29":
312 mountopt += ",newinstance"
313
314 ret.extend([
315 ("mount -n -t devpts -o %s pakfire_chroot_devpts" % mountopt, "dev/pts"),
316 ("mount -n -t tmpfs pakfire_chroot_shmfs", "dev/shm"),
317 ])
318
319 return ret
320
321 @property
322 def environ(self):
323 env = {
324 "BUILDROOT" : self.buildroot,
325
326 # XXX I want to get rid of these too and invoke a login shell
327 "HOME" : "/root",
328 "PATH" : "/sbin:/bin:/usr/sbin:/usr/bin",
329 }
330
331 # Inherit environment from distro
332 env.update(self.pakfire.distro.environ)
333
334 # XXX what do we need else?
335
336 return env
337
338 def do(self, command, shell=True, personality=None, *args, **kwargs):
339 ret = None
340 try:
341 # Environment variables
342 env = self.environ
343
344 if kwargs.has_key("env"):
345 env.update(kwargs.pop("env"))
346
347 self._mountall()
348
349 if not kwargs.has_key("chrootPath"):
350 kwargs["chrootPath"] = self.chrootPath()
351
352 ret = util.do(
353 command,
354 personality=personality,
355 shell=shell,
356 env=env,
357 logger=self.log,
358 *args,
359 **kwargs
360 )
361
362 finally:
363 self._umountall()
364
365 return ret
366
367 def make(self, *args, **kwargs):
368 # XXX need to get rid of this
369 env = { "PKGROOT" : "/usr/lib/buildsystem" }
370 try:
371 kwargs["env"].update(env)
372 except KeyError:
373 kwargs["env"] = env
374
375 return self.do("make -f /build/%s %s" % \
376 (os.path.basename(self.pkg.filename), " ".join(args)),
377 shell=True, **kwargs)
378
379 @property
380 def make_info(self):
381 if not hasattr(self, "_make_info"):
382 info = {}
383
384 output = self.make("buildinfo", returnOutput=True)
385
386 for line in output.splitlines():
387 # XXX temporarily
388 if not line:
389 break
390
391 m = re.match(r"^(\w+)=(.*)$", line)
392 if not m:
393 continue
394
395 info[m.group(1)] = m.group(2).strip("\"")
396
397 self._make_info = info
398
399 return self._make_info
400
401 @property
402 def packages(self):
403 if hasattr(self, "_packages"):
404 return self._packages
405
406 pkgs = []
407 output = self.make("packageinfo", returnOutput=True)
408
409 pkg = {}
410 for line in output.splitlines():
411 if not line:
412 pkgs.append(pkg)
413 pkg = {}
414
415 m = re.match(r"^(\w+)=(.*)$", line)
416 if not m:
417 continue
418
419 k, v = m.groups()
420 pkg[k] = v.strip("\"")
421
422 self._packages = []
423 for pkg in pkgs:
424 pkg = packages.VirtualPackage(pkg)
425 self._packages.append(pkg)
426
427 return self._packages
428
429 def make_requires(self):
430 return self.make_info.get("PKG_BUILD_DEPS", "").split()
431
432 def make_sources(self):
433 return self.make_info.get("PKG_FILES", "").split()
434
435 def build(self):
436 self.make("build")
437
438 for pkg in reversed(self.packages):
439 packager = packages.Packager(self.pakfire, pkg, self)
440 packager()
441
442 def dist(self):
443 self.pkg.dist(self)
444
445 def shell(self, args=[]):
446 # XXX need to add linux32 or linux64 to the command line
447 # XXX need to set CFLAGS here
448 command = "chroot %s /usr/bin/chroot-shell %s" % \
449 (self.chrootPath(), " ".join(args))
450
451 for key, val in self.environ.items():
452 command = "%s=\"%s\" " % (key, val) + command
453
454 # Empty the environment
455 command = "env -i - %s" % command
456
457 logging.debug("Shell command: %s" % command)
458
459 try:
460 self._mountall()
461
462 shell = os.system(command)
463 return os.WEXITSTATUS(shell)
464
465 finally:
466 self._umountall()