pakfire/builder.py

   1 #!/usr/bin/python
   2
   3 import fcntl
   4 import grp
   5 import logging
   6 import math
   7 import os
   8 import re
   9 import shutil
  10 import socket
  11 import stat
  12 import time
  13 import uuid
  14
  15 import depsolve
  16 import packages
  17 import repository
  18 import transaction
  19 import util
  20
  21 from constants import *
  22 from i18n import _
  23 from errors import BuildError, BuildRootLocked, Error
  24
  25
  26 class Builder(object):
  27         # The version of the kernel this machine is running.
  28         kernel_version = os.uname()[2]
  29
  30         def __init__(self, pakfire, pkg, build_id=None, **settings):
  31                 self.pakfire = pakfire
  32                 self.pkg = pkg
  33
  34                 self.settings = {
  35                         "enable_loop_devices" : True,
  36                         "enable_ccache"   : True,
  37                         "enable_icecream" : False,
  38                 }
  39                 self.settings.update(settings)
  40
  41                 self.buildroot = "/buildroot"
  42
  43                 # Lock the buildroot
  44                 self._lock = None
  45                 self.lock()
  46
  47                 # Save the build time.
  48                 self.build_time = int(time.time())
  49
  50                 # Save the build id and generate one if no build id was provided.
  51                 if not build_id:
  52                         build_id = uuid.uuid4()
  53
  54                 self.build_id = build_id
  55
  56         @property
  57         def info(self):
  58                 return {
  59                         "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
  60                         "build_host" : socket.gethostname(),
  61                         "build_id"   : self.build_id,
  62                         "build_time" : self.build_time,
  63                 }
  64
  65         @property
  66         def path(self):
  67                 return self.pakfire.path
  68
  69         def lock(self):
  70                 filename = os.path.join(self.path, ".lock")
  71
  72                 try:
  73                         self._lock = open(filename, "a+")
  74                 except IOError, e:
  75                         return 0
  76
  77                 try:
  78                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
  79                 except IOError, e:
  80                         raise BuildRootLocked, "Buildroot is locked"
  81
  82                 return 1
  83
  84         def unlock(self):
  85                 if self._lock:
  86                         self._lock.close()
  87                         self._lock = None
  88
  89         def copyin(self, file_out, file_in):
  90                 if file_in.startswith("/"):
  91                         file_in = file_in[1:]
  92
  93                 file_in = self.chrootPath(file_in)
  94
  95                 #if not os.path.exists(file_out):
  96                 #       return
  97
  98                 dir_in = os.path.dirname(file_in)
  99                 if not os.path.exists(dir_in):
 100                         os.makedirs(dir_in)
 101
 102                 logging.debug("%s --> %s" % (file_out, file_in))
 103
 104                 shutil.copy2(file_out, file_in)
 105
 106         def copyout(self, file_in, file_out):
 107                 if file_in.startswith("/"):
 108                         file_in = file_in[1:]
 109
 110                 file_in = self.chrootPath(file_in)
 111
 112                 #if not os.path.exists(file_in):
 113                 #       return
 114
 115                 dir_out = os.path.dirname(file_out)
 116                 if not os.path.exists(dir_out):
 117                         os.makedirs(dir_out)
 118
 119                 logging.debug("%s --> %s" % (file_in, file_out))
 120
 121                 shutil.copy2(file_in, file_out)
 122
 123         def copy_result(self, resultdir):
 124                 dir_in = self.chrootPath("result")
 125
 126                 for dir, subdirs, files in os.walk(dir_in):
 127                         basename = os.path.basename(dir)
 128                         dir = dir[len(self.chrootPath()):]
 129                         for file in files:
 130                                 file_in = os.path.join(dir, file)
 131
 132                                 file_out = os.path.join(
 133                                         resultdir,
 134                                         basename,
 135                                         file,
 136                                 )
 137
 138                                 self.copyout(file_in, file_out)
 139
 140         def extract(self, requires=None, build_deps=True):
 141                 """
 142                         Gets a dependency set and extracts all packages
 143                         to the environment.
 144                 """
 145                 if not requires:
 146                         requires = []
 147
 148                 # Add neccessary build dependencies.
 149                 requires += BUILD_PACKAGES
 150
 151                 # If we have ccache enabled, we need to extract it
 152                 # to the build chroot.
 153                 if self.settings.get("enable_ccache"):
 154                         requires.append("ccache")
 155
 156                 # If we have icecream enabled, we need to extract it
 157                 # to the build chroot.
 158                 if self.settings.get("enable_icecream"):
 159                         requires.append("icecream")
 160
 161                 # Get build dependencies from source package.
 162                 if isinstance(self.pkg, packages.SourcePackage):
 163                         for req in self.pkg.requires:
 164                                 requires.append(req)
 165
 166                 # Install all packages.
 167                 self.install(requires)
 168
 169                 # Copy the makefile and load source tarballs.
 170                 if isinstance(self.pkg, packages.Makefile):
 171                         self.pkg.extract(self)
 172
 173                 elif isinstance(self.pkg, packages.SourcePackage):
 174                         self.pkg.extract(_("Extracting: %s (source)") % self.pkg.name,
 175                                 prefix=os.path.join(self.path, "build"))
 176
 177                 # If we have a makefile, we can only get the build dependencies
 178                 # after we have extracted all the rest.
 179                 if build_deps and isinstance(self.pkg, packages.Makefile):
 180                         requires = self.make_requires()
 181                         if not requires:
 182                                 return
 183
 184                         self.install(requires)
 185
 186         def install(self, requires):
 187                 """
 188                         Install everything that is required in requires.
 189                 """
 190                 # If we got nothing to do, we quit immediately.
 191                 if not requires:
 192                         return
 193
 194                 ds = depsolve.DependencySet(self.pakfire)
 195                 for r in requires:
 196                         if isinstance(r, packages.BinaryPackage):
 197                                 ds.add_package(r)
 198                         else:
 199                                 ds.add_requires(r)
 200                 ds.resolve()
 201                 ds.dump()
 202
 203                 ts = transaction.Transaction(self.pakfire, ds)
 204                 ts.run()
 205
 206         def install_test(self):
 207                 pkgs = []
 208
 209                 # Connect packages to the FS repository.
 210                 r = repository.FileSystemRepository(self.pakfire)
 211
 212                 for dir, subdirs, files in os.walk(self.chrootPath("result")):
 213                         for file in files:
 214                                 file = os.path.join(dir, file)
 215
 216                                 if not file.endswith(PACKAGE_EXTENSION):
 217                                         continue
 218
 219                                 p = packages.BinaryPackage(self.pakfire, r, file)
 220                                 pkgs.append(p)
 221
 222                 self.install(pkgs)
 223
 224         @property
 225         def log(self):
 226                 # XXX for now, return the root logger
 227                 return logging.getLogger()
 228
 229         def chrootPath(self, *args):
 230                 # Remove all leading slashes
 231                 _args = []
 232                 for arg in args:
 233                         if arg.startswith("/"):
 234                                 arg = arg[1:]
 235                         _args.append(arg)
 236                 args = _args
 237
 238                 ret = os.path.join(self.path, *args)
 239                 ret = ret.replace("//", "/")
 240
 241                 assert ret.startswith(self.path)
 242
 243                 return ret
 244
 245         def prepare(self):
 246                 # Create directory.
 247                 if not os.path.exists(self.path):
 248                         os.makedirs(self.path)
 249
 250                 # Create important directories.
 251                 dirs = [
 252                         "build",
 253                         self.buildroot,
 254                         "dev",
 255                         "dev/pts",
 256                         "dev/shm",
 257                         "etc",
 258                         "proc",
 259                         "result",
 260                         "sys",
 261                         "tmp",
 262                         "usr/src",
 263                 ]
 264
 265                 # Create cache dir if ccache is enabled.
 266                 if self.settings.get("enable_ccache"):
 267                         dirs.append("var/cache/ccache")
 268
 269                         if not os.path.exists(CCACHE_CACHE_DIR):
 270                                 os.makedirs(CCACHE_CACHE_DIR)
 271
 272                 for dir in dirs:
 273                         dir = self.chrootPath(dir)
 274                         if not os.path.exists(dir):
 275                                 os.makedirs(dir)
 276
 277                 # Create neccessary files like /etc/fstab and /etc/mtab.
 278                 files = (
 279                         "etc/fstab",
 280                         "etc/mtab"
 281                 )
 282
 283                 for file in files:
 284                         file = self.chrootPath(file)
 285                         dir = os.path.dirname(file)
 286                         if not os.path.exists(dir):
 287                                 os.makedirs(dir)
 288                         f = open(file, "w")
 289                         f.close()
 290
 291                 self._prepare_dev()
 292                 self._prepare_dns()
 293
 294         def _prepare_dev(self):
 295                 prevMask = os.umask(0000)
 296
 297                 nodes = [
 298                         ("dev/null",    stat.S_IFCHR | 0666, os.makedev(1, 3)),
 299                         ("dev/full",    stat.S_IFCHR | 0666, os.makedev(1, 7)),
 300                         ("dev/zero",    stat.S_IFCHR | 0666, os.makedev(1, 5)),
 301                         ("dev/random",  stat.S_IFCHR | 0666, os.makedev(1, 8)),
 302                         ("dev/urandom", stat.S_IFCHR | 0444, os.makedev(1, 9)),
 303                         ("dev/tty",             stat.S_IFCHR | 0666, os.makedev(5, 0)),
 304                         ("dev/console", stat.S_IFCHR | 0600, os.makedev(5, 1)),
 305                 ]
 306
 307                 # If we need loop devices (which are optional) we create them here.
 308                 if self.settings["enable_loop_devices"]:
 309                         for i in range(0, 7):
 310                                 nodes.append(("dev/loop%d" % i, stat.S_IFBLK | 0660, os.makedev(7, i)))
 311
 312                 # Create all the nodes.
 313                 for node in nodes:
 314                         self._create_node(*node)
 315
 316                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 317                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 318                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 319                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 320
 321                 # make device node for el4 and el5
 322                 if self.kernel_version < "2.6.19":
 323                         self._make_node("dev/ptmx", stat.S_IFCHR | 0666, os.makedev(5, 2))
 324                 else:
 325                         os.symlink("/dev/pts/ptmx", self.chrootPath("dev", "ptmx"))
 326
 327                 os.umask(prevMask)
 328
 329         def _prepare_dns(self):
 330                 """
 331                         Add DNS resolution facility to chroot environment by copying
 332                         /etc/resolv.conf and /etc/hosts.
 333                 """
 334                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 335                         self.copyin(i, i)
 336
 337         def _create_node(self, filename, mode, device):
 338                 logging.debug("Create node: %s (%s)" % (filename, mode))
 339
 340                 filename = self.chrootPath(filename)
 341
 342                 # Create parent directory if it is missing.
 343                 dirname = os.path.dirname(filename)
 344                 if not os.path.exists(dirname):
 345                         os.makedirs(dirname)
 346
 347                 os.mknod(filename, mode, device)
 348
 349         def destroy(self):
 350                 logging.debug("Destroying environment %s" % self.path)
 351
 352                 if os.path.exists(self.path):
 353                         util.rm(self.path)
 354
 355         def cleanup(self):
 356                 logging.debug("Cleaning environemnt.")
 357
 358                 # Run make clean and let it cleanup its stuff.
 359                 self.make("clean")
 360
 361                 # Remove the build directory and buildroot.
 362                 dirs = ("build", self.buildroot, "result")
 363
 364                 for d in dirs:
 365                         d = self.chrootPath(d)
 366                         if not os.path.exists(d):
 367                                 continue
 368
 369                         util.rm(d)
 370                         os.makedirs(d)
 371
 372                 # Clear make_info cache.
 373                 if hasattr(self, "_make_info"):
 374                         del self._make_info
 375
 376         def _mountall(self):
 377                 self.log.debug("Mounting environment")
 378                 for cmd, mountpoint in self.mountpoints:
 379                         cmd = "%s %s" % (cmd, self.chrootPath(mountpoint))
 380                         util.do(cmd, shell=True)
 381
 382         def _umountall(self):
 383                 self.log.debug("Umounting environment")
 384                 for cmd, mountpoint in self.mountpoints:
 385                         cmd = "umount -n %s" % self.chrootPath(mountpoint)
 386                         util.do(cmd, raiseExc=0, shell=True)
 387
 388         @property
 389         def mountpoints(self):
 390                 ret = [
 391                         ("mount -n -t proc pakfire_chroot_proc", "proc"),
 392                         ("mount -n -t sysfs pakfire_chroot_sysfs", "sys"),
 393                 ]
 394
 395                 mountopt = "gid=%d,mode=0620,ptmxmode=0666" % grp.getgrnam("tty").gr_gid
 396                 if self.kernel_version >= "2.6.29":
 397                         mountopt += ",newinstance"
 398
 399                 ret.extend([
 400                         ("mount -n -t devpts -o %s pakfire_chroot_devpts" % mountopt, "dev/pts"),
 401                         ("mount -n -t tmpfs pakfire_chroot_shmfs", "dev/shm"),
 402                 ])
 403
 404                 if self.settings.get("enable_ccache"):
 405                         ret.append(("mount -n --bind %s" % CCACHE_CACHE_DIR, "var/cache/ccache"))
 406
 407                 return ret
 408
 409         @staticmethod
 410         def calc_parallelism():
 411                 """
 412                         Calculate how many processes to run
 413                         at the same time.
 414
 415                         We take the log10(number of processors) * factor
 416                 """
 417                 num = os.sysconf("SC_NPROCESSORS_CONF")
 418                 if num == 1:
 419                         return 2
 420                 else:
 421                         return int(round(math.log10(num) * 26))
 422
 423         @property
 424         def environ(self):
 425                 env = {
 426                         # Add HOME manually, because it is occasionally not set
 427                         # and some builds get in trouble then.
 428                         "HOME" : "/root",
 429                         "TERM" : os.environ.get("TERM", "dumb"),
 430                         "PS1"  : "\u:\w\$ ",
 431
 432                         "BUILDROOT" : self.buildroot,
 433                         "PARALLELISMFLAGS" : "-j%s" % self.calc_parallelism(),
 434                 }
 435
 436                 # Inherit environment from distro
 437                 env.update(self.pakfire.distro.environ)
 438
 439                 # Icecream environment settings
 440                 if self.settings.get("enable_icecream", None):
 441                         # Set the toolchain path
 442                         if self.settings.get("icecream_toolchain", None):
 443                                 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
 444
 445                         # Set preferred host if configured.
 446                         if self.settings.get("icecream_preferred_host", None):
 447                                 env["ICECC_PREFERRED_HOST"] = \
 448                                         self.settings.get("icecream_preferred_host")
 449
 450                 # XXX what do we need else?
 451
 452                 return env
 453
 454         def do(self, command, shell=True, personality=None, *args, **kwargs):
 455                 ret = None
 456                 try:
 457                         # Environment variables
 458                         env = self.environ
 459
 460                         if kwargs.has_key("env"):
 461                                 env.update(kwargs.pop("env"))
 462
 463                         logging.debug("Environment:")
 464                         for k, v in sorted(env.items()):
 465                                 logging.debug("  %s=%s" % (k, v))
 466
 467                         # Update personality it none was set
 468                         if not personality:
 469                                 personality = self.pakfire.distro.personality
 470
 471                         # Make every shell to a login shell because we set a lot of
 472                         # environment things there.
 473                         if shell:
 474                                 command = ["bash", "--login", "-c", command]
 475
 476                         self._mountall()
 477
 478                         if not kwargs.has_key("chrootPath"):
 479                                 kwargs["chrootPath"] = self.chrootPath()
 480
 481                         ret = util.do(
 482                                 command,
 483                                 personality=personality,
 484                                 shell=False,
 485                                 env=env,
 486                                 logger=self.log,
 487                                 *args,
 488                                 **kwargs
 489                         )
 490
 491                 finally:
 492                         self._umountall()
 493
 494                 return ret
 495
 496         def make(self, *args, **kwargs):
 497                 if isinstance(self.pkg, packages.Makefile):
 498                         filename = os.path.basename(self.pkg.filename)
 499                 elif isinstance(self.pkg, packages.SourcePackage):
 500                         filename = "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION)
 501
 502                 return self.do("make -f /build/%s %s" % (filename, " ".join(args)),
 503                         **kwargs)
 504
 505         @property
 506         def make_info(self):
 507                 if not hasattr(self, "_make_info"):
 508                         info = {}
 509
 510                         output = self.make("buildinfo", returnOutput=True)
 511
 512                         for line in output.splitlines():
 513                                 # XXX temporarily
 514                                 if not line:
 515                                         break
 516
 517                                 m = re.match(r"^(\w+)=(.*)$", line)
 518                                 if not m:
 519                                         continue
 520
 521                                 info[m.group(1)] = m.group(2).strip("\"")
 522
 523                         self._make_info = info
 524
 525                 return self._make_info
 526
 527         @property
 528         def packages(self):
 529                 if hasattr(self, "_packages"):
 530                         return self._packages
 531
 532                 pkgs = []
 533                 output = self.make("packageinfo", returnOutput=True)
 534
 535                 pkg = {}
 536                 for line in output.splitlines():
 537                         if not line:
 538                                 pkgs.append(pkg)
 539                                 pkg = {}
 540
 541                         m = re.match(r"^(\w+)=(.*)$", line)
 542                         if not m:
 543                                 continue
 544
 545                         k, v = m.groups()
 546                         pkg[k] = v.strip("\"")
 547
 548                 # Create a dummy repository to link the virtual packages to
 549                 repo = repository.DummyRepository(self.pakfire)
 550
 551                 self._packages = []
 552                 for pkg in pkgs:
 553                         pkg = packages.VirtualPackage(self.pakfire, pkg) # XXX had to remove repo here?!
 554                         self._packages.append(pkg)
 555
 556                 return self._packages
 557
 558         def make_requires(self):
 559                 return self.make_info.get("PKG_BUILD_DEPS", "").split()
 560
 561         def make_sources(self):
 562                 return self.make_info.get("PKG_FILES", "").split()
 563
 564         def create_icecream_toolchain(self):
 565                 if not self.settings.get("enable_icecream", None):
 566                         return
 567
 568                 out = self.do("icecc --build-native", returnOutput=True)
 569
 570                 for line in out.splitlines():
 571                         m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
 572                         if m:
 573                                 self.settings["icecream_toolchain"] = "/%s" % m.group(1)
 574
 575         def build(self):
 576                 self.create_icecream_toolchain()
 577
 578                 try:
 579                         self.make("build")
 580
 581                 except Error:
 582                         raise BuildError, "The build command failed."
 583
 584                 for pkg in reversed(self.packages):
 585                         packager = packages.BinaryPackager(self.pakfire, pkg, self)
 586                         packager()
 587
 588         def dist(self):
 589                 self.pkg.dist(self)
 590
 591         def shell(self, args=[]):
 592                 if not util.cli_is_interactive():
 593                         logging.warning("Cannot run shell on non-interactive console.")
 594                         return
 595
 596                 # Install all packages that are needed to run a shell.
 597                 self.install(SHELL_PACKAGES)
 598
 599                 # XXX need to set CFLAGS here
 600                 command = "/usr/sbin/chroot %s /usr/bin/chroot-shell %s" % \
 601                         (self.chrootPath(), " ".join(args))
 602
 603                 # Add personality if we require one
 604                 if self.pakfire.distro.personality:
 605                         command = "%s %s" % (self.pakfire.distro.personality, command)
 606
 607                 for key, val in self.environ.items():
 608                         command = "%s=\"%s\" " % (key, val) + command
 609
 610                 # Empty the environment
 611                 command = "env -i - %s" % command
 612
 613                 logging.debug("Shell command: %s" % command)
 614
 615                 try:
 616                         self._mountall()
 617
 618                         shell = os.system(command)
 619                         return os.WEXITSTATUS(shell)
 620
 621                 finally:
 622                         self._umountall()