pakfire/builder.py

   1 #!/usr/bin/python
   2
   3 import fcntl
   4 import grp
   5 import logging
   6 import math
   7 import os
   8 import re
   9 import shutil
  10 import socket
  11 import stat
  12 import time
  13 import uuid
  14
  15 import base
  16 import chroot
  17 import logger
  18 import packages
  19 import repository
  20 import util
  21
  22 from constants import *
  23 from i18n import _
  24 from errors import BuildError, BuildRootLocked, Error
  25
  26
  27 BUILD_LOG_HEADER = """
  28  ____       _     __ _            _           _ _     _
  29 |  _ \ __ _| | __/ _(_)_ __ ___  | |__  _   _(_) | __| | ___ _ __
  30 | |_) / _` | |/ / |_| | '__/ _ \ | '_ \| | | | | |/ _` |/ _ \ '__|
  31 |  __/ (_| |   <|  _| | | |  __/ | |_) | |_| | | | (_| |  __/ |
  32 |_|   \__,_|_|\_\_| |_|_|  \___| |_.__/ \__,_|_|_|\__,_|\___|_|
  33
  34         Time    : %(time)s
  35         Host    : %(host)s
  36         Version : %(version)s
  37
  38 """
  39
  40 class Builder(object):
  41         # The version of the kernel this machine is running.
  42         kernel_version = os.uname()[2]
  43
  44         def __init__(self, pkg=None, distro_config=None, build_id=None, logfile=None,
  45                         builder_mode="release", **pakfire_args):
  46                 # Set mode.
  47                 assert builder_mode in ("development", "release",)
  48                 self.mode = builder_mode
  49
  50                 # Disable the build repository in release mode.
  51                 if self.mode == "release":
  52                         if pakfire_args.has_key("disable_repos") and pakfire_args["disable_repos"]:
  53                                 pakfire_args["disable_repos"] += ["build",]
  54                         else:
  55                                 pakfire_args["disable_repos"] = ["build",]
  56
  57                 # Save the build id and generate one if no build id was provided.
  58                 if not build_id:
  59                         build_id = "%s" % uuid.uuid4()
  60
  61                 self.build_id = build_id
  62
  63                 # Setup the logging.
  64                 if logfile:
  65                         self.log = logging.getLogger(self.build_id)
  66                         # Propage everything to the root logger that we will see something
  67                         # on the terminal.
  68                         self.log.propagate = 1
  69                         self.log.setLevel(logging.INFO)
  70
  71                         # Add the given logfile to the logger.
  72                         h = logging.FileHandler(logfile)
  73                         self.log.addHandler(h)
  74
  75                         # Format the log output for the file.
  76                         f = logger.BuildFormatter()
  77                         h.setFormatter(f)
  78                 else:
  79                         # If no logile was given, we use the root logger.
  80                         self.log = logging.getLogger()
  81
  82                 logdata = {
  83                         "host"    : socket.gethostname(),
  84                         "time"    : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),
  85                         "version" : "Pakfire %s" % PAKFIRE_VERSION,
  86                 }
  87                 for line in BUILD_LOG_HEADER.splitlines():
  88                         self.log.info(line % logdata)
  89
  90                 # Create pakfire instance.
  91                 if pakfire_args.has_key("mode"):
  92                         del pakfire_args["mode"]
  93                 self.pakfire = base.Pakfire(mode="builder", distro_config=distro_config, **pakfire_args)
  94                 self.distro = self.pakfire.distro
  95                 self.path = self.pakfire.path
  96
  97                 # Open the package.
  98                 self.pkg = pkg
  99
 100                 # XXX need to make this configureable
 101                 self.settings = {
 102                         "enable_loop_devices" : True,
 103                         "enable_ccache"   : True,
 104                         "enable_icecream" : False,
 105                 }
 106                 #self.settings.update(settings)
 107
 108                 self.buildroot = "/buildroot"
 109
 110                 # Lock the buildroot
 111                 self._lock = None
 112                 self.lock()
 113
 114                 # Save the build time.
 115                 self.build_time = int(time.time())
 116
 117         def get_pkg(self):
 118                 return getattr(self, "_pkg", None)
 119
 120         def set_pkg(self, pkg):
 121                 if pkg is None:
 122                         self.__pkg = None
 123                         return
 124
 125                 self._pkg = packages.open(self.pakfire, None, pkg)
 126
 127                 # Log the package information.
 128                 if not isinstance(self._pkg, packages.Makefile):
 129                         self.log.info("Package information:")
 130                         for line in self._pkg.dump(long=True).splitlines():
 131                                 self.log.info("  %s" % line)
 132                         self.log.info("")
 133
 134                 assert self.pkg
 135
 136         pkg = property(get_pkg, set_pkg)
 137
 138         @property
 139         def arch(self):
 140                 """
 141                         Inherit architecture from distribution configuration.
 142                 """
 143                 return self.distro.arch
 144
 145         @property
 146         def info(self):
 147                 return {
 148                         "build_date" : time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(self.build_time)),
 149                         "build_host" : socket.gethostname(),
 150                         "build_id"   : self.build_id,
 151                         "build_time" : self.build_time,
 152                 }
 153
 154         def lock(self):
 155                 filename = os.path.join(self.path, ".lock")
 156
 157                 try:
 158                         self._lock = open(filename, "a+")
 159                 except IOError, e:
 160                         return 0
 161
 162                 try:
 163                         fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
 164                 except IOError, e:
 165                         raise BuildRootLocked, "Buildroot is locked"
 166
 167                 return 1
 168
 169         def unlock(self):
 170                 if self._lock:
 171                         self._lock.close()
 172                         self._lock = None
 173
 174         def copyin(self, file_out, file_in):
 175                 if file_in.startswith("/"):
 176                         file_in = file_in[1:]
 177
 178                 file_in = self.chrootPath(file_in)
 179
 180                 #if not os.path.exists(file_out):
 181                 #       return
 182
 183                 dir_in = os.path.dirname(file_in)
 184                 if not os.path.exists(dir_in):
 185                         os.makedirs(dir_in)
 186
 187                 logging.debug("%s --> %s" % (file_out, file_in))
 188
 189                 shutil.copy2(file_out, file_in)
 190
 191         def copyout(self, file_in, file_out):
 192                 if file_in.startswith("/"):
 193                         file_in = file_in[1:]
 194
 195                 file_in = self.chrootPath(file_in)
 196
 197                 #if not os.path.exists(file_in):
 198                 #       return
 199
 200                 dir_out = os.path.dirname(file_out)
 201                 if not os.path.exists(dir_out):
 202                         os.makedirs(dir_out)
 203
 204                 logging.debug("%s --> %s" % (file_in, file_out))
 205
 206                 shutil.copy2(file_in, file_out)
 207
 208         def copy_result(self, resultdir):
 209                 dir_in = self.chrootPath("result")
 210
 211                 for dir, subdirs, files in os.walk(dir_in):
 212                         basename = os.path.basename(dir)
 213                         dir = dir[len(self.chrootPath()):]
 214                         for file in files:
 215                                 file_in = os.path.join(dir, file)
 216
 217                                 file_out = os.path.join(
 218                                         resultdir,
 219                                         basename,
 220                                         file,
 221                                 )
 222
 223                                 self.copyout(file_in, file_out)
 224
 225         def extract(self, requires=None, build_deps=True):
 226                 """
 227                         Gets a dependency set and extracts all packages
 228                         to the environment.
 229                 """
 230                 if not requires:
 231                         requires = []
 232
 233                 # Add neccessary build dependencies.
 234                 requires += BUILD_PACKAGES
 235
 236                 # If we have ccache enabled, we need to extract it
 237                 # to the build chroot.
 238                 if self.settings.get("enable_ccache"):
 239                         requires.append("ccache")
 240
 241                 # If we have icecream enabled, we need to extract it
 242                 # to the build chroot.
 243                 if self.settings.get("enable_icecream"):
 244                         requires.append("icecream")
 245
 246                 # Get build dependencies from source package.
 247                 if isinstance(self.pkg, packages.SourcePackage):
 248                         for req in self.pkg.requires:
 249                                 requires.append(req)
 250
 251                 # Install all packages.
 252                 self.install(requires)
 253
 254                 # Copy the makefile and load source tarballs.
 255                 if isinstance(self.pkg, packages.Makefile):
 256                         self.pkg.extract(self)
 257
 258                 elif isinstance(self.pkg, packages.SourcePackage):
 259                         self.pkg.extract(_("Extracting: %s (source)") % self.pkg.name,
 260                                 prefix=os.path.join(self.path, "build"))
 261
 262                 # If we have a makefile, we can only get the build dependencies
 263                 # after we have extracted all the rest.
 264                 if build_deps and isinstance(self.pkg, packages.Makefile):
 265                         requires = self.make_requires()
 266                         if not requires:
 267                                 return
 268
 269                         self.install(requires)
 270
 271         def install(self, requires):
 272                 """
 273                         Install everything that is required in requires.
 274                 """
 275                 # If we got nothing to do, we quit immediately.
 276                 if not requires:
 277                         return
 278
 279                 # Create a request and fill it with what we need.
 280                 request = self.pakfire.create_request()
 281
 282                 for req in requires:
 283                         if isinstance(req, packages.BinaryPackage):
 284                                 req = req.friendly_name
 285
 286                         if "<" in req or ">" in req or "=" in req or req.startswith("/"):
 287                                 req = self.pakfire.create_relation(req)
 288
 289                         request.install(req)
 290
 291                 # Create a new solver instance.
 292                 solver = self.pakfire.create_solver()
 293
 294                 # Do the solving.
 295                 transaction = solver.solve(request, allow_downgrade=True)
 296
 297                 # XXX check for errors
 298                 if not transaction:
 299                         raise DependencyError, "Could not resolve dependencies"
 300
 301                 # Show the user what is going to be done.
 302                 transaction.dump(logger=self.log)
 303
 304                 # Run the transaction.
 305                 transaction.run()
 306
 307         def install_test(self):
 308                 # XXX currently disabled
 309                 return
 310
 311                 pkgs = []
 312                 for dir, subdirs, files in os.walk(self.chrootPath("result")):
 313                         for file in files:
 314                                 pkgs.append(os.path.join(dir, file))
 315
 316                 self.pakfire.localinstall(pkgs)
 317
 318         def chrootPath(self, *args):
 319                 # Remove all leading slashes
 320                 _args = []
 321                 for arg in args:
 322                         if arg.startswith("/"):
 323                                 arg = arg[1:]
 324                         _args.append(arg)
 325                 args = _args
 326
 327                 ret = os.path.join(self.path, *args)
 328                 ret = ret.replace("//", "/")
 329
 330                 assert ret.startswith(self.path)
 331
 332                 return ret
 333
 334         def prepare(self):
 335                 prepared_tag = ".prepared"
 336
 337                 if os.path.exists(self.chrootPath(prepared_tag)):
 338                         return
 339
 340                 # Create directory.
 341                 if not os.path.exists(self.path):
 342                         os.makedirs(self.path)
 343
 344                 # Create important directories.
 345                 dirs = [
 346                         "build",
 347                         self.buildroot,
 348                         "dev",
 349                         "dev/pts",
 350                         "dev/shm",
 351                         "etc",
 352                         "proc",
 353                         "result",
 354                         "sys",
 355                         "tmp",
 356                         "usr/src",
 357                 ]
 358
 359                 # Create cache dir if ccache is enabled.
 360                 if self.settings.get("enable_ccache"):
 361                         dirs.append("var/cache/ccache")
 362
 363                         if not os.path.exists(CCACHE_CACHE_DIR):
 364                                 os.makedirs(CCACHE_CACHE_DIR)
 365
 366                 for dir in dirs:
 367                         dir = self.chrootPath(dir)
 368                         if not os.path.exists(dir):
 369                                 os.makedirs(dir)
 370
 371                 # Create neccessary files like /etc/fstab and /etc/mtab.
 372                 files = (
 373                         "etc/fstab",
 374                         "etc/mtab",
 375                         prepared_tag,
 376                 )
 377
 378                 for file in files:
 379                         file = self.chrootPath(file)
 380                         dir = os.path.dirname(file)
 381                         if not os.path.exists(dir):
 382                                 os.makedirs(dir)
 383                         f = open(file, "w")
 384                         f.close()
 385
 386                 self._prepare_dev()
 387                 self._prepare_dns()
 388
 389         def _prepare_dev(self):
 390                 prevMask = os.umask(0000)
 391
 392                 nodes = [
 393                         ("dev/null",    stat.S_IFCHR | 0666, os.makedev(1, 3)),
 394                         ("dev/full",    stat.S_IFCHR | 0666, os.makedev(1, 7)),
 395                         ("dev/zero",    stat.S_IFCHR | 0666, os.makedev(1, 5)),
 396                         ("dev/random",  stat.S_IFCHR | 0666, os.makedev(1, 8)),
 397                         ("dev/urandom", stat.S_IFCHR | 0444, os.makedev(1, 9)),
 398                         ("dev/tty",             stat.S_IFCHR | 0666, os.makedev(5, 0)),
 399                         ("dev/console", stat.S_IFCHR | 0600, os.makedev(5, 1)),
 400                 ]
 401
 402                 # If we need loop devices (which are optional) we create them here.
 403                 if self.settings["enable_loop_devices"]:
 404                         for i in range(0, 7):
 405                                 nodes.append(("dev/loop%d" % i, stat.S_IFBLK | 0660, os.makedev(7, i)))
 406
 407                 # Create all the nodes.
 408                 for node in nodes:
 409                         self._create_node(*node)
 410
 411                 os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
 412                 os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
 413                 os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
 414                 os.symlink("/proc/self/fd",   self.chrootPath("dev", "fd"))
 415
 416                 # make device node for el4 and el5
 417                 if self.kernel_version < "2.6.19":
 418                         self._make_node("dev/ptmx", stat.S_IFCHR | 0666, os.makedev(5, 2))
 419                 else:
 420                         os.symlink("/dev/pts/ptmx", self.chrootPath("dev", "ptmx"))
 421
 422                 os.umask(prevMask)
 423
 424         def _prepare_dns(self):
 425                 """
 426                         Add DNS resolution facility to chroot environment by copying
 427                         /etc/resolv.conf and /etc/hosts.
 428                 """
 429                 for i in ("/etc/resolv.conf", "/etc/hosts"):
 430                         self.copyin(i, i)
 431
 432         def _create_node(self, filename, mode, device):
 433                 logging.debug("Create node: %s (%s)" % (filename, mode))
 434
 435                 filename = self.chrootPath(filename)
 436
 437                 # Create parent directory if it is missing.
 438                 dirname = os.path.dirname(filename)
 439                 if not os.path.exists(dirname):
 440                         os.makedirs(dirname)
 441
 442                 os.mknod(filename, mode, device)
 443
 444         def destroy(self):
 445                 logging.debug("Destroying environment %s" % self.path)
 446
 447                 if os.path.exists(self.path):
 448                         util.rm(self.path)
 449
 450         def cleanup(self):
 451                 logging.debug("Cleaning environemnt.")
 452
 453                 # Run make clean and let it cleanup its stuff.
 454                 self.make("clean")
 455
 456                 # Remove the build directory and buildroot.
 457                 dirs = ("build", self.buildroot, "result")
 458
 459                 for d in dirs:
 460                         d = self.chrootPath(d)
 461                         if not os.path.exists(d):
 462                                 continue
 463
 464                         util.rm(d)
 465                         os.makedirs(d)
 466
 467                 # Clear make_info cache.
 468                 if hasattr(self, "_make_info"):
 469                         del self._make_info
 470
 471         def _mountall(self):
 472                 self.log.debug("Mounting environment")
 473                 for cmd, mountpoint in self.mountpoints:
 474                         cmd = "%s %s" % (cmd, self.chrootPath(mountpoint))
 475                         chroot.do(cmd, shell=True)
 476
 477         def _umountall(self):
 478                 self.log.debug("Umounting environment")
 479                 for cmd, mountpoint in self.mountpoints:
 480                         cmd = "umount -n %s" % self.chrootPath(mountpoint)
 481                         chroot.do(cmd, raiseExc=0, shell=True)
 482
 483         @property
 484         def mountpoints(self):
 485                 ret = [
 486                         ("mount -n -t proc pakfire_chroot_proc", "proc"),
 487                         ("mount -n -t sysfs pakfire_chroot_sysfs", "sys"),
 488                 ]
 489
 490                 mountopt = "gid=%d,mode=0620,ptmxmode=0666" % grp.getgrnam("tty").gr_gid
 491                 if self.kernel_version >= "2.6.29":
 492                         mountopt += ",newinstance"
 493
 494                 ret.extend([
 495                         ("mount -n -t devpts -o %s pakfire_chroot_devpts" % mountopt, "dev/pts"),
 496                         ("mount -n -t tmpfs pakfire_chroot_shmfs", "dev/shm"),
 497                 ])
 498
 499                 if self.settings.get("enable_ccache"):
 500                         ret.append(("mount -n --bind %s" % CCACHE_CACHE_DIR, "var/cache/ccache"))
 501
 502                 return ret
 503
 504         @staticmethod
 505         def calc_parallelism():
 506                 """
 507                         Calculate how many processes to run
 508                         at the same time.
 509
 510                         We take the log10(number of processors) * factor
 511                 """
 512                 num = os.sysconf("SC_NPROCESSORS_CONF")
 513                 if num == 1:
 514                         return 2
 515                 else:
 516                         return int(round(math.log10(num) * 26))
 517
 518         @property
 519         def environ(self):
 520                 env = {
 521                         # Add HOME manually, because it is occasionally not set
 522                         # and some builds get in trouble then.
 523                         "HOME" : "/root",
 524                         "TERM" : os.environ.get("TERM", "dumb"),
 525                         "PS1"  : "\u:\w\$ ",
 526
 527                         "BUILDROOT" : self.buildroot,
 528                         "PARALLELISMFLAGS" : "-j%s" % self.calc_parallelism(),
 529                 }
 530
 531                 # Inherit environment from distro
 532                 env.update(self.pakfire.distro.environ)
 533
 534                 # Icecream environment settings
 535                 if self.settings.get("enable_icecream", None):
 536                         # Set the toolchain path
 537                         if self.settings.get("icecream_toolchain", None):
 538                                 env["ICECC_VERSION"] = self.settings.get("icecream_toolchain")
 539
 540                         # Set preferred host if configured.
 541                         if self.settings.get("icecream_preferred_host", None):
 542                                 env["ICECC_PREFERRED_HOST"] = \
 543                                         self.settings.get("icecream_preferred_host")
 544
 545                 # XXX what do we need else?
 546
 547                 return env
 548
 549         def do(self, command, shell=True, personality=None, logger=None, *args, **kwargs):
 550                 ret = None
 551                 try:
 552                         # Environment variables
 553                         env = self.environ
 554
 555                         if kwargs.has_key("env"):
 556                                 env.update(kwargs.pop("env"))
 557
 558                         logging.debug("Environment:")
 559                         for k, v in sorted(env.items()):
 560                                 logging.debug("  %s=%s" % (k, v))
 561
 562                         # Update personality it none was set
 563                         if not personality:
 564                                 personality = self.distro.personality
 565
 566                         # Make every shell to a login shell because we set a lot of
 567                         # environment things there.
 568                         if shell:
 569                                 command = ["bash", "--login", "-c", command]
 570
 571                         self._mountall()
 572
 573                         if not kwargs.has_key("chrootPath"):
 574                                 kwargs["chrootPath"] = self.chrootPath()
 575
 576                         ret = chroot.do(
 577                                 command,
 578                                 personality=personality,
 579                                 shell=False,
 580                                 env=env,
 581                                 logger=logger,
 582                                 *args,
 583                                 **kwargs
 584                         )
 585
 586                 finally:
 587                         self._umountall()
 588
 589                 return ret
 590
 591         def make(self, *args, **kwargs):
 592                 if isinstance(self.pkg, packages.Makefile):
 593                         filename = os.path.basename(self.pkg.filename)
 594                 elif isinstance(self.pkg, packages.SourcePackage):
 595                         filename = "%s.%s" % (self.pkg.name, MAKEFILE_EXTENSION)
 596
 597                 return self.do("make -f /build/%s %s" % (filename, " ".join(args)),
 598                         **kwargs)
 599
 600         @property
 601         def make_info(self):
 602                 if not hasattr(self, "_make_info"):
 603                         info = {}
 604
 605                         output = self.make("buildinfo", returnOutput=True)
 606
 607                         for line in output.splitlines():
 608                                 # XXX temporarily
 609                                 if not line:
 610                                         break
 611
 612                                 m = re.match(r"^(\w+)=(.*)$", line)
 613                                 if not m:
 614                                         continue
 615
 616                                 info[m.group(1)] = m.group(2).strip("\"")
 617
 618                         self._make_info = info
 619
 620                 return self._make_info
 621
 622         @property
 623         def packages(self):
 624                 if hasattr(self, "_packages"):
 625                         return self._packages
 626
 627                 pkgs = []
 628                 output = self.make("packageinfo", returnOutput=True)
 629
 630                 pkg = {}
 631                 for line in output.splitlines():
 632                         if not line:
 633                                 pkgs.append(pkg)
 634                                 pkg = {}
 635
 636                         m = re.match(r"^(\w+)=(.*)$", line)
 637                         if not m:
 638                                 continue
 639
 640                         k, v = m.groups()
 641                         pkg[k] = v.strip("\"")
 642
 643                 self._packages = []
 644                 for pkg in pkgs:
 645                         pkg = packages.VirtualPackage(self.pakfire, pkg)
 646                         self._packages.append(pkg)
 647
 648                 return self._packages
 649
 650         def make_requires(self):
 651                 return self.make_info.get("PKG_BUILD_DEPS", "").split()
 652
 653         def make_sources(self):
 654                 return self.make_info.get("PKG_FILES", "").split()
 655
 656         def create_icecream_toolchain(self):
 657                 if not self.settings.get("enable_icecream", None):
 658                         return
 659
 660                 out = self.do("icecc --build-native", returnOutput=True)
 661
 662                 for line in out.splitlines():
 663                         m = re.match(r"^creating ([a-z0-9]+\.tar\.gz)", line)
 664                         if m:
 665                                 self.settings["icecream_toolchain"] = "/%s" % m.group(1)
 666
 667         def build(self):
 668                 self.create_icecream_toolchain()
 669
 670                 try:
 671                         self.make("build", logger=self.log)
 672
 673                 except Error:
 674                         raise BuildError, "The build command failed."
 675
 676                 for pkg in reversed(self.packages):
 677                         packager = packages.BinaryPackager(self.pakfire, pkg, self)
 678                         packager()
 679
 680         def dist(self):
 681                 self.pkg.dist(self)
 682
 683         def shell(self, args=[]):
 684                 if not util.cli_is_interactive():
 685                         logging.warning("Cannot run shell on non-interactive console.")
 686                         return
 687
 688                 # Install all packages that are needed to run a shell.
 689                 self.install(SHELL_PACKAGES)
 690
 691                 # XXX need to set CFLAGS here
 692                 command = "/usr/sbin/chroot %s /usr/bin/chroot-shell %s" % \
 693                         (self.chrootPath(), " ".join(args))
 694
 695                 # Add personality if we require one
 696                 if self.pakfire.distro.personality:
 697                         command = "%s %s" % (self.pakfire.distro.personality, command)
 698
 699                 for key, val in self.environ.items():
 700                         command = "%s=\"%s\" " % (key, val) + command
 701
 702                 # Empty the environment
 703                 command = "env -i - %s" % command
 704
 705                 logging.debug("Shell command: %s" % command)
 706
 707                 try:
 708                         self._mountall()
 709
 710                         shell = os.system(command)
 711                         return os.WEXITSTATUS(shell)
 712
 713                 finally:
 714                         self._umountall()