naoki/backend.py

   1 #!/usr/bin/python
   2
   3
   4 import os
   5 import shutil
   6 import smtplib
   7 import urlgrabber
   8 import urlgrabber.progress
   9 import urllib
  10
  11 import chroot
  12 import util
  13
  14 from exception import *
  15 from constants import *
  16
  17 __cache = {
  18         "package_names" : None,
  19         "group_names" : None,
  20 }
  21
  22 # Python 2.4 does not have that email module, so
  23 # we disable the mail function here.
  24 try:
  25         import email.mime.multipart
  26         import email.mime.text
  27         have_email = 1
  28 except ImportError:
  29         have_email = 0
  30
  31 try:
  32         import hashlib
  33         have_hashlib = 1
  34 except ImportError:
  35         import sha
  36         have_hashlib = 0
  37
  38 def find_package_info(name, toolchain=False, **kwargs):
  39         for repo in get_repositories(toolchain):
  40                 if not os.path.exists(os.path.join(repo.path, name, name + ".nm")):
  41                         continue
  42
  43                 return PackageInfo(name, repo=repo, **kwargs)
  44
  45 def find_package(name, naoki, toolchain=False):
  46         package = find_package_info(name, toolchain)
  47         if package:
  48                 return package.getPackage(naoki)
  49
  50         return None
  51
  52 def parse_package_info(names, toolchain=False, **kwargs):
  53         packages = []
  54         for name in names:
  55                 package = find_package_info(name, toolchain, **kwargs)
  56                 if package:
  57                         packages.append(package)
  58
  59         return packages
  60
  61 def parse_package(names, toolchain=False, naoki=None):
  62         packages = parse_package_info(names, toolchain)
  63
  64         return [Package(package.name, naoki=naoki, toolchain=toolchain) \
  65                 for package in packages]
  66
  67 def get_package_names(toolchain=False):
  68         if not __cache["package_names"]:
  69                 names = []
  70                 for repo in get_repositories(toolchain):
  71                         names.extend(repo.package_names)
  72
  73                 __cache["package_names"] = sorted(names)
  74
  75         return __cache["package_names"]
  76
  77 def get_group_names():
  78         if not __cache["group_names"]:
  79                 groups = []
  80                 for package in parse_package_info(get_package_names()):
  81                         if not package.group in groups:
  82                                 groups.append(package.group)
  83
  84                 __cache["group_names"] = sorted(groups)
  85
  86         return __cache["group_names"]
  87
  88 def find_package_name(name, toolchain=False):
  89         if name in get_package_names(toolchain):
  90                 return name
  91
  92         for package in get_package_names(toolchain):
  93                 if os.path.basename(package) == name:
  94                         return package
  95
  96 def depsolve(packages, recursive=False, build=False, toolchain=False):
  97         deps = []
  98         for package in packages:
  99                 if not package in deps:
 100                         deps.append(package)
 101
 102         if not recursive or not deps:
 103                 return deps
 104
 105         while True:
 106                 length = len(deps)
 107                 for dep in deps[:]:
 108                         deps.extend(dep.dependencies)
 109                         if build and not toolchain:
 110                                 deps.extend(dep.dependencies_build)
 111
 112                 new_deps = []
 113                 for dep in deps:
 114                         if not dep in new_deps:
 115                                 new_deps.append(dep)
 116
 117                 deps = new_deps
 118
 119                 if length == len(deps):
 120                         break
 121
 122         deps.sort()
 123         return deps
 124
 125 def deptree(packages, toolchain=False):
 126         ret = [packages]
 127
 128         while True:
 129                 next = []
 130                 stage = ret[-1][:]
 131                 for package in stage[:]:
 132                         for dep in package.dependencies_all:
 133                                 if dep in ret[-1]:
 134                                         stage.remove(package)
 135                                         next.append(package)
 136                                         break
 137
 138                 ret[-1] = stage
 139                 if next:
 140                         ret.append(next)
 141                         continue
 142
 143                 break
 144
 145         return ret
 146
 147 def depsort(packages, toolchain=False):
 148         ret = []
 149         for l1 in deptree(packages, toolchain=toolchain):
 150                 ret.extend(l1)
 151         return ret
 152
 153 def calc_hash(data):
 154         if have_hashlib:
 155                 obj = hashlib.sha1(data)
 156         else:
 157                 obj = sha.new(data)
 158
 159         return obj.hexdigest()
 160
 161 def download(files, logger=None):
 162         for file in files:
 163                 filepath = os.path.join(TARBALLDIR, file)
 164
 165                 if not os.path.exists(TARBALLDIR):
 166                         os.makedirs(TARBALLDIR)
 167
 168                 if os.path.exists(filepath):
 169                         continue
 170
 171                 url = config["sources_download_url"] + "/" + file
 172
 173                 if logger:
 174                         logger.debug("Retrieving %s" % url)
 175
 176                 g = urlgrabber.grabber.URLGrabber(
 177                         user_agent = "%sSourceGrabber/%s" % (config["distro_name"], config["distro_version"],),
 178                         progress_obj = urlgrabber.progress.TextMeter(),
 179                         quote=0,
 180                 )
 181
 182                 try:
 183                         gobj = g.urlopen(url)
 184                 except urlgrabber.grabber.URLGrabError, e:
 185                         if logger:
 186                                 logger.error("Could not retrieve %s - %s" % (url, e))
 187                         raise
 188
 189                 data = gobj.read()
 190                 gobj.close()
 191
 192                 if gobj.hdr.has_key("X-Hash-Sha1"):
 193                         hash_server = gobj.hdr["X-Hash-Sha1"]
 194                         msg = "Comparing hashes - %s" % hash_server
 195
 196                         hash_calculated = calc_hash(data)
 197                         if hash_calculated == hash_server:
 198                                 if logger:
 199                                         logger.debug(msg + " - OK")
 200                         else:
 201                                 if logger:
 202                                         logger.error(msg + " - ERROR")
 203                                 raise DownloadError, "Hash sum of downloaded file does not match"
 204
 205                 fobj = open(filepath, "w")
 206                 fobj.write(data)
 207                 fobj.close()
 208
 209
 210 class PackageInfo(object):
 211         __data = {}
 212
 213         def __init__(self, name, repo=None, arch=arches.current["name"]):
 214                 self._name = name
 215                 self.repo = repo
 216
 217                 self.arch = arch
 218
 219         def __cmp__(self, other):
 220                 return cmp(self.name, other.name)
 221
 222         def __repr__(self):
 223                 return "<PackageInfo %s>" % self.name
 224
 225         def get_data(self):
 226                 if not self.__data.has_key(self.name):
 227                         self.__data[self.name] = self.fetch()
 228
 229                 return self.__data[self.name]
 230
 231         def set_data(self, data):
 232                 self.__data[self.name] = data
 233
 234         _data = property(get_data, set_data)
 235
 236         def fetch(self):
 237                 env = os.environ.copy()
 238                 env.update(config.environment)
 239                 env.update({
 240                         "PKG_ARCH" : self.arch,
 241                         "PKGROOT" : PKGSDIR,
 242                 })
 243                 output = util.do("make -f %s" % self.filename, shell=True,
 244                         cwd=os.path.join(PKGSDIR, self.repo.name, self.name), returnOutput=1, env=env)
 245
 246                 ret = {}
 247                 for line in output.splitlines():
 248                         a = line.split("=", 1)
 249                         if not len(a) == 2: continue
 250                         key, val = a
 251                         ret[key] = val.strip("\"")
 252
 253                 ret["FINGERPRINT"] = self.fingerprint
 254
 255                 return ret
 256
 257         def fmtstr(self, s):
 258                 return s % self.all
 259
 260         def getPackage(self, naoki):
 261                 return Package(self.name, naoki)
 262
 263         @property
 264         def all(self):
 265                 return {
 266                         "build_deps"  : [dep.name for dep in self.dependencies_build],
 267                         "deps"        : [dep.name for dep in self.dependencies],
 268                         "description" : self.description,
 269                         "filename"    : self.filename,
 270                         "fingerprint" : self.fingerprint,
 271                         "files"       : self.package_files,
 272                         "group"       : self.group,
 273                         "id"          : self.id,
 274                         "license"     : self.license,
 275                         "maintainer"  : self.maintainer,
 276                         "name"        : self.name,
 277                         "objects"     : self.objects,
 278                         "patches"     : self.patches,
 279                         "release"     : self.release,
 280                         "summary"     : self.summary,
 281                         "url"         : self.url,
 282                         "version"     : self.version,
 283                 }
 284
 285         @property
 286         def buildable(self):
 287                 return self.dependencies_unbuilt == []
 288
 289         @property
 290         def built(self):
 291                 for file in self.package_files:
 292                         if not os.path.exists(os.path.join(PACKAGESDIR, file)):
 293                                 return False
 294
 295                 return True
 296
 297         def _dependencies(self, s, recursive=False, toolchain=False):
 298                 c = s + "_CACHE"
 299                 if not self._data.has_key(c):
 300                         deps = parse_package_info(self._data.get(s).split(" "), toolchain=toolchain)
 301                         self._data.update({c : depsolve(deps, recursive)})
 302
 303                 return self._data.get(c)
 304
 305         @property
 306         def dependencies(self):
 307                 if self.__toolchain:
 308                         return self.dependencies_toolchain
 309
 310                 return self._dependencies("PKG_DEPENDENCIES")
 311
 312         @property
 313         def dependencies_build(self):
 314                 return self._dependencies("PKG_BUILD_DEPENDENCIES")
 315
 316         @property
 317         def dependencies_built(self):
 318                 ret = []
 319                 for dep in self.dependencies_all:
 320                         if dep.built:
 321                                 ret.append(dep)
 322
 323                 return ret
 324
 325         @property
 326         def dependencies_unbuilt(self):
 327                 ret = []
 328                 for dep in self.dependencies_all:
 329                         if not dep.built:
 330                                 ret.append(dep)
 331
 332                 return ret
 333
 334         @property
 335         def dependencies_all(self):
 336                 deps = self.dependencies
 337                 if not self.__toolchain:
 338                         deps.extend(self.dependencies_build)
 339                 return depsolve(deps, build=True, recursive=True, toolchain=self.__toolchain)
 340
 341         @property
 342         def dependencies_toolchain(self):
 343                 return self._dependencies("PKG_TOOLCHAIN_DEPENDENCIES", toolchain=True)
 344
 345         @property
 346         def description(self):
 347                 return self._data.get("PKG_DESCRIPTION")
 348
 349         @property
 350         def filename(self):
 351                 return os.path.join(PKGSDIR, self.repo.name, self.name,
 352                         os.path.basename(self.name)) + ".nm"
 353
 354         @property
 355         def fingerprint(self):
 356                 return "%d" % os.stat(self.filename).st_mtime
 357
 358         @property
 359         def group(self):
 360                 return self._data.get("PKG_GROUP")
 361
 362         @property
 363         def id(self):
 364                 return "%s-%s-%s" % (self.name, self.version, self.release)
 365
 366         @property
 367         def license(self):
 368                 return self._data.get("PKG_LICENSE")
 369
 370         @property
 371         def maintainer(self):
 372                 return self._data.get("PKG_MAINTAINER")
 373
 374         @property
 375         def name(self):
 376                 return self._name
 377
 378         @property
 379         def objects(self):
 380                 return self._data.get("PKG_OBJECTS").split(" ")
 381
 382         @property
 383         def package_files(self):
 384                 return self._data.get("PKG_PACKAGES_FILES").split(" ")
 385
 386         @property
 387         def patches(self):
 388                 return self._data.get("PKG_PATCHES").split(" ")
 389
 390         @property
 391         def release(self):
 392                 return self._data.get("PKG_REL")
 393
 394         @property
 395         def summary(self):
 396                 return self._data.get("PKG_SUMMARY")
 397
 398         @property
 399         def url(self):
 400                 return self._data.get("PKG_URL")
 401
 402         @property
 403         def version(self):
 404                 return self._data.get("PKG_VER")
 405
 406         @property
 407         def __toolchain(self):
 408                 return self.repo.name == "toolchain"
 409
 410
 411 class Package(object):
 412         def __init__(self, name, naoki, toolchain=False):
 413                 self.info = find_package_info(name, toolchain)
 414
 415                 assert naoki
 416                 self.naoki = naoki
 417
 418                 #self.log.debug("Initialized package object %s" % name)
 419
 420         def __repr__(self):
 421                 return "<Package %s>" % self.info.name
 422
 423         def __cmp__(self, other):
 424                 return cmp(self.name, other.name)
 425
 426         def __getattr__(self, attr):
 427                 return getattr(self.info, attr)
 428
 429         @property
 430         def name(self):
 431                 return self.info.name
 432
 433         def build(self):
 434                 environment = chroot.PackageEnvironment(self)
 435                 environment.build()
 436
 437         def download(self):
 438                 download(self.info.objects, logger=self.log)
 439
 440         def extract(self, dest):
 441                 files = [os.path.join(PACKAGESDIR, file) for file in self.info.package_files]
 442                 if not files:
 443                         return
 444
 445                 self.log.debug("Extracting %s..." % files)
 446                 util.do("%s --root=%s %s" % (os.path.join(TOOLSDIR, "decompressor"),
 447                         dest, " ".join(files)), shell=True)
 448
 449         def getEnvironment(self, *args, **kwargs):
 450                 return chroot.PackageEnvironment(self, *args, **kwargs)
 451
 452         @property
 453         def log(self):
 454                 return self.naoki.logging.getBuildLogger(self.info.id)
 455
 456
 457 def get_repositories(toolchain=False):
 458         if toolchain:
 459                 return [Repository("toolchain")]
 460
 461         repos = []
 462         for repo in os.listdir(PKGSDIR):
 463                 if os.path.isdir(os.path.join(PKGSDIR, repo)):
 464                         repos.append(repo)
 465
 466         repos.remove("toolchain")
 467
 468         return [Repository(repo) for repo in repos]
 469
 470 class Repository(object):
 471         def __init__(self, name):
 472                 self.name = name
 473
 474         def __repr__(self):
 475                 return "<Repository %s>" % self.name
 476
 477         @property
 478         def packages(self):
 479                 packages = []
 480                 for package in os.listdir(self.path):
 481                         package = PackageInfo(package, repo=self)
 482                         packages.append(package)
 483
 484                 return packages
 485
 486         @property
 487         def package_names(self):
 488                 return [package.name for package in self.packages]
 489
 490         @property
 491         def path(self):
 492                 return os.path.join(PKGSDIR, self.name)
 493
 494
 495 class BinaryRepository(object):
 496         DIRS = ("db", "packages")
 497
 498         def __init__(self, name, naoki=None, arch=None):
 499                 self.name = name
 500                 self.arch = arch or arches.current
 501                 self.repo = Repository(self.name)
 502
 503                 assert naoki
 504                 self.naoki = naoki
 505
 506         def build(self):
 507                 if not self.buildable:
 508                         raise Exception, "Cannot build repository"
 509
 510                 # Create temporary directory layout
 511                 util.rm(self.repopath("tmp"))
 512                 for dir in self.DIRS:
 513                         util.mkdir(self.repopath("tmp", dir))
 514
 515                 # Copy packages
 516                 for package in self.packages:
 517                         for file in package.package_files:
 518                                 shutil.copy(os.path.join(PACKAGESDIR, file),
 519                                         self.repopath("tmp", "packages"))
 520
 521                 # TODO check repository's sanity
 522                 # TODO create repoview
 523                 f = open(self.repopath("tmp", "db", "package-list.txt"), "w")
 524                 for package in self.packages:
 525                         s = "%-40s" % package.fmtstr("%(name)s-%(version)s-%(release)s")
 526                         s += " | %s\n" % package.summary
 527                         f.write(s)
 528                 f.close()
 529
 530                 for dir in self.DIRS:
 531                         util.rm(self.repopath(dir))
 532                         shutil.move(self.repopath("tmp", dir), self.repopath(dir))
 533                 util.rm(self.repopath("tmp"))
 534
 535         def clean(self):
 536                 if os.path.exists(self.path):
 537                         self.log.debug("Cleaning up repository: %s" % self.path)
 538                         util.rm(self.path)
 539
 540         def repopath(self, *args):
 541                 return os.path.join(self.path, *args)
 542
 543         @property
 544         def buildable(self):
 545                 for package in self.packages:
 546                         if package.built:
 547                                 continue
 548                         return False
 549
 550                 return True
 551
 552         @property
 553         def log(self):
 554                 return self.naoki.log
 555
 556         @property
 557         def packages(self):
 558                 packages = []
 559                 for package in parse_package_info(get_package_names(), arch=self.arch["name"]):
 560                         if not package.repo.name == self.name:
 561                                 continue
 562                         packages.append(package)
 563                 return packages
 564
 565         @property
 566         def path(self):
 567                 return os.path.join(REPOSDIR, self.name, self.arch["name"])
 568
 569 def report_error_by_mail(package):
 570         log = package.naoki.log
 571
 572         # Do not send a report if no recipient is configured
 573         if not config["error_report_recipient"]:
 574                 return
 575
 576         if not have_email:
 577                 log.error("Can't send mail because this python version does not support this")
 578                 return
 579
 580         try:
 581                 connection = smtplib.SMTP(config["smtp_server"])
 582                 #connection.set_debuglevel(1)
 583
 584                 if config["smtp_user"] and config["smtp_password"]:
 585                         connection.login(config["smtp_user"], config["smtp_password"])
 586
 587         except SMTPConnectError, e:
 588                 log.error("Could not establish a connection to the smtp server: %s" % e)
 589                 return
 590         except SMTPAuthenticationError, e:
 591                 log.error("Could not successfully login to the smtp server: %s" % e)
 592                 return
 593
 594         msg = email.mime.multipart.MIMEMultipart()
 595         msg["From"] = config["error_report_sender"]
 596         msg["To"] = config["error_report_recipient"]
 597         msg["Subject"] = config["error_report_subject"] % package.all
 598         msg.preamble = 'You will not see this in a MIME-aware mail reader.\n'
 599
 600         body = """\
 601 The package %(name)s had a difficulty to build itself.
 602 This email will give you a short report about the error.
 603
 604 Package information:
 605   Name    : %(name)s - %(summary)s
 606   Version : %(version)s
 607   Release : %(release)s
 608
 609   This package in maintained by %(maintainer)s.
 610
 611
 612 A detailed logfile is attached.
 613
 614 Sincerely,
 615     Naoki
 616         """ % package.all
 617
 618         msg.attach(email.mime.text.MIMEText(body))
 619
 620         # Read log and append it to mail
 621         logfile = os.path.join(LOGDIR, package.id + ".log")
 622         if os.path.exists(logfile):
 623                 log = []
 624                 f = open(logfile)
 625                 line = f.readline()
 626                 while line:
 627                         line = line.rstrip("\n")
 628                         if line.endswith(LOG_MARKER):
 629                                 # Reset log
 630                                 log = []
 631
 632                         log.append(line)
 633                         line = f.readline()
 634
 635                 f.close()
 636
 637         log = email.mime.text.MIMEText("\n".join(log), _subtype="plain")
 638         log.add_header('Content-Disposition', 'attachment',
 639                 filename="%s.log" % package.id)
 640         msg.attach(log)
 641
 642         try:
 643                 connection.sendmail(config["error_report_sender"],
 644                         config["error_report_recipient"], msg.as_string())
 645         except Exception, e:
 646                 log.error("Could not send error report: %s: %s" % (e.__class__.__name__, e))
 647                 return
 648
 649         connection.quit()