naoki/backend.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import urlgrabber
   5 import urlgrabber.progress
   6 import urllib
   7
   8 import chroot
   9 import util
  10
  11 from constants import *
  12
  13 __cache = {
  14         "package_names" : None,
  15         "group_names" : None,
  16 }
  17
  18 def find_package_info(name, toolchain=False):
  19         for repo in get_repositories(toolchain):
  20                 if not os.path.exists(os.path.join(repo.path, name, name + ".nm")):
  21                         continue
  22
  23                 return PackageInfo(name, repo=repo)
  24
  25 def find_package(name, toolchain=False):
  26         package = find_package_info(name, toolchain)
  27         if package:
  28                 package = backend.Package(package)
  29
  30         return package
  31
  32 def parse_package_info(names, toolchain=False):
  33         packages = []
  34         for name in names:
  35                 package = find_package_info(name, toolchain)
  36                 if package:
  37                         packages.append(package)
  38
  39         return packages
  40
  41 def parse_package(names, toolchain=False, naoki=None):
  42         packages = parse_package_info(names, toolchain)
  43
  44         return [Package(package.name, naoki=naoki, toolchain=toolchain) \
  45                 for package in packages]
  46
  47 def get_package_names(toolchain=False):
  48         if not __cache["package_names"]:
  49                 names = []
  50                 for repo in get_repositories(toolchain):
  51                         names.extend(repo.package_names)
  52
  53                 __cache["package_names"] = sorted(names)
  54
  55         return __cache["package_names"]
  56
  57 def get_group_names():
  58         if not __cache["group_names"]:
  59                 groups = []
  60                 for package in parse_package_info(get_package_names()):
  61                         if not package.group in groups:
  62                                 groups.append(package.group)
  63
  64                 __cache["group_names"] = sorted(groups)
  65
  66         return __cache["group_names"]
  67
  68 def find_package_name(name, toolchain=False):
  69         if name in get_package_names(toolchain):
  70                 return name
  71
  72         for package in get_package_names(toolchain):
  73                 if os.path.basename(package) == name:
  74                         return package
  75
  76 def depsolve(packages, recursive=False, build=False, toolchain=False):
  77         deps = []
  78         for package in packages:
  79                 if not package in deps:
  80                         deps.append(package)
  81
  82         if not recursive or not deps:
  83                 return deps
  84
  85         while True:
  86                 length = len(deps)
  87                 for dep in deps[:]:
  88                         deps.extend(dep.dependencies)
  89                         if build and not toolchain:
  90                                 deps.extend(dep.dependencies_build)
  91
  92                 new_deps = []
  93                 for dep in deps:
  94                         if not dep in new_deps:
  95                                 new_deps.append(dep)
  96
  97                 deps = new_deps
  98
  99                 if length == len(deps):
 100                         break
 101
 102         deps.sort()
 103         return deps
 104
 105 def deptree(packages, toolchain=False):
 106         ret = [packages]
 107
 108         while True:
 109                 next = []
 110                 stage = ret[-1][:]
 111                 for package in stage[:]:
 112                         for dep in package.dependencies_all:
 113                                 if dep in ret[-1]:
 114                                         stage.remove(package)
 115                                         next.append(package)
 116                                         break
 117
 118                 ret[-1] = stage
 119                 if next:
 120                         ret.append(next)
 121                         continue
 122
 123                 break
 124
 125         return ret
 126
 127 def depsort(packages, toolchain=False):
 128         ret = []
 129         for l1 in deptree(packages, toolchain=toolchain):
 130                 ret.extend(l1)
 131         return ret
 132
 133 def download(files, logger=None):
 134         for file in files:
 135                 filepath = os.path.join(TARBALLDIR, file)
 136
 137                 if not os.path.exists(TARBALLDIR):
 138                         os.makedirs(TARBALLDIR)
 139
 140                 if os.path.exists(filepath):
 141                         continue
 142
 143                 url = config["sources_download_url"] + "/" + file
 144
 145                 if logger:
 146                         logger.debug("Retrieving %s" % url)
 147
 148                 g = urlgrabber.grabber.URLGrabber(
 149                         user_agent = "%sSourceGrabber/%s" % (config["distro_name"], config["distro_version"],),
 150                         progress_obj = urlgrabber.progress.TextMeter(),
 151                         quote=0,
 152                 )
 153
 154                 try:
 155                         gobj = g.urlopen(url)
 156                 except urlgrabber.grabber.URLGrabError, e:
 157                         logger.error("Could not retrieve %s - %s" % (url, e))
 158                         raise
 159
 160                 # XXX Need to check SHA1 sum here
 161
 162                 fobj = open(filepath, "w")
 163                 fobj.write(gobj.read())
 164                 fobj.close()
 165                 gobj.close()
 166
 167 class PackageInfo(object):
 168         __data = {}
 169
 170         def __init__(self, name, repo=None):
 171                 self._name = name
 172                 self.repo = repo
 173
 174                 self.arch = arches.current["name"]
 175
 176         #def __cmp__(self, other):
 177         #       return cmp(self.name, other.name)
 178
 179         def __repr__(self):
 180                 return "<PackageInfo %s>" % self.name
 181
 182         def get_data(self):
 183                 if not self.__data.has_key(self.name):
 184                         self.__data[self.name] = self.fetch()
 185
 186                 return self.__data[self.name]
 187
 188         def set_data(self, data):
 189                 self.__data[self.name] = data
 190
 191         _data = property(get_data, set_data)
 192
 193         def fetch(self):
 194                 env = os.environ.copy()
 195                 env.update(config.environment)
 196                 env.update({
 197                         "PKG_ARCH" : self.arch,
 198                         "PKGROOT" : PKGSDIR,
 199                 })
 200                 output = util.do("make -f %s" % self.filename, shell=True,
 201                         cwd=os.path.join(PKGSDIR, self.repo.name, self.name), returnOutput=1, env=env)
 202
 203                 ret = {}
 204                 for line in output.splitlines():
 205                         a = line.split("=", 1)
 206                         if not len(a) == 2: continue
 207                         key, val = a
 208                         ret[key] = val.strip("\"")
 209
 210                 ret["FINGERPRINT"] = self.fingerprint
 211
 212                 return ret
 213
 214         def fmtstr(self, s):
 215                 return s % self.all
 216
 217         def getPackage(self, naoki):
 218                 return Package(self.name, naoki)
 219
 220         @property
 221         def all(self):
 222                 return {
 223                         "build_deps"  : [dep.name for dep in self.dependencies_build],
 224                         "deps"        : [dep.name for dep in self.dependencies],
 225                         "description" : self.description,
 226                         "filename"    : self.filename,
 227                         "fingerprint" : self.fingerprint,
 228                         "files"       : self.package_files,
 229                         "group"       : self.group,
 230                         "license"     : self.license,
 231                         "maintainer"  : self.maintainer,
 232                         "name"        : self.name,
 233                         "objects"     : self.objects,
 234                         "patches"     : self.patches,
 235                         "release"     : self.release,
 236                         "summary"     : self.summary,
 237                         "url"         : self.url,
 238                         "version"     : self.version,
 239                 }
 240
 241         @property
 242         def buildable(self):
 243                 return self.dependencies_unbuilt == []
 244
 245         @property
 246         def built(self):
 247                 for file in self.package_files:
 248                         if not os.path.exists(os.path.join(PACKAGESDIR, file)):
 249                                 return False
 250
 251                 return True
 252
 253         def _dependencies(self, s, recursive=False, toolchain=False):
 254                 c = s + "_CACHE"
 255                 if not self._data.has_key(c):
 256                         deps = parse_package(self._data.get(s).split(" "), toolchain=toolchain)
 257                         self._data.update({c : depsolve(deps, recursive)})
 258
 259                 return self._data.get(c)
 260
 261         @property
 262         def dependencies(self):
 263                 if self.__toolchain:
 264                         return self.dependencies_toolchain
 265
 266                 return self._dependencies("PKG_DEPENDENCIES")
 267
 268         @property
 269         def dependencies_build(self):
 270                 return self._dependencies("PKG_BUILD_DEPENDENCIES")
 271
 272         @property
 273         def dependencies_built(self):
 274                 ret = []
 275                 for dep in self.dependencies_all:
 276                         if dep.built:
 277                                 ret.append(dep)
 278
 279                 return ret
 280
 281         @property
 282         def dependencies_unbuilt(self):
 283                 ret = []
 284                 for dep in self.dependencies_all:
 285                         if not dep.built:
 286                                 ret.append(dep)
 287
 288                 return ret
 289
 290         @property
 291         def dependencies_all(self):
 292                 deps = self.dependencies
 293                 if not self.__toolchain:
 294                         deps.extend(self.dependencies_build)
 295                 return depsolve(deps, build=True, recursive=True, toolchain=self.__toolchain)
 296
 297         @property
 298         def dependencies_toolchain(self):
 299                 return self._dependencies("PKG_TOOLCHAIN_DEPENDENCIES", toolchain=True)
 300
 301         @property
 302         def description(self):
 303                 return self._data.get("PKG_DESCRIPTION")
 304
 305         @property
 306         def filename(self):
 307                 return os.path.join(PKGSDIR, self.repo.name, self.name,
 308                         os.path.basename(self.name)) + ".nm"
 309
 310         @property
 311         def fingerprint(self):
 312                 return "%d" % os.stat(self.filename).st_mtime
 313
 314         @property
 315         def group(self):
 316                 return self._data.get("PKG_GROUP")
 317
 318         @property
 319         def id(self):
 320                 return "%s-%s-%s" % (self.name, self.version, self.release)
 321
 322         @property
 323         def license(self):
 324                 return self._data.get("PKG_LICENSE")
 325
 326         @property
 327         def maintainer(self):
 328                 return self._data.get("PKG_MAINTAINER")
 329
 330         @property
 331         def name(self):
 332                 return self._name
 333
 334         @property
 335         def objects(self):
 336                 return self._data.get("PKG_OBJECTS").split(" ")
 337
 338         @property
 339         def package_files(self):
 340                 return self._data.get("PKG_PACKAGES_FILES").split(" ")
 341
 342         @property
 343         def patches(self):
 344                 return self._data.get("PKG_PATCHES").split(" ")
 345
 346         @property
 347         def release(self):
 348                 return self._data.get("PKG_REL")
 349
 350         @property
 351         def summary(self):
 352                 return self._data.get("PKG_SUMMARY")
 353
 354         @property
 355         def url(self):
 356                 return self._data.get("PKG_URL")
 357
 358         @property
 359         def version(self):
 360                 return self._data.get("PKG_VER")
 361
 362         @property
 363         def __toolchain(self):
 364                 return self.repo.name == "toolchain"
 365
 366
 367 class Package(object):
 368         def __init__(self, name, naoki, toolchain=False):
 369                 self.info = find_package_info(name, toolchain)
 370                 self.naoki = naoki
 371
 372                 #self.log.debug("Initialized package object %s" % name)
 373
 374         def __repr__(self):
 375                 return "<Package %s>" % self.info.name
 376
 377         def __cmp__(self, other):
 378                 return cmp(self.name, other.name)
 379
 380         def __getattr__(self, attr):
 381                 return getattr(self.info, attr)
 382
 383         def build(self):
 384                 environment = chroot.Environment(self)
 385                 environment.build()
 386
 387         def download(self):
 388                 download(self.info.objects, logger=self.log)
 389
 390         def extract(self, dest):
 391                 files = [os.path.join(PACKAGESDIR, file) for file in self.package_files]
 392                 if not files:
 393                         return
 394
 395                 self.log.debug("Extracting %s..." % files)
 396                 util.do("%s --root=%s %s" % (os.path.join(TOOLSDIR, "decompressor"),
 397                         dest, " ".join(files)), shell=True)
 398
 399         @property
 400         def log(self):
 401                 return self.naoki.logging.getBuildLogger(self.info.id)
 402
 403
 404 def get_repositories(toolchain=False):
 405         if toolchain:
 406                 return [Repository("toolchain")]
 407
 408         repos = []
 409         for repo in os.listdir(PKGSDIR):
 410                 if os.path.isdir(os.path.join(PKGSDIR, repo)):
 411                         repos.append(repo)
 412
 413         repos.remove("toolchain")
 414
 415         return [Repository(repo) for repo in repos]
 416
 417 class Repository(object):
 418         def __init__(self, name):
 419                 self.name = name
 420
 421         def __repr__(self):
 422                 return "<Repository %s>" % self.name
 423
 424         @property
 425         def packages(self):
 426                 packages = []
 427                 for package in os.listdir(self.path):
 428                         package = PackageInfo(package, repo=self)
 429                         packages.append(package)
 430
 431                 return packages
 432
 433         @property
 434         def package_names(self):
 435                 return [package.name for package in self.packages]
 436
 437         @property
 438         def path(self):
 439                 return os.path.join(PKGSDIR, self.name)
 440
 441 if __name__ == "__main__":
 442         pi = PackageInfo("core/grub")
 443
 444         print pi.dependencies