naoki/backend.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import urlgrabber
   5 import urllib
   6
   7 import chroot
   8 import util
   9
  10 from constants import *
  11
  12 __cache = {
  13         "package_names" : None,
  14         "group_names" : None,
  15 }
  16
  17 def get_package_names(toolchain=False):
  18         if not __cache["package_names"]:
  19                 names = []
  20                 for repo in get_repositories(toolchain):
  21                         names.extend(repo.package_names)
  22
  23                 __cache["package_names"] = sorted(names)
  24
  25         return __cache["package_names"]
  26
  27 def get_group_names():
  28         if not __cache["group_names"]:
  29                 groups = []
  30                 for package in get_package_names():
  31                         package = PackageInfo(package)
  32                         if not package.group in groups:
  33                                 groups.append(package.group)
  34
  35                 __cache["group_names"] = sorted(groups)
  36
  37         return __cache["group_names"]
  38
  39 def find_package_name(name, toolchain=False):
  40         if name in get_package_names(toolchain):
  41                 return name
  42
  43         for package in get_package_names(toolchain):
  44                 if os.path.basename(package) == name:
  45                         return package
  46
  47 def depsolve(packages, recursive=False):
  48         deps = []
  49         for package in packages:
  50                 if not package in deps:
  51                         deps.append(package)
  52
  53         if not recursive or not deps:
  54                 return deps
  55
  56         while True:
  57                 length = len(deps)
  58                 for dep in deps[:]:
  59                         deps.extend(dep.dependencies)
  60
  61                 new_deps = []
  62                 for dep in deps:
  63                         if not dep in new_deps:
  64                                 new_deps.append(dep)
  65
  66                 deps = new_deps
  67
  68                 if length == len(deps):
  69                         break
  70
  71         deps.sort()
  72         return deps
  73
  74 def deptree(packages):
  75         ret = [packages]
  76
  77         while True:
  78                 next = []
  79                 stage = ret[-1][:]
  80                 for package in stage[:]:
  81                         for dep in package.info.dependencies_all:
  82                                 if dep in ret[-1]:
  83                                         stage.remove(package)
  84                                         next.append(package)
  85                                         break
  86
  87                 ret[-1] = stage
  88                 if next:
  89                         ret.append(next)
  90                         continue
  91
  92                 break
  93
  94         return ret
  95
  96 def depsort(packages):
  97         ret = []
  98         for l1 in deptree(packages):
  99                 ret.extend(l1)
 100         return ret
 101
 102 def download(files, logger=None):
 103         for file in files:
 104                 filepath = os.path.join(TARBALLDIR, file)
 105
 106                 if not os.path.exists(TARBALLDIR):
 107                         os.makedirs(TARBALLDIR)
 108
 109                 if os.path.exists(filepath):
 110                         continue
 111
 112                 url = config["sources_download_url"] + "/%s" % urllib.pathname2url(file)
 113
 114                 if logger:
 115                         logger.debug("Retrieving %s" % url)
 116
 117                 g = urlgrabber.grabber.URLGrabber(
 118                         user_agent = "%sSourceGrabber/%s" % (config["distro_name"], config["distro_version"],),
 119                         progress_obj = urlgrabber.progress.TextMeter(),
 120                 )
 121
 122                 try:
 123                         gobj = g.urlopen(url)
 124                 except urlgrabber.grabber.URLGrabError, e:
 125                         logger.error("Could not retrieve %s - %s" % (url, e))
 126                         raise
 127
 128                 # XXX Need to check SHA1 sum here
 129
 130                 fobj = open(filepath, "w")
 131                 fobj.write(gobj.read())
 132                 fobj.close()
 133                 gobj.close()
 134
 135 class PackageInfo(object):
 136         __data = {}
 137
 138         def __init__(self, name):
 139                 self._name = name
 140
 141         def __repr__(self):
 142                 return "<PackageInfo %s>" % self.name
 143
 144         def get_data(self):
 145                 if not self.__data.has_key(self.name):
 146                         self.__data[self.name] = self.fetch()
 147
 148                 return self.__data[self.name]
 149
 150         def set_data(self, data):
 151                 self.__data[self.name] = data
 152
 153         _data = property(get_data, set_data)
 154
 155         def fetch(self):
 156                 env = os.environ.copy()
 157                 env.update(config.environment)
 158                 env["PKGROOT"] = PKGSDIR
 159                 output = util.do("make -f %s" % self.filename, shell=True,
 160                         cwd=os.path.join(PKGSDIR, self.name), returnOutput=1, env=env)
 161
 162                 ret = {}
 163                 for line in output.splitlines():
 164                         a = line.split("=", 1)
 165                         if not len(a) == 2: continue
 166                         key, val = a
 167                         ret[key] = val.strip("\"")
 168
 169                 ret["FINGERPRINT"] = self.fingerprint
 170
 171                 return ret
 172
 173         def fmtstr(self, s):
 174                 return s % self.all
 175
 176         def getPackage(self, naoki):
 177                 return Package(self.name, naoki)
 178
 179         @property
 180         def all(self):
 181                 return {
 182                         "description" : self.description,
 183                         "filename"    : self.filename,
 184                         "fingerprint" : self.fingerprint,
 185                         "group"       : self.group,
 186                         "license"     : self.license,
 187                         "maintainer"  : self.maintainer,
 188                         "name"        : self.name,
 189                         "objects"     : self.objects,
 190                         "patches"     : self.patches,
 191                         "release"     : self.release,
 192                         "summary"     : self.summary,
 193                         "version"     : self.version,
 194                 }
 195
 196         def _dependencies(self, s, recursive=False):
 197                 c = s + "_CACHE"
 198                 if not self._data.has_key(c):
 199                         deps = []
 200                         for name in self._data.get(s).split(" "):
 201                                 name = find_package_name(name)
 202                                 if name:
 203                                         deps.append(Dependency(name))
 204
 205                         self._data.update({c : depsolve(deps, recursive)})
 206
 207                 return self._data.get(c)
 208
 209         @property
 210         def dependencies(self):
 211                 return self._dependencies("PKG_DEPENDENCIES")
 212
 213         @property
 214         def dependencies_build(self):
 215                 return self._dependencies("PKG_BUILD_DEPENDENCIES")
 216
 217         @property
 218         def dependencies_all(self):
 219                 return depsolve(self.dependencies + self.dependencies_build, recursive=True)
 220
 221         @property
 222         def description(self):
 223                 return self._data.get("PKG_DESCRIPTION")
 224
 225         @property
 226         def filename(self):
 227                 return os.path.join(PKGSDIR, self.name, os.path.basename(self.name)) + ".nm"
 228
 229         @property
 230         def fingerprint(self):
 231                 return "%d" % os.stat(self.filename).st_mtime
 232
 233         @property
 234         def group(self):
 235                 return self._data.get("PKG_GROUP")
 236
 237         @property
 238         def id(self):
 239                 return "%s-%s-%s" % (self.name, self.version, self.release)
 240
 241         @property
 242         def license(self):
 243                 return self._data.get("PKG_LICENSE")
 244
 245         @property
 246         def maintainer(self):
 247                 return self._data.get("PKG_MAINTAINER")
 248
 249         @property
 250         def name(self):
 251                 return self._name
 252
 253         @property
 254         def objects(self):
 255                 return self._data.get("PKG_OBJECTS").split(" ")
 256
 257         @property
 258         def package_files(self):
 259                 return self._data.get("PKG_PACKAGES_FILES").split(" ")
 260
 261         @property
 262         def patches(self):
 263                 return self._data.get("PKG_PATCHES").split(" ")
 264
 265         @property
 266         def release(self):
 267                 return self._data.get("PKG_REL")
 268
 269         @property
 270         def summary(self):
 271                 return self._data.get("PKG_SUMMARY")
 272
 273         @property
 274         def version(self):
 275                 return self._data.get("PKG_VER")
 276
 277
 278 class Dependency(PackageInfo):
 279         def __repr__(self):
 280                 return "<Dependency %s>" % self.name
 281
 282
 283 class Package(object):
 284         def __init__(self, name, naoki):
 285                 self.info = PackageInfo(name)
 286                 self.naoki = naoki
 287
 288                 #self.log.debug("Initialized package object %s" % name)
 289
 290         def build(self):
 291                 environment = chroot.Environment(self)
 292                 environment.build()
 293
 294         def download(self):
 295                 download(self.info.objects, logger=self.log)
 296
 297         def extract(self, dest):
 298                 files = [os.path.join(PACKAGESDIR, file) for file in self.info.package_files]
 299                 if not files:
 300                         return
 301
 302                 self.log.debug("Extracting %s..." % files)
 303                 util.do("%s --root=%s %s" % (os.path.join(TOOLSDIR, "decompressor"),
 304                         dest, " ".join(files)), shell=True)
 305
 306         @property
 307         def log(self):
 308                 return self.naoki.logging.getBuildLogger(self.info.id)
 309
 310
 311 def get_repositories(toolchain=False):
 312         if toolchain:
 313                 return Repository("toolchain")
 314
 315         repos = []
 316         for repo in os.listdir(PKGSDIR):
 317                 if os.path.isdir(os.path.join(PKGSDIR, repo)):
 318                         repos.append(repo)
 319
 320         repos.remove("toolchain")
 321
 322         return [Repository(repo) for repo in repos]
 323
 324 class Repository(object):
 325         def __init__(self, name):
 326                 self.name = name
 327
 328         def __repr__(self):
 329                 return "<Repository %s>" % self.name
 330
 331         @property
 332         def packages(self):
 333                 packages = []
 334                 for package in os.listdir(self.path):
 335                         package = PackageInfo(os.path.join(self.name, package))
 336                         packages.append(package)
 337
 338                 return packages
 339
 340         @property
 341         def package_names(self):
 342                 return [package.name for package in self.packages]
 343
 344         @property
 345         def path(self):
 346                 return os.path.join(PKGSDIR, self.name)
 347
 348 if __name__ == "__main__":
 349         pi = PackageInfo("core/grub")
 350
 351         print pi.dependencies