From: Michael Tremer Date: Sun, 12 Sep 2010 16:31:29 +0000 (+0200) Subject: naoki: A bigger refactoring to get things clean. X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=246d1051d51a71fd6bc003c667946ecdf08cf9ae;p=ipfire-3.x.git naoki: A bigger refactoring to get things clean. This references #53. --- diff --git a/config/architectures.conf b/config/architectures.conf index 22b65e406..e01a00823 100644 --- a/config/architectures.conf +++ b/config/architectures.conf @@ -1,3 +1,7 @@ +[default] + +; The default architecture +default = i686 [i686] diff --git a/naoki/__init__.py b/naoki/__init__.py index 8c02d4a6a..a4180c9aa 100644 --- a/naoki/__init__.py +++ b/naoki/__init__.py @@ -7,12 +7,12 @@ import random import sys import time -import backend +import architectures import build -import chroot import environ +import generators import logger -import repo +import repositories import terminal import util @@ -47,27 +47,40 @@ class Naoki(object): "package" : self.call_package, "source" : self.call_source, "shell" : self.call_shell, - "repository" : self.call_repository, "generate" : self.call_generate, "batch" : self.call_batch, } return actionmap[args.action.name](args.action) + def _get_source_repos(self, arch=None): + if not arch: + arches = architectures.Architectures() + arch = arches.get_default() + + return repositories.SourceRepositories(arch=arch) + def call_build(self, args): - builder = build.Builder() + # Source repository + repo = self._get_source_repos() + + # Initialize job queue + jobs = build.Jobs() if args.all: raise Exception, "XXX to be implemented" else: for name in args.packages: - p = repo.find_source_package(name) + p = repo.find_package_by_name(name) if not p: raise Exception, "Could not find package: %s" % name - builder.add(p) + p = build.Build(p) + jobs.add(p) - return builder.run(ignore_dependency_errors=args.ignore_dependency_errors) + #return builder.run(ignore_dependency_errors=args.ignore_dependency_errors) + while jobs.has_jobs: + jobs.process_next() def call_package(self, args): if not args.has_key("action"): @@ -77,14 +90,16 @@ class Naoki(object): actionmap = { "info" : self.call_package_info, "list" : self.call_package_list, - "tree" : self.call_package_tree, "groups" : self.call_package_groups, } return actionmap[args.action.name](args.action) def call_package_info(self, args): - for package in backend.parse_package_info(args.packages): + # Source repositories + repo = self._get_source_repos() + + for package in repo.packages: if args.long: print package.fmtstr("""\ -------------------------------------------------------------------------------- @@ -117,13 +132,15 @@ Release : %(release)s """) def call_package_list(self, args): - for package in backend.parse_package_info(backend.get_package_names()): + repo = self._get_source_repos() + + for package in repo.packages: # Skip unbuilt packages if we want built packages - if args.built and not package.built: + if args.built and not package.is_built: continue # Skip built packages if we want unbuilt only - if args.unbuilt and package.built: + if args.unbuilt and package.is_built: continue if args.long: @@ -131,12 +148,11 @@ Release : %(release)s else: print package.name - def call_package_tree(self, args): - print backend.deptree(backend.parse_package(backend.get_package_names(), naoki=self)) - def call_package_groups(self, args): - groups = backend.get_group_names() - print "\n".join(groups) + # XXX + #groups = backend.get_group_names() + #print "\n".join(groups) + pass def call_source(self, args): if not args.has_key("action"): @@ -146,69 +162,50 @@ Release : %(release)s actionmap = { "download" : self.call_source_download, "upload" : self.call_source_upload, - "clean" : self.call_source_clean, } return actionmap[args.action.name](args.action) def call_source_download(self, args): - for package in backend.parse_package(args.packages or \ - backend.get_package_names(), naoki=self): - package.download() + repo = self._get_source_repos() - def call_source_upload(self, args): - pass # TODO + for package in repo.packages: + if args.packages: + if not package.name in args.packages: + continue - def call_source_clean(self, args): - self.log.info("Remove all unused files") - files = os.listdir(TARBALLDIR) - for package in backend.parse_package_info(backend.get_package_names()): - for object in package.objects: - if object in files: - files.remove(object) + package.source_donwload() - for file in sorted(files): - self.log.info("Removing %s..." % file) - os.remove(os.path.join(TARBALLDIR, file)) + def call_source_upload(self, args): + pass # TODO def call_shell(self, args): - p = repo.find_source_package(args.package) - if not p: - raise Exception, "Could not find package: %s" % args.package - - build_set = build.BuildSet(p) - - return build_set.shell() + # Load architecture set + arches = architectures.Architectures() - def call_repository(self, args): - actionmap = { - "clean" : self.call_repository_clean, - "build" : self.call_repository_build, - } - - return actionmap[args.action.name](args.action) + # Choose default architecture + arch = arches.get_default() - def call_repository_clean(self, repo, args): - if args.names == ["all"]: - args.names = [r.name for r in backend.get_repositories()] + # Load all source packages + repo = repositories.SourceRepositories(arch=arch) - for name in args.names: - repo = backend.BinaryRepository(name, naoki=self) - repo.clean() + # Pick the one we need + p = repo.find_package_by_name(args.package) + if not p: + raise Exception, "Could not find package: %s" % args.package - def call_repository_build(self, args): - if args.names == ["all"]: - args.names = [r.name for r in backend.get_repositories()] + # Initialize and run the shell + shell = build.PackageShell(p) - for name in args.names: - repo = backend.BinaryRepository(name, naoki=self) - repo.build() + return shell.shell() def call_generate(self, args): if not args.type in ("iso",): return - gen = chroot.Generator(self, arches.current, args.type) + arch = architectures.Architectures().get_default() + + gen = generators.Generator(args.type, arch) return gen.run() def call_batch(self, args): @@ -230,31 +227,35 @@ Release : %(release)s return actionmap[args.action.name](args.action) def call_batch_cron(self, args): - packages = [] - packages_may = [] - for package in backend.parse_package_info(backend.get_package_names()): - if not package.built and package.buildable: - packages.append(package) - continue - - # If package was altered since last build - if package.last_change >= package.last_build: - packages.append(package) - continue - - if package.buildable: - packages_may.append(package) + pkgs = [] + candidates = [] - packages_may = sorted(packages_may, key=lambda p: p.last_build) + # Choose architecture + arches = architectures.Architectures() + arch = arches.get_default() - while len(packages) < 10 and packages_may: - package = packages_may.pop(0) - packages.append(package) - - # Bad hack because we lack a _build method - args.packages = [p.name for p in packages] - args.onlydeps = False - args.withdeps = False - args.shell = False - - self.call_build(args) + repo = repositories.SourceRepositories(arch=arch) + for package in repo.packages: + if not package.is_built: + pkgs.append(package) + else: + candidates.append(package) + + # Initialize a job queue + jobs = build.Jobs() + + # Add all unbuilt packages to the job queue + for package in pkgs: + package = build.Build(package) + jobs.add(package) + + # If we have less than ten packages in the queue we add some random + # ones + need_counter = 10 - len(jobs) + if need_counter >= 0: + for candidate in random.sample(candidates, need_counter): + candidate = build.Build(candidate) + jobs.add(candidate) + + while jobs.has_jobs: + jobs.process_next() diff --git a/naoki/arches.py b/naoki/arches.py deleted file mode 100644 index e6d1253de..000000000 --- a/naoki/arches.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/python - -from ConfigParser import ConfigParser - -from constants import * - -_architectures = [] - -def all(): - return _architectures - -def get(name): - for arch in _architectures: - if arch.name == name: - return arch - - raise Exception, "Architecture was not found: %s" % name - -def get_default(): - for a in _architectures: - if a.default: - return a - - return "i686" # XXX for now - -def set_default(name): - for a in _architectures: - if a.name == name: - a.default = True - else: - a.default = False - -def read(configuration_file): - p = ConfigParser() - p.read(configuration_file) - - for arch in p.sections(): - settings = {} - for key, val in p.items(arch): - settings[key] = val - a = Architecture(arch, **settings) - _architectures.append(a) - -class Architecture(object): - def __init__(self, name, **settings): - self.name = name - - self.settings = { - "default" : False, - "package_dir" : os.path.join(PACKAGESDIR, self.name), - } - self.settings.update(settings) - - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.name) - - def __getattr__(self, key): - try: - return self.settings[key] - except KeyError: - raise ValueError - - def get_default(self): - return self.settings["default"] - - def set_default(self, val): - self.settings["default"] - - default = property(get_default, set_default) - - -read(config["architecture_config"]) - -if __name__ == "__main__": - for a in _architectures: - print a, a.settings diff --git a/naoki/architectures.py b/naoki/architectures.py new file mode 100644 index 000000000..ec9b85798 --- /dev/null +++ b/naoki/architectures.py @@ -0,0 +1,136 @@ +#!/usr/bin/python + +import logging +import os.path + +from ConfigParser import ConfigParser, DEFAULTSECT + +from constants import * +from decorators import * + +class Architecture(object): + def __init__(self, name, **settings): + + self._settings = { + "default" : False, + "name" : name, + } + self._settings.update(settings) + + logging.debug("Set up new architecture: %s" % self._settings) + + def __repr__(self): + s = "<%s %s" % (self.__class__.__name__, self._settings["name"]) + if self.default: + s += " (default)" + s += ">" + return s + + @property + def buildable(self): + """ Check if this architecture is buildable on the local host""" + + return True # TODO + + def get_default(self): + return self._settings["default"] + + def set_default(self, value): + self._settings["default"] = value + + default = property(get_default, set_default) + + def __getattr__(self, attr): + try: + return self._settings[attr] + except KeyError: + raise AttributeError, attr + + +@singleton +class Architectures(object): + def __init__(self): + self._architectures = [] + + # Try to read the default architectures + self.read(ARCHES_DEFAULT) + + def read(self, filename): + logging.debug("Reading architectures from %s" % filename) + + if not os.path.exists(filename): + return + + p = ConfigParser() + p.read(filename) + + default = {} + + for arch in p.sections(): + if arch == "default": + default = p.items(arch) + continue + + settings = {} + for key, val in p.items(arch): + settings[key] = val + + a = Architecture(arch, **settings) + self.add_architecture(a) + + for key, val in default: + if key == "default": + self.default = val + + def add_architecture(self, arch): + assert isinstance(arch, Architecture) + + self._architectures.append(arch) + + def get_default(self): + for a in self._architectures: + if a.default: + return a + + raise Exception, "Cannot find default architecture" + + def set_default(self, arch): + if arch is None: + return + + if not arch in [a.name for a in self.all]: + raise Exception, "Cannot set default architecture: Unknown architecture %s" % arch + + logging.debug("Setting default architecture: %s" % arch) + + for a in self._architectures: + if a.name == arch: + a.default = True + else: + a.default = False + + default = property(get_default, set_default) + + def get(self, name): + for arch in self._architectures: + if arch.name == name: + return arch + + raise Exception, "Could not find arch: %s" % name + + @property + def all(self): + return self._architectures[:] + + +if __name__ == "__main__": + arches = Architectures() + + for arch in arches.all: + print "Name: %s" % arch.name + for key, val in arch._settings.items(): + print " %-20s : %s" % (key, val) + + print arches.get("i686") + print arches.default + diff --git a/naoki/backend.py b/naoki/backend.py deleted file mode 100644 index 2753ea5ae..000000000 --- a/naoki/backend.py +++ /dev/null @@ -1,676 +0,0 @@ -#!/usr/bin/python - - -import os -import shutil -import smtplib -import urlgrabber -import urlgrabber.progress -import urllib - -import chroot -import util - -from exception import * -from constants import * - -__cache = { - "package_names" : None, - "group_names" : None, -} - -# Python 2.4 does not have that email module, so -# we disable the mail function here. -try: - import email.mime.multipart - import email.mime.text - have_email = 1 -except ImportError: - have_email = 0 - -try: - import hashlib - have_hashlib = 1 -except ImportError: - import sha - have_hashlib = 0 - -def find_package_info(name, toolchain=False, **kwargs): - for repo in get_repositories(toolchain): - if not os.path.exists(os.path.join(repo.path, name, name + ".nm")): - continue - - return PackageInfo(name, repo=repo, **kwargs) - -def find_package(name, naoki, toolchain=False): - package = find_package_info(name, toolchain) - if package: - return package.getPackage(naoki) - - return None - -def parse_package_info(names, toolchain=False, **kwargs): - packages = [] - for name in names: - package = find_package_info(name, toolchain, **kwargs) - if package: - packages.append(package) - - return packages - -def parse_package(names, toolchain=False, naoki=None): - packages = parse_package_info(names, toolchain) - - return [Package(package.name, naoki=naoki, toolchain=toolchain) \ - for package in packages] - -def get_package_names(toolchain=False): - if not __cache["package_names"]: - names = [] - for repo in get_repositories(toolchain): - names.extend(repo.package_names) - - __cache["package_names"] = sorted(names) - - return __cache["package_names"] - -def get_group_names(): - if not __cache["group_names"]: - groups = [] - for package in parse_package_info(get_package_names()): - if not package.group in groups: - groups.append(package.group) - - __cache["group_names"] = sorted(groups) - - return __cache["group_names"] - -def find_package_name(name, toolchain=False): - if name in get_package_names(toolchain): - return name - - for package in get_package_names(toolchain): - if os.path.basename(package) == name: - return package - -def depsolve(packages, recursive=False, build=False, toolchain=False): - deps = [] - for package in packages: - if not package in deps: - deps.append(package) - - if not recursive or not deps: - return deps - - while True: - length = len(deps) - for dep in deps[:]: - deps.extend(dep.dependencies) - if build and not toolchain: - deps.extend(dep.dependencies_build) - - new_deps = [] - for dep in deps: - if not dep in new_deps: - new_deps.append(dep) - - deps = new_deps - - if length == len(deps): - break - - deps.sort() - return deps - -def deptree(packages, toolchain=False): - ret = [packages] - - while True: - next = [] - stage = ret[-1][:] - for package in stage[:]: - for dep in package.dependencies_all: - if dep in ret[-1]: - stage.remove(package) - next.append(package) - break - - ret[-1] = stage - if next: - ret.append(next) - continue - - break - - return ret - -def depsort(packages, toolchain=False): - ret = [] - for l1 in deptree(packages, toolchain=toolchain): - ret.extend(l1) - return ret - -def calc_hash(data): - if have_hashlib: - obj = hashlib.sha1(data) - else: - obj = sha.new(data) - - return obj.hexdigest() - -def download(files, logger=None): - for file in files: - filepath = os.path.join(TARBALLDIR, file) - - if not os.path.exists(TARBALLDIR): - os.makedirs(TARBALLDIR) - - if os.path.exists(filepath): - continue - - url = config["sources_download_url"] + "/" + file - - if logger: - logger.debug("Retrieving %s" % url) - - g = urlgrabber.grabber.URLGrabber( - user_agent = "%sSourceGrabber/%s" % (config["distro_name"], config["distro_version"],), - progress_obj = urlgrabber.progress.TextMeter(), - quote=0, - ) - - try: - gobj = g.urlopen(url) - except urlgrabber.grabber.URLGrabError, e: - if logger: - logger.error("Could not retrieve %s - %s" % (url, e)) - raise - - data = gobj.read() - gobj.close() - - if gobj.hdr.has_key("X-Hash-Sha1"): - hash_server = gobj.hdr["X-Hash-Sha1"] - msg = "Comparing hashes - %s" % hash_server - - hash_calculated = calc_hash(data) - if hash_calculated == hash_server: - if logger: - logger.debug(msg + " - OK") - else: - if logger: - logger.error(msg + " - ERROR") - raise DownloadError, "Hash sum of downloaded file does not match" - - fobj = open(filepath, "w") - fobj.write(data) - fobj.close() - - -class PackageInfo(object): - __data = {} - - def __init__(self, name, repo=None, arch="i686"): - self._name = name - self.repo = repo - - self.arch = arch - - def __cmp__(self, other): - return cmp(self.name, other.name) - - def __repr__(self): - return "" % self.name - - def get_data(self): - if not self.__data.has_key(self.name): - self.__data[self.name] = self.fetch() - - return self.__data[self.name] - - def set_data(self, data): - self.__data[self.name] = data - - _data = property(get_data, set_data) - - def fetch(self): - env = os.environ.copy() - env.update(config.environment) - env.update({ - "PKG_ARCH" : self.arch, - "PKGROOT" : PKGSDIR, - }) - output = util.do("make -f %s" % self.filename, shell=True, - cwd=os.path.join(PKGSDIR, self.repo.name, self.name), returnOutput=1, env=env) - - ret = {} - for line in output.splitlines(): - a = line.split("=", 1) - if not len(a) == 2: continue - key, val = a - ret[key] = val.strip("\"") - - ret["FINGERPRINT"] = self.fingerprint - - return ret - - def fmtstr(self, s): - return s % self.all - - def getPackage(self, naoki): - return Package(self.name, naoki) - - @property - def all(self): - return { - "build_deps" : [dep.name for dep in self.dependencies_build], - "deps" : [dep.name for dep in self.dependencies], - "description" : self.description, - "filename" : self.filename, - "fingerprint" : self.fingerprint, - "files" : self.package_files, - "group" : self.group, - "id" : self.id, - "license" : self.license, - "maintainer" : self.maintainer, - "name" : self.name, - "objects" : self.objects, - "patches" : self.patches, - "release" : self.release, - "summary" : self.summary, - "url" : self.url, - "version" : self.version, - } - - @property - def buildable(self): - return self.dependencies_unbuilt == [] - - @property - def built(self): - for file in self.package_files: - if not os.path.exists(os.path.join(PACKAGESDIR, file)): - return False - - return True - - def _dependencies(self, s, recursive=False, toolchain=False): - c = s + "_CACHE" - if not self._data.has_key(c): - deps = parse_package_info(self._data.get(s).split(" "), toolchain=toolchain) - self._data.update({c : depsolve(deps, recursive)}) - - return self._data.get(c) - - @property - def dependencies(self): - if self.__toolchain: - return self.dependencies_toolchain - - return self._dependencies("PKG_DEPENDENCIES") - - @property - def dependencies_build(self): - return self._dependencies("PKG_BUILD_DEPENDENCIES") - - @property - def dependencies_built(self): - ret = [] - for dep in self.dependencies_all: - if dep.built: - ret.append(dep) - - return ret - - @property - def dependencies_unbuilt(self): - ret = [] - for dep in self.dependencies_all: - if not dep.built: - ret.append(dep) - - return ret - - @property - def dependencies_all(self): - deps = self.dependencies - if not self.__toolchain: - deps.extend(self.dependencies_build) - return depsolve(deps, build=True, recursive=True, toolchain=self.__toolchain) - - @property - def dependencies_toolchain(self): - return self._dependencies("PKG_TOOLCHAIN_DEPENDENCIES", toolchain=True) - - @property - def dependencies_reversed(self): - deps = [] - for package in parse_package_info(get_package_names()): - if self.name in [dep.name for dep in package.dependencies]: - deps.append(package.name) - - return deps - - @property - def description(self): - return self._data.get("PKG_DESCRIPTION") - - @property - def filename(self): - return os.path.join(PKGSDIR, self.repo.name, self.name, - os.path.basename(self.name)) + ".nm" - - @property - def fingerprint(self): - return "%d" % self.last_change - - @property - def group(self): - return self._data.get("PKG_GROUP") - - @property - def id(self): - return "%s-%s-%s" % (self.name, self.version, self.release) - - @property - def last_build(self): - file = os.path.join(PACKAGESDIR, self.package_files[0]) - if not os.path.exists(file): - return 0 - - return os.stat(file).st_mtime - - @property - def last_change(self): - return os.stat(self.filename).st_mtime - - @property - def license(self): - return self._data.get("PKG_LICENSE") - - @property - def maintainer(self): - return self._data.get("PKG_MAINTAINER") - - @property - def name(self): - return self._name - - @property - def objects(self): - return self._data.get("PKG_OBJECTS").split(" ") - - @property - def package_files(self): - return self._data.get("PKG_PACKAGES_FILES").split(" ") - - @property - def patches(self): - return self._data.get("PKG_PATCHES").split(" ") - - @property - def release(self): - return self._data.get("PKG_REL") - - @property - def summary(self): - return self._data.get("PKG_SUMMARY") - - @property - def url(self): - return self._data.get("PKG_URL") - - @property - def version(self): - return self._data.get("PKG_VER") - - @property - def __toolchain(self): - return self.repo.name == "toolchain" - - -class Package(object): - def __init__(self, name, naoki, toolchain=False): - self.info = find_package_info(name, toolchain) - - assert naoki - self.naoki = naoki - - #self.log.debug("Initialized package object %s" % name) - - def __repr__(self): - return "" % self.info.name - - def __cmp__(self, other): - return cmp(self.name, other.name) - - def __getattr__(self, attr): - return getattr(self.info, attr) - - @property - def name(self): - return self.info.name - - def build(self): - environment = chroot.PackageEnvironment(self) - environment.build() - - def download(self): - download(self.info.objects, logger=self.log) - - def extract(self, dest): - files = [os.path.join(PACKAGESDIR, file) for file in self.info.package_files] - if not files: - return - - self.log.debug("Extracting %s..." % files) - util.do("%s --root=%s %s" % (os.path.join(TOOLSDIR, "decompressor"), - dest, " ".join(files)), shell=True, logger=self.log) - - def getEnvironment(self, *args, **kwargs): - return chroot.PackageEnvironment(self, *args, **kwargs) - - @property - def logfile(self): - return os.path.join(LOGDIR, self.repo.name, self.info.id) + ".log" - - @property - def log(self): - return self.naoki.logging.getBuildLogger(self) - - -def get_repositories(toolchain=False): - if toolchain: - return [Repository("toolchain")] - - repos = [] - for repo in os.listdir(PKGSDIR): - if os.path.isdir(os.path.join(PKGSDIR, repo)): - repos.append(repo) - - repos.remove("toolchain") - - return [Repository(repo) for repo in repos] - -class Repository(object): - def __init__(self, name): - self.name = name - - def __repr__(self): - return "" % self.name - - @property - def packages(self): - packages = [] - for package in os.listdir(self.path): - package = PackageInfo(package, repo=self) - packages.append(package) - - return packages - - @property - def package_names(self): - return [package.name for package in self.packages] - - @property - def path(self): - return os.path.join(PKGSDIR, self.name) - - -class BinaryRepository(object): - DIRS = ("db", "packages") - - def __init__(self, name, naoki=None, arch=None): - self.name = name - self.arch = arch or arches.current - self.repo = Repository(self.name) - - assert naoki - self.naoki = naoki - - def build(self): - if not self.buildable: - raise Exception, "Cannot build repository" - - # Create temporary directory layout - util.rm(self.repopath("tmp")) - for dir in self.DIRS: - util.mkdir(self.repopath("tmp", dir)) - - # Copy packages - for package in self.packages: - for file in package.package_files: - shutil.copy(os.path.join(PACKAGESDIR, file), - self.repopath("tmp", "packages")) - - # TODO check repository's sanity - # TODO create repoview - f = open(self.repopath("tmp", "db", "package-list.txt"), "w") - for package in self.packages: - s = "%-40s" % package.fmtstr("%(name)s-%(version)s-%(release)s") - s += " | %s\n" % package.summary - f.write(s) - f.close() - - for dir in self.DIRS: - util.rm(self.repopath(dir)) - shutil.move(self.repopath("tmp", dir), self.repopath(dir)) - util.rm(self.repopath("tmp")) - - def clean(self): - if os.path.exists(self.path): - self.log.debug("Cleaning up repository: %s" % self.path) - util.rm(self.path) - - def repopath(self, *args): - return os.path.join(self.path, *args) - - @property - def buildable(self): - for package in self.packages: - if package.built: - continue - return False - - return True - - @property - def log(self): - return self.naoki.log - - @property - def packages(self): - packages = [] - for package in parse_package_info(get_package_names(), arch=self.arch["name"]): - if not package.repo.name == self.name: - continue - packages.append(package) - return packages - - @property - def path(self): - return os.path.join(REPOSDIR, self.name, self.arch["name"]) - -def report_error_by_mail(package): - log = package.naoki.log - - # Do not send a report if no recipient is configured - if not config["error_report_recipient"]: - return - - if not have_email: - log.error("Can't send mail because this python version does not support this") - return - - try: - connection = smtplib.SMTP(config["smtp_server"]) - #connection.set_debuglevel(1) - - if config["smtp_user"] and config["smtp_password"]: - connection.login(config["smtp_user"], config["smtp_password"]) - - except smtplib.SMTPConnectError, e: - log.error("Could not establish a connection to the smtp server: %s" % e) - return - except smtplib.SMTPAuthenticationError, e: - log.error("Could not successfully login to the smtp server: %s" % e) - return - - msg = email.mime.multipart.MIMEMultipart() - msg["From"] = config["error_report_sender"] - msg["To"] = config["error_report_recipient"] - msg["Subject"] = config["error_report_subject"] % package.all - msg.preamble = 'You will not see this in a MIME-aware mail reader.\n' - - body = """\ -The package %(name)s had a difficulty to build itself. -This email will give you a short report about the error. - -Package information: - Name : %(name)s - %(summary)s - Version : %(version)s - Release : %(release)s - - This package in maintained by %(maintainer)s. - - -A detailed logfile is attached. - -Sincerely, - Naoki - """ % package.all - - msg.attach(email.mime.text.MIMEText(body)) - - # Read log and append it to mail - loglines = [] - if os.path.exists(package.logfile): - f = open(package.logfile) - line = f.readline() - while line: - line = line.rstrip() - if line.endswith(LOG_MARKER): - # Reset log - loglines = [] - - loglines.append(line) - line = f.readline() - - f.close() - - if not loglines: - loglines = ["Logfile wasn't found."] - - log = email.mime.text.MIMEText("\n".join(loglines), _subtype="plain") - log.add_header('Content-Disposition', 'attachment', - filename="%s.log" % package.id) - msg.attach(log) - - try: - connection.sendmail(config["error_report_sender"], - config["error_report_recipient"], msg.as_string()) - except Exception, e: - log.error("Could not send error report: %s: %s" % (e.__class__.__name__, e)) - return - - connection.quit() diff --git a/naoki/build.py b/naoki/build.py index 5489db8f8..b705cd9ae 100644 --- a/naoki/build.py +++ b/naoki/build.py @@ -1,90 +1,104 @@ #!/usr/bin/python import logging +import uuid -import deps +import dependencies import environ from constants import * from exception import * -class BuildSet(object): + +class Build(object): def __init__(self, package): self.package = package - self.dependency_set = deps.DependencySet(self.package) + # Generate a random, but unique id + self.id = uuid.uuid4() - logging.debug("Successfully created BuildSet for %s" % self.package) + # Create dependency set + self.dependency_set = dependencies.DependencySet(arch=self.arch) - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.package.name) + # Add all mandatory packages and build dependencies + deps = [dependencies.Dependency(p) for p in config["mandatory_packages"]] + deps += self.package.get_dependencies("build") + for package in deps: + self.dependency_set.add_dependency(package) - def _resolve(self, ignore_errors=False): - try: - self.dependency_set.resolve() - except DependencyResolutionError, e: - if ignore_errors: - logging.warning("Ignoring dependency errors: %s" % e) - else: - raise + self.settings = { + "ignore_dependency_errors" : False, + } + + def __repr__(self): + return "<%s %s-%s:%s>" % \ + (self.__class__.__name__, self.id, self.package.name, self.arch.name) @property def arch(self): return self.package.arch - def print_info(self): - logging.info("Building: %s" % self.package.id) + def build(self, **settings): + self.settings.update(settings) + + logging.info("Building: %s (%s)" % (self.package.id, self.id)) + logging.info("") logging.info(" %s" % self.package.summary) logging.info("") - def build(self, ignore_dependency_errors=False): - logging.debug("Running build process for %s" % self) - self.print_info() + # Resolve the dependencies + try: + self.dependency_set.resolve() + except DependencyResolutionError, e: + if self.settings["ignore_dependency_errors"]: + logging.warning("Ignoring dependency errors: %s" % e) + else: + raise - self._resolve(ignore_errors=ignore_dependency_errors) + e = environ.Build(self.package) - env = environ.Environment(self) - env.build() + # Extract all tools + for package in self.dependency_set.packages: + e.extract(package) - run = build + # Build :D + e.build() - def shell(self): - logging.debug("Running shell for %s" % self) - self.print_info() - # Add some packages that are kind of nice in a shell - # like an editor and less... - for dependency in [deps.Dependency(d) for d in config["shell_packages"]]: - logging.debug("Adding shell dependency: %s" % dependency) - self.dependency_set.add_dependency(dependency) +class Jobs(object): + def __init__(self): + self.__jobs = [] + self.__error_jobs = [] - self._resolve() + logging.debug("Initialized jobs queue") - env = environ.Shell(self) - env.shell() + def __len__(self): + return len(self.__jobs) + def add(self, job): + assert isinstance(job, Build) -class Builder(object): - def __init__(self): - self._items = [] - - logging.debug("Successfully created Builder instance") + self.__jobs.append(job) - def add(self, i): - self._items.append(BuildSet(i)) - logging.debug("Added %s to %s" % (i, self)) + @property + def all(self): + return self.__jobs[:] - def _reorder(self): - logging.debug("Reordering BuildSets") + @property + def has_jobs(self): + return self.__jobs != [] - def run(self, *args, **kwargs): - self._reorder() + def process_next(self): + if not self.__jobs: + return - logging.info("Running build process") + job = self.__jobs[0] - while self._items: - i = self._items.pop(0) + try: + job.build() + finally: + self.__jobs.remove(job) - # Run the actual build - i.run(*args, **kwargs) +class PackageShell(environ.Shell): + pass diff --git a/naoki/chroot.py b/naoki/chroot.py deleted file mode 100644 index 3b71c1008..000000000 --- a/naoki/chroot.py +++ /dev/null @@ -1,545 +0,0 @@ -#!/usr/bin/python - -import fcntl -import grp -import logging -import os -import random -import shutil -import stat -import time - -import arches -import backend -import util -from constants import * -from exception import * -from logger import getLog - -class Environment(object): - kernel_version = os.uname()[2] - - def __init__(self, naoki, arch="i686"): - self.arch = arch - self.config = config - self.naoki = naoki - - self.initialized = False - self.__buildroot = None - - self.toolchain = Toolchain(self.arch["name"]) - - # Create initial directory that we can set the lock - util.mkdir(self.chrootPath()) - - # Lock environment. Throws exception if function cannot set the lock. - self.lock() - - def init(self, clean=True): - marker = self.chrootPath(".initialized") - self.log.debug("Initialize environment %s..." % self.chrootPath()) - - if clean: - self.clean() - - # If marker exists, we won't reinit again - if os.path.exists(marker): - return - - # create dirs - dirs = ( - CACHEDIR, - CCACHEDIR, - IMAGESDIR, - PACKAGESDIR, - self.chrootPath("bin"), - self.chrootPath("etc"), - self.chrootPath("lib"), - self.chrootPath("proc"), - self.chrootPath("root"), - self.chrootPath("sbin"), - self.chrootPath("sys"), - self.chrootPath("tmp"), - self.chrootPath("tools_%s" % self.arch["name"]), - self.chrootPath("usr/src/cache"), - self.chrootPath("usr/src/ccache"), - self.chrootPath("usr/src/images"), - self.chrootPath("usr/src/packages"), - self.chrootPath("usr/src/pkgs"), - self.chrootPath("usr/src/src"), - self.chrootPath("usr/src/tools"), - self.chrootPath("var/tmp"), - ) - for item in dirs: - util.mkdir(item) - - # touch files - files = ( - "etc/fstab", - "etc/mtab", - ) - for item in files: - util.touch(self.chrootPath(item)) - - self._setupDev() - self._setupUsers() - self._setupDns() - - self.toolchain.extract(self.chrootPath()) - - self.extractAll() - - self.toolchain.adjust(self.chrootPath()) - - # Set marker - util.touch(marker) - - @property - def buildroot(self): - if not self.__buildroot: - self.__buildroot = "buildroot.%s" % util.random_string() - - return self.__buildroot - - def lock(self): - self.log.debug("Trying to lock environment") - - try: - self._lock = open(self.chrootPath(".lock"), "a+") - except IOError, e: - return 0 - - try: - fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError, e: - raise BuildRootLocked, "Environment is locked by another process" - - return 1 - - def clean(self): - if os.path.exists(self.chrootPath()): - util.rm(self.chrootPath()) - - @property - def environ(self): - env = config.environment.copy() - env.update({ - "HOME" : "/root", - "BASEDIR" : "/usr/src", - "PKGROOT" : "/usr/src/pkgs", - "TOOLS_DIR" : "/tools_%s" % self.arch["name"], - "TARGET" : "%s-ipfire-linux-gnu" % self.arch["machine"], - "TARGET_MACHINE" : self.arch["machine"], - "PATH" : CHROOT_PATH + ":/tools_%(arch)s/sbin:/tools_%(arch)s/bin" \ - % { "arch" : self.arch["name"], }, - "BUILDROOT" : "/%s" % self.buildroot, - "CHROOT" : "1", - "CFLAGS" : self.arch["cflags"], - "CXXFLAGS" : self.arch["cxxflags"], - "PKG_ARCH" : self.arch["name"], - }) - - ccache_path = os.path.join("tools_%s" % self.arch["name"], - "usr", "ccache", "bin") - if os.path.exists(self.chrootPath(ccache_path)): - env.update({ - "PATH" : "/%s:%s" % (ccache_path, env["PATH"]), - "CCACHE_COMPILERCHECK" : "none", - "CCACHE_COMPRESS" : "1", - "CCACHE_DIR" : "/usr/src/ccache", - }) - - return env - - def doChroot(self, command, shell=True, *args, **kwargs): - ret = None - try: - env = self.environ - - if kwargs.has_key("env"): - env.update(kwargs.pop("env")) - - self._mountall() - - if not kwargs.has_key("chrootPath"): - kwargs["chrootPath"] = self.chrootPath() - - ret = util.do(command, personality=self.arch["personality"], - shell=shell, env=env, logger=self.log, *args, **kwargs) - - finally: - self._umountall() - - return ret - - def chrootPath(self, *args): - raise NotImplementedError - - def _setupDev(self): - self.log.debug("Setting up /dev and /proc") - - # files in /dev - util.rm(self.chrootPath("dev")) - util.mkdir(self.chrootPath("dev", "pts")) - util.mkdir(self.chrootPath("dev", "shm")) - prevMask = os.umask(0000) - - devNodes = [ - (stat.S_IFCHR | 0666, os.makedev(1, 3), "dev/null"), - (stat.S_IFCHR | 0666, os.makedev(1, 7), "dev/full"), - (stat.S_IFCHR | 0666, os.makedev(1, 5), "dev/zero"), - (stat.S_IFCHR | 0666, os.makedev(1, 8), "dev/random"), - (stat.S_IFCHR | 0444, os.makedev(1, 9), "dev/urandom"), - (stat.S_IFCHR | 0666, os.makedev(5, 0), "dev/tty"), - (stat.S_IFCHR | 0600, os.makedev(5, 1), "dev/console") - ] - - # make device node for el4 and el5 - if self.kernel_version < "2.6.19": - devNodes.append((stat.S_IFCHR | 0666, os.makedev(5, 2), "dev/ptmx")) - - for i in devNodes: - # create node - os.mknod(self.chrootPath(i[2]), i[0], i[1]) - - os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin")) - os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout")) - os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr")) - os.symlink("/proc/self/fd", self.chrootPath("dev", "fd")) - - if self.kernel_version >= "2.6.19": - os.symlink("/dev/pts/ptmx", self.chrootPath("dev", "ptmx")) - - os.umask(prevMask) - - def _setupUsers(self): - self.log.debug("Creating users") - f = open(self.chrootPath("etc", "passwd"), "w") - f.write("root:x:0:0:root:/root:/bin/bash\n") - f.write("nobody:x:99:99:Nobody:/:/sbin/nologin\n") - f.close() - - f = open(self.chrootPath("etc", "group"), "w") - f.write("root:x:0:root\n") - f.write("nobody:x:99:\n") - f.close() - - def _setupDns(self): - self.log.debug("Setting up DNS") - nameservers = [] - f = open("/etc/resolv.conf") - for line in f.readlines(): - if line.startswith("nameserver"): - nameservers.append(line.split(" ")[-1]) - f.close() - - self.log.debug("Using nameservers: %s" % nameservers) - - f = open(self.chrootPath("etc", "resolv.conf"), "w") - for nameserver in nameservers: - f.write("nameserver %s" % nameserver) - f.close() - - self.log.debug("Creating record for localhost") - f = open(self.chrootPath("etc", "hosts"), "w") - f.write("127.0.0.1 localhost\n") - f.close() - - def _mountall(self): - """mount 'normal' fs like /dev/ /proc/ /sys""" - self.log.debug("Mounting chroot") - for cmd in self.mountCmds: - util.do(cmd, shell=True) - - def _umountall(self): - """umount all mounted chroot fs.""" - self.log.debug("Umounting chroot") - for cmd in self.umountCmds: - util.do(cmd, raiseExc=0, shell=True) - - @property - def log(self): - return getLog() - - def shell(self, args=[]): - command = "chroot %s /usr/src/tools/chroot-shell %s" % \ - (self.chrootPath(), " ".join(args)) - - for key, val in self.environ.items(): - command = "%s=\"%s\" " % (key, val) + command - - try: - self._mountall() - - shell = os.system(command) - return os.WEXITSTATUS(shell) - - finally: - self._umountall() - - @property - def umountCmds(self): - ret = ( - "umount -n %s" % self.chrootPath("proc"), - "umount -n %s" % self.chrootPath("sys"), - "umount -n %s" % self.chrootPath("usr", "src", "cache"), - "umount -n %s" % self.chrootPath("usr", "src", "ccache"), - "umount -n %s" % self.chrootPath("usr", "src", "images"), - "umount -n %s" % self.chrootPath("usr", "src", "packages"), - "umount -n %s" % self.chrootPath("usr", "src", "pkgs"), - "umount -n %s" % self.chrootPath("usr", "src", "src"), - "umount -n %s" % self.chrootPath("usr", "src", "tools"), - "umount -n %s" % self.chrootPath("dev", "pts"), - "umount -n %s" % self.chrootPath("dev", "shm") - ) - - return ret - - @property - def mountCmds(self): - ret = [ - "mount -n -t proc naoki_chroot_proc %s" % self.chrootPath("proc"), - "mount -n -t sysfs naoki_chroot_sysfs %s" % self.chrootPath("sys"), - "mount -n --bind %s %s" % (CACHEDIR, self.chrootPath("usr", "src", "cache")), - "mount -n --bind %s %s" % (CCACHEDIR, self.chrootPath("usr", "src", "ccache")), - "mount -n --bind %s %s" % (IMAGESDIR, self.chrootPath("usr", "src", "images")), - "mount -n --bind %s %s" % (PACKAGESDIR, self.chrootPath("usr", "src", "packages")), - "mount -n --bind %s %s" % (PKGSDIR, self.chrootPath("usr", "src", "pkgs")), - "mount -n --bind %s %s" % (os.path.join(BASEDIR, "src"), self.chrootPath("usr", "src", "src")), - "mount -n --bind %s %s" % (TOOLSDIR, self.chrootPath("usr", "src", "tools")), - ] - - mountopt = "gid=%d,mode=0620,ptmxmode=0666" % grp.getgrnam("tty").gr_gid - if self.kernel_version >= "2.6.29": - mountopt += ",newinstance" - - ret.extend([ - "mount -n -t devpts -o %s naoki_chroot_devpts %s" % (mountopt, self.chrootPath("dev", "pts")), - "mount -n -t tmpfs naoki_chroot_shmfs %s" % self.chrootPath("dev", "shm")]) - - return ret - - def extractAll(self): - raise NotImplementedError - - -class PackageEnvironment(Environment): - def __init__(self, package, *args, **kwargs): - self.package = package - - Environment.__init__(self, naoki=package.naoki, *args, **kwargs) - - def build(self): - self.log.debug(LOG_MARKER) - - self.package.download() - - # Save start time - time_start = time.time() - - try: - self.make("package") - except Error: - if config["cleanup_on_failure"]: - self.clean() - backend.report_error_by_mail(self.package) - raise - - time_end = time.time() - self.log.debug("Package build took %.2fs" % (time_end - time_start)) - - if config["cleanup_on_success"]: - self.clean() - - def chrootPath(self, *args): - return os.path.join(BUILDDIR, "environments", self.package.info.id, *args) - - def extractAll(self): - packages = [p.getPackage(self.naoki) \ - for p in self.package.info.dependencies_all] - - for package in packages: - package.extract(self.chrootPath()) - - def make(self, target): - file = "/usr/src%s" % self.package.info.filename[len(BASEDIR):] - - return self.doChroot("make -C %s -f %s %s" % \ - (os.path.dirname(file), file, target), shell=True) - - @property - def log(self): - return self.package.log - - def shell(self, *args): - self.make("prepare") - Environment.shell(self, *args) - self.clean() - - -class ShellEnvironment(Environment): - def chrootPath(self, *args): - return os.path.join(BUILDDIR, "environments", "shell", *args) - - def extractAll(self): - pass - - -class Toolchain(object): - def __init__(self, arch): - util.mkdir(TOOLCHAINSDIR) - - self.arch = arches[arch] - - # Create a filename object - self.filename = "toolchain-%s.%s.tar.gz" % \ - (self.arch["name"], config["toolchain_version"],) - - # Store the path including the filename - self.path = os.path.join(TOOLCHAINSDIR, self.filename) - - self.build_dir = os.path.join(BUILDDIR, "toolchains", - "tools_%s.%s" % (self.arch["name"], config["toolchain_version"])) - - self.log = getLog() - - @property - def exists(self): - return os.path.exists(self.path) - - def download(self): - self.log.info("Downloading toolchain...") - pass - - def cmd(self, args=[]): - cmd = "%s" % os.path.join(TOOLSDIR, "toolchain") - if args: - cmd += " " - cmd += " ".join(args) - util.do(cmd, cwd=self.build_dir, shell=True, logger=self.log, - env={ "TOOLS_DIR" : self.build_dir }) - - # TODO: - # - logging - def make(self, pkg, target): - env = config.environment.copy() - env.update({ - "BASEDIR" : BASEDIR, - "PATH" : "/tools_%(arch)s/sbin:/tools_%(arch)s/bin:%(path)s" % \ - { "arch" : self.arch["name"], "path" : os.environ["PATH"], }, - "PKGROOT" : PKGSDIR, - "ROOT" : self.build_dir, - "TARGET" : "%s-ipfire-linux-gnu" % self.arch["machine"], - "TARGET_MACHINE" : self.arch["machine"], - "TOOLCHAIN" : "1", - "TOOLS_DIR" : "/tools_%s" % self.arch["name"], - - "CFLAGS" : self.arch["cflags"], - "CXXFLAGS" : self.arch["cxxflags"], - }) - - command = "make -C %s -f %s %s" % \ - (os.path.dirname(pkg.filename), pkg.filename, target) - - return util.do(command, shell=True, env=env, personality=self.arch["personality"], - logger=self.log) - - def build_package(self, pkg): - self.log.info("Building %s..." % pkg.name) - - source_dir = os.path.join(self.build_dir, "usr/src") - - util.rm(source_dir) - util.mkdir(source_dir) - - self.checkLink() - - return self.make(pkg, "package") - - def compress(self): - self.cmd(["compress", self.path, self.build_dir]) - - def extract(self, path): - self.cmd(["extract", self.path, os.path.join(path, "tools_%s" % self.arch["name"])]) - - def adjust(self, path): - self.cmd(["adjust", path]) - - def build(self, naoki): - self.log.info("Building toolchain...") - - packages = backend.get_package_names(toolchain=True) - packages = backend.parse_package(packages, toolchain=True, naoki=naoki) - packages = backend.depsort(packages) - for pkg in packages: - if os.path.exists(os.path.join(self.path, pkg.name)): - continue - self.build_package(pkg) - self.compress() - - def checkLink(self): - link = "/tools_%s" % self.arch["name"] - destination = os.path.abspath(self.build_dir) - - if not os.path.islink(link): - # File is not a link. Remove it... - util.rm(link) - - else: - # If link points to correct destination we break up - if os.path.abspath(os.readlink(link)) == destination: - return - os.unlink(link) - - - os.symlink(destination, link) - - -class Generator(Environment): - def __init__(self, naoki, arch, type): - self.type = type - Environment.__init__(self, naoki, arch) - - def chrootPath(self, *args): - return os.path.join(BUILDDIR, "generators", self.type, self.arch["name"], *args) - - def run(self): - self.init() - - # Extracting installer packages - util.mkdir(self.chrootPath("installer")) - - for package in self.get_packages("installer"): - package.extract(self.chrootPath("installer")) - - all_package_files = [] - for package in self.get_packages("all"): - all_package_files.extend(package.package_files) - - self.doChroot("/usr/src/tools/generator %s" % self.type, - env={"ALL_PACKAGES" : " ".join(all_package_files)}) - - def get_packages(self, type): - _packages = { - "all" : backend.get_package_names(), - "build" : [ "arping", "bash", "coreutils", "cpio", "curl", - "dhcp", "findutils", "grep", "iproute2", "iputils", "kbd", - "less", "module-init-tools", "procps", "sed", "sysvinit", - "udev", "util-linux-ng", "which", "dvdrtools", "kernel", - "squashfs-tools", "syslinux", "zlib",], - "installer" : ["initscripts", "kernel", "pomona", "upstart"], - } - _package_infos = backend.parse_package_info(_packages[type]) - - packages = [] - for package in backend.depsolve(_package_infos, recursive=True): - package = package.getPackage(self.naoki) - if not package in packages: - packages.append(package) - - return packages - - def extractAll(self): - # Extract required tools - for package in self.get_packages("build"): - package.extract(self.chrootPath()) diff --git a/naoki/constants.py b/naoki/constants.py index 86e625c74..1d3aa3df8 100644 --- a/naoki/constants.py +++ b/naoki/constants.py @@ -12,6 +12,7 @@ CACHEDIR = os.path.join(BASEDIR, "cache") CCACHEDIR = os.path.join(BASEDIR, "ccache") CONFIGDIR = os.path.join(BASEDIR, "config") DOCDIR = os.path.join(BASEDIR, "doc") +GENDIR = os.path.join(BUILDDIR, "generators") IMAGESDIR = os.path.join(BUILDDIR, "images") LOGDIR = os.path.join(BASEDIR, "logs") PKGSDIR = os.path.join(BASEDIR, "pkgs") @@ -23,6 +24,8 @@ TARBALLDIR = os.path.join(CACHEDIR, "tarballs") TOOLCHAINSDIR = os.path.join(CACHEDIR, "toolchains") PATCHESDIR = os.path.join(CACHEDIR, "patches") +ARCHES_DEFAULT = os.path.join(CONFIGDIR, "architectures.conf") + CONFIGFILE = os.path.join(CONFIGDIR, "naoki.conf") CHROOT_PATH = "/sbin:/bin:/usr/sbin:/usr/bin" @@ -63,6 +66,7 @@ class Config(object): "patch", "sed", "tar", + "which", "xz", ], "shell_packages" : [ @@ -157,42 +161,6 @@ class Config(object): return ret -#class Architectures(object): -# def __init__(self, configfile): -# parser = ConfigParser.ConfigParser() -# parser.read(configfile) -# -# arches = {} -# for arch in parser.sections(): -# arches[arch] = { "name" : arch } -# for key, val in parser.items(arch): -# arches[arch][key] = val -# -# self._arches = arches -# self.__current = None -# -# def set(self, arch): -# self.__current = arch -# -# @property -# def all(self): -# return self._arches -# -# @property -# def default(self): -# return self._arches.get("i686") -# -# @property -# def current(self): -# if not self.__current: -# return self.default -# return self._arches[self.__current] -# -# def __getitem__(self, key): -# return self._arches[key] -# - # Create a globally useable instance of the configuration config = Config() -#arches = Architectures(config["architecture_config"]) diff --git a/naoki/decorators.py b/naoki/decorators.py new file mode 100644 index 000000000..5b26ace0b --- /dev/null +++ b/naoki/decorators.py @@ -0,0 +1,12 @@ +#!/usr/bin/python + +# A decorator to use singleton on a class +# http://en.wikipedia.org/wiki/Singleton_pattern#Python_.28using_decorators.29 +def singleton(cls): + instance_container = [] + def getinstance(): + if not len(instance_container): + instance_container.append(cls()) + return instance_container[0] + return getinstance + diff --git a/naoki/deps.py b/naoki/dependencies.py similarity index 80% rename from naoki/deps.py rename to naoki/dependencies.py index cbe833280..7f21b22ae 100644 --- a/naoki/deps.py +++ b/naoki/dependencies.py @@ -4,15 +4,15 @@ import logging import os import re -import arches -import paks -import repo +import architectures +import repositories +import packages -from constants import * from exception import * DEP_INVALID, DEP_FILE, DEP_LIBRARY, DEP_PACKAGE = range(4) + class Dependency(object): def __init__(self, identifier, origin=None): self.identifier = identifier @@ -27,7 +27,6 @@ class Dependency(object): s += " by %s" % self.origin.name s += "(%s)" % os.path.basename(self.origin.filename) s += ">" - return s @property @@ -51,16 +50,27 @@ class Provides(Dependency): class DependencySet(object): - def __init__(self, package): - self.package = package + def __init__(self, dependencies=[], arch=None): + if not arch: + arches = architectures.Architectures() + arch = arches.get_default() + self.arch = arch + + self.repo = repositories.BinaryRepository(self.arch) - self.repo_bin = repo.BinaryRepository(self.arch.name) + # initialize package lists + self._dependencies = [] + self._items = [] + self._provides = [] + + # add all provided dependencies + for dependency in dependencies: + self.add_dependency(dependency) logging.debug("Successfully initialized %s" % self) - self.reset() def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.package.id) + return "<%s>" % (self.__class__.__name__) def add_dependency(self, item): assert isinstance(item, Dependency) @@ -78,7 +88,7 @@ class DependencySet(object): logging.debug("Added new dependency %s" % item) def add_package(self, item): - assert isinstance(item, paks.BinaryPackage) + assert isinstance(item, packages.BinaryPackage) if item in self._items: return @@ -101,30 +111,13 @@ class DependencySet(object): self._provides.append(item) self._provides.sort() - def reset(self): - self._dependencies = [] - self._items = [] - self._provides = [] - logging.debug("Reset %s" % self) - def resolve(self): logging.debug("Resolving %s" % self) - #self.reset() - - # Always add the default packages - _dependencies = [Dependency(i) for i in config["mandatory_packages"]] - - # Add build dependencies from package - _dependencies += self.package.get_dependencies("build") - - for dependency in _dependencies: - self.add_dependency(dependency) - # Safe for endless loop counter = 1000 - while self.dependencies: + while self._dependencies: counter -= 1 if not counter: logging.debug("Maximum count of dependency loop was reached") @@ -135,21 +128,21 @@ class DependencySet(object): logging.debug("Processing dependency: %s" % dependency.identifier) if dependency.type == DEP_PACKAGE: - package = self.repo_bin.find_package_by_name(dependency.identifier) + package = self.repo.find_package_by_name(dependency.identifier) if package: # Found a package and add this self.add_package(package) continue elif dependency.type == DEP_LIBRARY: - for package in self.repo_bin.list(): + for package in self.repo.all: for provides in package.get_provides(): if provides.match(dependency): self.add_package(package) break elif dependency.type == DEP_FILE: - package = self.repo_bin.find_package_by_file(dependency.identifier) + package = self.repo.find_package_by_file(dependency.identifier) if package: self.add_package(package) continue @@ -169,10 +162,6 @@ class DependencySet(object): #logging.error("Unresolveable dependencies: %s" % self.dependencies) raise DependencyResolutionError, "%s" % self.dependencies - @property - def arch(self): - return self.package.arch - @property def dependencies(self): return sorted(self._dependencies) @@ -191,3 +180,13 @@ class DependencySet(object): @property def provides(self): return self._provides + + +if __name__ == "__main__": + import architectures + arches = architectures.Architectures() + + ds = DependencySet(arch=arches.get("i686")) + ds.add_dependency(Dependency("/bin/bash")) + ds.resolve() + print ds.packages diff --git a/naoki/environ.py b/naoki/environ.py index d3de724e5..fd14efcc1 100644 --- a/naoki/environ.py +++ b/naoki/environ.py @@ -6,11 +6,11 @@ import os import stat import logger +import mail import util from constants import * - # XXX to be moved to somewhere else ENVIRONMENT_ARGS = ["PATH", "PWD" ] @@ -26,37 +26,26 @@ def set(e): return env -class Environment(object): + +class _Environment(object): kernel_version = os.uname()[2] - def __init__(self, buildset): - self.buildset = buildset + def __init__(self, arch): + self.arch = arch logging.debug("Successfully initialized %s" % self) # XXX check if already locked self.prepare() - @property - def arch(self): - return self.buildset.arch - - @property - def package(self): - return self.buildset.package - - @property - def logger(self): - return logging.getLogger() # XXX just for now - def chrootPath(self, *args): - return os.path.join(BUILDDIR, "environments", self.package.id, *args) + raise NotImplementedError def clean(self): logging.debug("Cleaning environment %s" % self) if os.path.exists(self.chrootPath()): util.rm(self.chrootPath()) - + def prepare(self): self.clean() @@ -168,15 +157,12 @@ class Environment(object): f.write("127.0.0.1 localhost\n") f.close() - @property - def buildroot(self): - if not hasattr(self, "__buildroot"): - self.__buildroot = "buildroot.%s" % util.random_string() + def extract(self, package, *args): + logging.info("Extracting %s" % package) - return self.__buildroot + package.extract(self.chrootPath(*args)) - @property - def environ(self): + def variables(self): env = set({ "HOME" : "/root", "PATH" : "/sbin:/bin:/usr/sbin:/usr/bin", @@ -184,7 +170,6 @@ class Environment(object): "PKGROOT" : "/usr/src/pkgs", "TARGET" : "%s-ipfire-linux-gnu" % self.arch.machine, "TARGET_MACHINE" : self.arch.machine, - "BUILDROOT" : "/%s" % self.buildroot, "CHROOT" : "1", # XXX to be removed "CFLAGS" : self.arch.cflags, "CXXFLAGS" : self.arch.cxxflags, @@ -204,7 +189,7 @@ class Environment(object): def doChroot(self, command, shell=True, *args, **kwargs): ret = None try: - env = self.environ + env = self.variables() if kwargs.has_key("env"): env.update(kwargs.pop("env")) @@ -226,32 +211,6 @@ class Environment(object): return ret - def extract(self): - logging.debug("Extracting all packages and tools") - for i in self.buildset.dependency_set.packages: - i.extract(self.chrootPath()) - - def build(self, *args, **kwargs): - # Extract all packages and tools - self.extract() - - try: - self.make("package") - except: - if config["cleanup_on_failure"]: - self.clean() - # XXX send email report - raise - - if config["cleanup_on_success"]: - self.clean() - - def make(self, target): - file = "/usr/src%s" % self.package.filename[len(BASEDIR):] - - return self.doChroot("make -C %s -f %s %s" % \ - (os.path.dirname(file), file, target), shell=True) - def _mountall(self): logging.debug("Mounting environment") for cmd in self.mountCmds: @@ -302,18 +261,66 @@ class Environment(object): return ret + @property + def logger(self): + return logging.getLogger() -class Shell(Environment): - def shell(self, args=[]): - self.extract() +class Build(_Environment): + def __init__(self, package): + self.package = package + + _Environment.__init__(self, self.package.arch) + + @property + def logger(self): + return logging.getLogger() # XXX just for now + + def chrootPath(self, *args): + return os.path.join(BUILDDIR, "environments", self.package.id, *args) + + @property + def buildroot(self): + if not hasattr(self, "__buildroot"): + self.__buildroot = "buildroot.%s" % util.random_string() + + return self.__buildroot + + def variables(self): + v = _Environment.variables(self) + v.update({ "BUILDROOT" : "/%s" % self.buildroot }) + return v + + def build(self, *args, **kwargs): + try: + self.make("package") + except: + if config["cleanup_on_failure"]: + self.clean() + + # Send email report about an error + mail.report_error(self.package) + raise + + if config["cleanup_on_success"]: + self.clean() + + def make(self, target): + file = "/usr/src%s" % self.package.filename[len(BASEDIR):] + + return self.doChroot("make -C %s -f %s %s" % \ + (os.path.dirname(file), file, target), shell=True) + + +class Shell(Build): + def shell(self, args=[]): # Preparing source... self.make("prepare") command = "chroot %s /usr/src/tools/chroot-shell %s" % \ (self.chrootPath(), " ".join(args)) - for key, val in self.environ.items(): + for key, val in self.variables().items(): command = "%s=\"%s\" " % (key, val) + command if self.package.source_dir: diff --git a/naoki/generators.py b/naoki/generators.py new file mode 100644 index 000000000..bcb027bee --- /dev/null +++ b/naoki/generators.py @@ -0,0 +1,73 @@ +#!/usr/bin/python + +import logging +import os + +import dependencies +import repositories +import util + +from constants import * +from environ import _Environment + + +class Generator(_Environment): + def __init__(self, type, arch=None): + _Environment.__init__(self, arch) + + self.type = type + + self.repos = repositories.SourceRepositories(self.arch) + + for r in self.repos.all: + if not r.completely_built: + raise Exception, "The repo is not built completely: %s" % r + + self.build_deps = dependencies.DependencySet(arch=self.arch) + deps = [ + "basesystem", + "dracut", + "squashfs-tools", + "syslinux", + "/usr/bin/mkisofs", + ] + for dep in deps: + dep = dependencies.Dependency(dep) + self.build_deps.add_dependency(dep) + + self.installer_deps = dependencies.DependencySet(arch=self.arch) + deps = [ + "basesystem", + #"installer", + ] + for dep in deps: + dep = dependencies.Dependency(dep) + self.installer_deps.add_dependency(dep) + + def chrootPath(self, *args): + return os.path.join(GENDIR, self.arch.name, *args) + + @property + def logger(self): + return logging.getLogger() + + def run(self): + # Unpacking packages we need in this environment + logging.info("Resolving dependencies...") + self.build_deps.resolve() + self.installer_deps.resolve() + + for package in self.build_deps.packages: + self.extract(package) + + util.mkdir(self.chrootPath("installer")) + for package in self.installer_deps.packages: + self.extract(package, "installer") + + util.mkdir(self.chrootPath("packages")) + for p in self.repos.packages: + for bin in p.binary_files: + os.link(os.path.join(PACKAGESDIR, self.arch.name, bin), + self.chrootPath("packages", bin)) + + self.doChroot("/usr/src/tools/generator %s" % self.type, shell=True) diff --git a/naoki/mail.py b/naoki/mail.py new file mode 100644 index 000000000..fdd723539 --- /dev/null +++ b/naoki/mail.py @@ -0,0 +1,93 @@ +#!/usr/bin/python + +import email.mime.multipart +import email.mime.text +import logging +import os +import smtplib + +from constants import * + +def report_error(package): + # Do not send a report if no recipient is configured + if not config["error_report_recipient"]: + return + + try: + connection = smtplib.SMTP(config["smtp_server"]) + #connection.set_debuglevel(1) + + if config["smtp_user"] and config["smtp_password"]: + connection.login(config["smtp_user"], config["smtp_password"]) + + except smtplib.SMTPConnectError, e: + logging.error("Could not establish a connection to the smtp server: %s" % e) + return + except smtplib.SMTPAuthenticationError, e: + logging.error("Could not successfully login to the smtp server: %s" % e) + return + + msg = email.mime.multipart.MIMEMultipart() + msg["From"] = config["error_report_sender"] + msg["To"] = config["error_report_recipient"] + msg["Subject"] = config["error_report_subject"] % package.all + msg.preamble = 'You will not see this in a MIME-aware mail reader.\n' + + body = """\ +The package %(name)s had a difficulty to build itself. +This email will give you a short report about the error. + +Package information: + Name : %(name)s - %(summary)s + Version : %(version)s + Release : %(release)s + + This package in maintained by %(maintainer)s. + + +A detailed logfile is attached. + +Sincerely, + Naoki + """ % { + "name" : package.name, + "summary" : package.summary, + "version" : package.version, + "release" : package.release, + "maintainer" : package.maintainer, + } + + msg.attach(email.mime.text.MIMEText(body)) + + # Read log and append it to mail + loglines = [] + if os.path.exists(package.logfile): + f = open(package.logfile) + line = f.readline() + while line: + line = line.rstrip() + if line.endswith(LOG_MARKER): + # Reset log + loglines = [] + + loglines.append(line) + line = f.readline() + + f.close() + + if not loglines: + loglines = ["Logfile wasn't found."] + + log = email.mime.text.MIMEText("\n".join(loglines), _subtype="plain") + log.add_header('Content-Disposition', 'attachment', + filename="%s.log" % package.id) + msg.attach(log) + + try: + connection.sendmail(config["error_report_sender"], + config["error_report_recipient"], msg.as_string()) + except Exception, e: + logging.error("Could not send error report: %s: %s" % (e.__class__.__name__, e)) + return + + connection.quit() diff --git a/naoki/paks.py b/naoki/packages.py similarity index 73% rename from naoki/paks.py rename to naoki/packages.py index 26795420c..9f9aeb5e8 100644 --- a/naoki/paks.py +++ b/naoki/packages.py @@ -3,10 +3,9 @@ import logging import os import re -import tarfile -import arches -import deps +import architectures +import dependencies import environ import io import util @@ -14,14 +13,18 @@ import util from constants import * class Package(object): - def __init__(self, name, arch): - self.name = name - self.arch = arches.get(arch) - def __repr__(self): return "<%s %s:%s>" % \ (self.__class__.__name__, self.name, self.arch.name) + #@property + #def arch(self): + # raise NotImplementedError + + @property + def name(self): + return self._info["PKG_NAME"] + @property def id(self): return "%s-%s-%s.%s" % \ @@ -37,16 +40,14 @@ class Package(object): class SourcePackage(Package): - def __init__(self, name, repo, arch): - Package.__init__(self, name, arch) - self.repo = repo + def __init__(self, filename, repo, arch): + self.arch = arch + self.filename = filename + self.repository = repo self.init() - logging.debug("Successfully initialized %s" % self) - @property - def filename(self): - return os.path.join(self.repo.path, self.name, self.name + ".nm") + logging.debug("Successfully initialized %s" % self) def init(self): self._info = {} @@ -56,9 +57,9 @@ class SourcePackage(Package): "PKGROOT" : PKGSDIR, }) - output = util.do("make -f %s" % self.filename, + output = util.do("make -f %s" % os.path.basename(self.filename), shell=True, - cwd=os.path.join(self.repo.path, self.name), + cwd=os.path.dirname(self.filename), returnOutput=1, env=env) @@ -73,6 +74,7 @@ class SourcePackage(Package): key, val = m.groups() self._info[key] = val.strip("\"") + # XXX should return a dependencyset def get_dependencies(self, type=""): type2var = { "" : "PKG_DEPENDENCIES", @@ -81,15 +83,28 @@ class SourcePackage(Package): type = type2var[type] - return [deps.Dependency(d, origin=self) for d in self._info[type].split()] + return [dependencies.Dependency(d, origin=self) for d in self._info[type].split()] + + @property + def source_dir(self): + return self._info.get("SOURCE_DIR", "") @property def summary(self): return self._info["PKG_SUMMARY"] @property - def source_dir(self): - return self._info.get("SOURCE_DIR", "") + def binary_files(self): + return self._info.get("PKG_PACKAGES_FILES").split() + + @property + def is_built(self): + for file in self.binary_files: + file = os.path.join(PACKAGESDIR, self.arch.name, file) + if not os.path.exists(file): + return False + + return True class BinaryPackage(Package): @@ -117,7 +132,6 @@ class BinaryPackage(Package): f.close() return ret - def init(self): self._info = {} self._filelist = [] @@ -145,7 +159,15 @@ class BinaryPackage(Package): @property def arch(self): - return arches.get(self._info.get("PKG_ARCH", arches.get_default())) + arches = architectures.Architectures() + + arch = self._info.get("PKG_ARCH", None) + if arch: + arch = arches.get(arch) + else: + arch = arches.get_default() + + return arch @property def name(self): @@ -157,10 +179,10 @@ class BinaryPackage(Package): # Compatibility to older package format objects += self._info.get("PKG_REQUIRES", "").split() - return [deps.Dependency(o, origin=self) for o in objects] + return [dependencies.Dependency(o, origin=self) for o in objects] def get_provides(self): - return [deps.Provides(p, origin=self) \ + return [dependencies.Provides(p, origin=self) \ for p in self._info.get("PKG_PROVIDES", "").split()] @property diff --git a/naoki/repo.py b/naoki/repo.py deleted file mode 100644 index 667a0ed48..000000000 --- a/naoki/repo.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/python - -# XXX make better version comparison - -import logging -import os -import re - -import arches -import paks - -from constants import * - -def find_source_repos(): - # XXX detection for all repositories missing - return [SourceRepository("core", arches.get_default()),] - -def find_source_package(name): - for r in find_source_repos(): - p = r.get_package(name) - if p: - return p - -class Repository(object): - def __init__(self, arch): - self.arch = arches.get(arch) - - logging.debug("Successfully initialized %s" % self) - - -class SourceRepository(Repository): - _cache = {} - - def __init__(self, name, arch): - self.name = name - - Repository.__init__(self, arch) - - @property - def path(self): - return os.path.join(PKGSDIR, self.name) - - def list(self): - if not self._cache.has_key(self.arch.name): - l = [] - - for package in os.listdir(self.path): - filename = os.path.join(self.path, package, package + ".nm") - if not os.path.exists(filename): - continue - - package = paks.SourcePackage(package, repo=self, arch=self.arch.name) - - l.append(package) - - self._cache[self.arch.name] = l - - return self._cache[self.arch.name] - - def get_package(self, name): - for package in self.list(): - if package.name == name: - return package - - return None - - -class BinaryRepository(Repository): - _cache = {} - - @property - def path(self): - return os.path.join(PACKAGESDIR, self.arch.name) - - def list(self): - if not self._cache.has_key(self.arch.name): - l = [] - - for package in os.listdir(self.path): - # Construct filename - package = os.path.join(self.path, package) - - # Create package instance - package = paks.BinaryPackage(package) - - # Append package to list - l.append(package) - - self._cache[self.arch.name] = l - - return self._cache[self.arch.name] - - def get_packages(self, name): - l = [] - - # Check every package if name is equal - for package in self.list(): - if package.name == name: - l.append(package) - - return l - - def get_latest_package(self, name): - # XXX to be done - l = self.get_packages(name) - if not len(l): - return - - return l[-1] - - def find_package_by_name(self, name, all=False): - if all: - return self.get_packages(name) - - return self.get_latest_package(name) - - def find_package_by_file(self, file): - for package in self.list(): - if file in package.filelist: - return package - # XXX first match is not always the best result - - - - - - - diff --git a/naoki/repositories.py b/naoki/repositories.py new file mode 100644 index 000000000..012fd1271 --- /dev/null +++ b/naoki/repositories.py @@ -0,0 +1,215 @@ +#!/usr/bin/python + +import logging +import operator +import os + +import architectures +import packages + +from constants import * +from decorators import * + +class Repository(object): + def __init__(self, arch): + assert isinstance(arch, architectures.Architecture) + self.arch = arch + + logging.debug("Successfully initialized %s" % self) + + def __repr__(self): + return "<%s %s>" % (self.__class__.__name__, self.path) + + @property + def path(self): + raise NotImplementedError + + +class BinaryRepository(Repository): + _cache = {} + + @property + def path(self): + return os.path.join(PACKAGESDIR, self.arch.name) + + def find_package_by_name(self, name): + pkgs = self.find_packages_by_name(name) + + if pkgs: + return pkgs[0] + + def find_packages_by_name(self, name): + pkgs = [] + + for package in self.all: + if package.name == name: + pkgs.append(package) + + return sorted(pkgs, key=operator.attrgetter("release"), reverse=True) + + @property + def all(self): + if not self._cache.has_key(self.arch.name): + l = [] + + for package in os.listdir(self.path): + # Construct filename + package = os.path.join(self.path, package) + + # Create package instance + package = packages.BinaryPackage(package) + + # Append package to list + l.append(package) + + self._cache[self.arch.name] = l + + return self._cache[self.arch.name] + + def find_package_by_file(self, filename): + pkgs = self.find_packages_by_file(filename) + + if pkgs: + return pkgs[0] + + def find_packages_by_file(self, filename): + pkgs = [] + + for package in self.all: + if filename in package.filelist: + pkgs.append(package) + + return sorted(pkgs, key=operator.attrgetter("release"), reverse=True) + + +class SourceRepository(Repository): + def __init__(self, name, arch): + self.name = name + + Repository.__init__(self, arch) + + def __iter__(self): + pkgs = [os.path.join(self.path, p, p + ".nm") for p in self.packages] + + args = { + "arch" : self.arch, + "repo" : self, + } + + return PackageIterator(pkgs, packages.SourcePackage, args) + + def find_package_by_name(self, name, fast=False): + if fast: + filename = os.path.join(self.path, name, name + ".nm") + if os.path.exists(filename): + return packages.SourcePackage(filename, repo=self, arch=self.arch) + + else: + for package in self: + if package.name == name: + return package + + @property + def path(self): + return os.path.join(PKGSDIR, self.name) + + @property + def packages(self): + return os.listdir(self.path) + + @property + def completely_built(self): + for p in self: + if not p.is_built: + return False + + return True + + +class PackageIterator(object): + def __init__(self, paks, package_cls=None, cls_args={}): + self.packages = paks + self.package_cls = package_cls + self.package_cls_args = cls_args + + self.__i = 0 + + def __iter__(self): + return self + + def next(self): + if self.__i >= len(self.packages): + raise StopIteration + + pkgs = self.packages[self.__i] + self.__i += 1 + + if self.package_cls: + package = self.package_cls(pkgs, **self.package_cls_args) + + return package + + +class SourceRepositories(object): + def __init__(self, arch): + assert isinstance(arch, architectures.Architecture) + self.arch = arch + + self._repositories = [] + + self.find_all() + + def find_all(self): + for repo in os.listdir(PKGSDIR): + # Skip all non-directories + if not os.path.isdir(os.path.join(PKGSDIR, repo)): + continue + + logging.debug("Found source repository: %s" % repo) + + repo = SourceRepository(repo, self.arch) + self._repositories.append(repo) + + @property + def all(self): + return self._repositories[:] + + def find_package_by_name(self, name): + for fast in (True, False): + for repo in self._repositories: + package = repo.find_package_by_name(name, fast=fast) + if package: + return package + + @property + def packages(self): + #pkgs = [] + #for repository in self._repositories: + # pkgs += [p for p in repository] + # + #return pkgs + + for repository in self._repositories: + for p in repository: + yield p + + +if __name__ == "__main__": + arches = architectures.Architectures() + + r = SourceRepositories(arches.get("i686")) + + for repo in r.all: + print "%s" % repo.name + + #for pack in repo: + # print " %s" % pack.name + + #for package in ("gcc", "screen", "iptables", "system-release", "glibc", "nonexistant"): + # print "Searching for %s" % package, + # print r.find_package_by_name(package) + + b = BinaryRepository(arches.get("i686")) + + print b.find_packages_by_name("aiccu") + print b.find_package_by_name("aiccu") diff --git a/naoki/terminal.py b/naoki/terminal.py index f0fc67a37..0ec6f570d 100644 --- a/naoki/terminal.py +++ b/naoki/terminal.py @@ -6,9 +6,11 @@ import struct import sys import termios -import arches +import architectures from constants import * +arches = architectures.Architectures() + class ParsingError(Exception): pass @@ -273,7 +275,7 @@ class Commandline(object): Option("quiet", ["-q", "--quiet"], help="Set quiet mode"), Option("debug", ["-d", "--debug"], help="Set debugging mode"), Choice("arch", ["-a", "--arch"], help="Set architecture", - choices=[arch.name for arch in arches.all()]), + choices=[arch.name for arch in arches.all]), ], parsers=[ # Build @@ -326,7 +328,6 @@ class Commandline(object): arguments=[ List("packages"), ]), - Parser("clean", help="Cleanup unused tarballs"), ]), # Check @@ -350,22 +351,6 @@ class Commandline(object): Argument("package", help="Package to process."), ]), - # Repository - Parser("repository", - help="Repository commands", - parsers=[ - Parser("clean", - help="Cleanup the repository", - arguments=[ - List("names", help="List of repositories"), - ]), - Parser("build", - help="Build the repository", - arguments=[ - List("names", help="List of repositories"), - ]), - ]), - # Generator Parser("generate", help="Generator command", diff --git a/src/bootloader/installer.conf b/src/bootloader/installer.conf index 210224fbd..902297247 100644 --- a/src/bootloader/installer.conf +++ b/src/bootloader/installer.conf @@ -22,12 +22,12 @@ MENU hiddenrow 5 LABEL install MENU label ^Install a new @NAME@ system MENU default - KERNEL @SNAME@0 - APPEND initrd=initrd0 root=CDLABEL=@NAME@_@VERSION@ rootfstype=iso9660 mode=install quiet ro + KERNEL vmlinuz-@KERNEL@ + APPEND initrd=initrd-@KERNEL@.img root=CDLABEL=@NAME@_@VERSION@ rootfstype=iso9660 mode=install quiet ro label rescue MENU label ^Rescue installed @NAME@ system - KERNEL @SNAME@0 - APPEND initrd=initrd0 root=CDLABEL=@NAME@_@VERSION@ rootfstype=iso9660 mode=rescue quiet ro + KERNEL vmlinuz-@KERNEL@ + APPEND initrd=initrd-@KERNEL@.img root=CDLABEL=@NAME@_@VERSION@ rootfstype=iso9660 mode=rescue quiet ro label local MENU label Boot from ^local drive LOCALBOOT 0xffff diff --git a/tools/decompressor b/tools/decompressor index 92503d720..8aaaf96a2 100755 --- a/tools/decompressor +++ b/tools/decompressor @@ -54,7 +54,7 @@ fi FILES=$(for i in $FILES; do echo $i; done | sort -u) for file in $FILES; do - echo "Extracting $(basename ${file})..." + #echo "Extracting $(basename ${file})..." cpio --extract --quiet -H newc --to-stdout data.img < $file | \ tar --extract --use-compress-program=xz -C ${ROOT} done diff --git a/tools/generator b/tools/generator index c6d3937bd..69a10241f 100755 --- a/tools/generator +++ b/tools/generator @@ -1,13 +1,31 @@ -#!/bin/bash -x +#!/bin/bash BOOTLOADER_DIR=/usr/src/src/bootloader IMAGES_DIR=/usr/src/images -MKLIVERAMFS=$(dirname ${0})/mkliveramfs - ISO_FILENAME=${DISTRO_SNAME}-${DISTRO_VERSION}-${TARGET}.iso -install_config() { +. $(dirname ${0})/common-functions + +function find_latest_kernel_release() { + local i + for i in /lib/modules/*; do + i=$(basename ${i}) + done + + echo "${i}" +} + +KERNEL_RELEASE=$(find_latest_kernel_release) + +function installer_image() { + local file=${1} + local dir=${2} + + cd ${dir} && mksquashfs * ${file} -no-progress -e boot +} + +function install_config() { local src=${1} local dest=${2} @@ -16,10 +34,11 @@ install_config() { -e "s/@SNAME@/${DISTRO_SNAME}/g" \ -e "s/@VERSION@/${DISTRO_VERSION}/g" \ -e "s/@SLOGAN@/${DISTRO_SLOGAN}/g" \ + -e "s/@KERNEL@/${KERNEL_RELEASE}/g" \ < ${src} > ${dest} } -install_isolinux() { +function install_isolinux() { local dest=${1} mkdir -p ${dest} 2>/dev/null @@ -35,87 +54,43 @@ install_isolinux() { install_kernel ${dest} } -install_kernel() { +function install_kernel() { local dest=${1} mkdir -p ${dest} 2>/dev/null - local release - local releases - for release in /lib/modules/*; do - [ ! -d "${release}" ] && continue - releases="${releases} $(basename ${release})" + local file + for file in vmlinuz-${KERNEL_RELEASE} initrd-${KERNEL_RELEASE}.img; do + cp -f /boot/${file} ${dest}/${file} done - releases=$(echo ${releases}) - - # Grab first release - release=$(awk '{ print $1 }' <<<${releases}) - - if [ -z "${release}" ]; then - echo "No kernel image installed" >&2 - return 1 - fi - - cp -f /boot/vmlinuz-${release} ${dest}/${DISTRO_SNAME}0 - - # Create liveramfs - ${MKLIVERAMFS} -v --with-net -f ${dest}/initrd0 ${release} } -copy_packages() { - local dest=${1} - - mkdir -p ${dest} 2>/dev/null - - for i in ${ALL_PACKAGES}; do - cp -f /usr/src/packages/${i} ${dest}/ - done -} - -make_installer() { - local dest=${1} - - mkdir -p ${dest} 2>/dev/null - - pushd /installer - - # Copy installer overlay - cp -prf /usr/src/src/install/* . - - rm -rf /installer/usr/include - - for dir in dev proc sys; do - mkdir -p ${dir} 2>/dev/null - done - install -dv -m 1777 tmp var/tmp - - # XXX add -comp lzma here when kernel supports it - mksquashfs * ${dest}/installer.sfs -no-progress - - popd -} - -## main ## +case "${1}" in + iso) + log DEBUG "Creating ISO image..." -type=${1} + ISO_DIR=$(mktemp -d) + ISO_FILE=$(mktemp) -case "${type}" in - iso) - mkdir /iso - cd /iso + # Copy installer image to ISO + installer_image ${ISO_DIR}/installer.sfs /installer - copy_packages packages + # Install bootloader + install_isolinux ${ISO_DIR}/isolinux - make_installer /iso - install_isolinux /iso/isolinux + # "Copy" packages + mv /packages ${ISO_DIR}/Packages - cd /iso && \ + cd ${ISO_DIR} && \ mkisofs -J -r -V "${DISTRO_NAME}_${DISTRO_VERSION}" \ -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot \ - -boot-load-size 4 -boot-info-table . > ${IMAGES_DIR}/${ISO_FILENAME} + -boot-load-size 4 -boot-info-table . > ${ISO_FILE} + + cat ${ISO_FILE} > ${IMAGES_DIR}/${ISO_FILENAME} ;; - "") + *) + log ERROR "Unknown type of image: ${1}" exit 1 ;; esac