From: Michael Tremer Date: Fri, 19 Mar 2010 12:29:19 +0000 (+0100) Subject: naoki: Refactor some more things on naoki. X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=928cd1193dfaa4eeb6f7bb57dc6fb57352a51ee7;p=ipfire-3.x.git naoki: Refactor some more things on naoki. --- diff --git a/make.sh b/make.sh index c8c182319..46b3ae0dd 100755 --- a/make.sh +++ b/make.sh @@ -1,16 +1,8 @@ #!/usr/bin/python import sys -try: - import argparse -except ImportError: - import naoki.argparse as argparse - import naoki -arches = naoki.arches -config = naoki.config - # silence Python 2.6 buggy warnings about Exception.message if sys.version_info[:2] == (2, 6): import warnings @@ -19,120 +11,12 @@ if sys.version_info[:2] == (2, 6): message="BaseException.message has been deprecated as of Python 2.6", category=DeprecationWarning) -# Command line parsing -parser = argparse.ArgumentParser( - description = "Command to control the naoki buildsystem" -) - -parser.add_argument("-q", "--quiet", action="store_true", help="run in silent mode") -parser.add_argument("-d", "--debug", action="store_true", help="run in debug mode") -parser.add_argument("-a", "--arch", default=arches.default["name"], help="set architecture") -subparsers = parser.add_subparsers(help="sub-command help") - -# Build command -parser_build = subparsers.add_parser("build", help="build command") -parser_build.set_defaults(action="build") -parser_build.add_argument("package", nargs="+", help="package name") - - -# Toolchain command -parser_toolchain = subparsers.add_parser("toolchain", help="toolchain command") -parser_toolchain.set_defaults(action="toolchain") -subparsers_toolchain = parser_toolchain.add_subparsers(help="sub-command help") - - # toolchain build -parser_toolchain_build = subparsers_toolchain.add_parser("build", help="build toolchain") -parser_toolchain_build.set_defaults(subaction="build") - - # toolchain build -parser_toolchain_download = subparsers_toolchain.add_parser("download", help="download toolchain") -parser_toolchain_download.set_defaults(subaction="download") - - -# Package command -parser_package = subparsers.add_parser("package", help="package command") -parser_package.set_defaults(action="package") -subparsers_package = parser_package.add_subparsers(help="sub-command help") - - # package info [-l, --long] -parser_package_info = subparsers_package.add_parser("info", help="show package information") -parser_package_info.set_defaults(subaction="info") -parser_package_info.add_argument("-l", "--long", action="store_true", help="print in long format") -parser_package_info.add_argument("--machine", action="store_true", help="output in machine readable format") -parser_package_info.add_argument("--wiki", action="store_true", help="output in wiki format") -parser_package_info.add_argument("package", nargs="+", help="package name") - - # package tree -parser_package_tree = subparsers_package.add_parser("tree", help="show package tree") -parser_package_tree.set_defaults(subaction="tree") - - # package list [-l, --long] -parser_package_list = subparsers_package.add_parser("list", help="show package list") -parser_package_list.set_defaults(subaction="list") -parser_package_list.add_argument("-l", "--long", action="store_true", help="print list in long format") - - # package groups -parser_package_groups = subparsers_package.add_parser("groups", help="show package groups") -parser_package_groups.set_defaults(subaction="groups") -parser_package_groups.add_argument("--wiki", action="store_true", help="output in wiki format") - - -# Source command -parser_source = subparsers.add_parser("source", help="source command") -parser_source.set_defaults(action="source") -subparsers_source = parser_source.add_subparsers(help="sub-command help") - - # source download -parser_source_download = subparsers_source.add_parser("download", help="download source tarballs") -parser_source_download.set_defaults(subaction="download") -parser_source_download.add_argument("package", nargs="*", help="package name") - - # source upload -parser_source_upload = subparsers_source.add_parser("upload", help="upload source tarballs") -parser_source_upload.set_defaults(subaction="upload") -parser_source_upload.add_argument("package", nargs="*", help="package name") - - # source clean -parser_source_clean = subparsers_source.add_parser("clean", help="cleanup unused source tarballs") -parser_source_clean.set_defaults(subaction="clean") - - -# Check command -parser_check = subparsers.add_parser("check", help="check command") -parser_check.set_defaults(action="check") -subparsers_check = parser_check.add_subparsers(help="sub-command help") - - # check host -parser_check_host = subparsers_check.add_parser("host", help="check if host fulfills requirements") -parser_check_host.set_defaults(subaction="host") - - -# Batch command -parser_batch = subparsers.add_parser("batch", help="batch command") -parser_batch.set_defaults(action="batch") -subparsers_batch = parser_batch.add_subparsers(help="sub-command help") - - # batch cron -parser_batch_cron = subparsers_batch.add_parser("cron", help="gets called by a cron daemon") -parser_batch_cron.set_defaults(subaction="cron") - - -# parse the command line -args = parser.parse_args() - -if args.debug: - print "Command line arguments:", args - -# Set default arch -arches.set(args.arch) - -kwargs = {} -for key, val in args._get_kwargs(): - kwargs[key] = val +# Initialize system +n = naoki.Naoki() try: - n = naoki.Naoki() - n(**kwargs) + # Run... + n.run() exitStatus = 0 except (SystemExit,): diff --git a/naoki/__init__.py b/naoki/__init__.py index 212f81fda..1a236d48e 100644 --- a/naoki/__init__.py +++ b/naoki/__init__.py @@ -1,73 +1,70 @@ #!/usr/bin/python import ConfigParser -import curses -import logging -import logging.config -import logging.handlers import os.path import sys import time +import backend import logger -import package +import terminal import util from constants import * - -# fix for python 2.4 logging module bug: -logging.raiseExceptions = 0 +from handlers import * class Naoki(object): def __init__(self): - self.setup_logging() + # First, setup the logging + self.logging = logger.Logging(self) + + # Second, parse the command line options + self.cli = terminal.Commandline(self) + self.log.debug("Successfully initialized naoki instance") for k, v in config.items(): self.log.debug(" %s: %s" % (k, v)) - def setup_logging(self): - self.log = logging.getLogger() + def run(self): + args = self.cli.args + print "DEBUG", args - log_ini = config["log_config_file"] - if os.path.exists(log_ini): - logging.config.fileConfig(log_ini) + # If there is no action provided, exit + if not args.has_key("action"): + self.cli.help() + sys.exit(1) - if sys.stderr.isatty(): - curses.setupterm() - self.log.handlers[0].setFormatter(logger._ColorLogFormatter()) + actionmap = { + "build" : self.call_build, + "toolchain" : self.call_toolchain, + "package" : self.call_package, + "source" : self.call_source, + } - if config["quiet"]: - self.log.handlers[0].setLevel(logging.WARNING) - else: - self.log.handlers[0].setLevel(logging.INFO) - - if not os.path.isdir(LOGDIR): - os.makedirs(LOGDIR) - fh = logging.handlers.RotatingFileHandler(config["log_file"], - maxBytes=10*1024**2, backupCount=6) - fh.setFormatter(logging.Formatter("[%(levelname)s] %(message)s")) - fh.setLevel(logging.NOTSET) - self.log.addHandler(fh) - - def __call__(self, action, **kwargs): - if action == "build": - self.call_build(kwargs.get("package")) - - elif action == "toolchain": - self.call_toolchain(kwargs.get("subaction"), kwargs.get("arch")) - - elif action == "package": - self.call_package(kwargs.pop("subaction"), **kwargs) - - def call_toolchain(self, subaction, arch): - tc = chroot.Toolchain(arch) - - if subaction == "build": - tc.build() - - elif subaction == "download": - tc.download() + return actionmap[args.action.name](args.action) + + def call_toolchain(self, args): + if not args.has_key("action"): + self.cli.help() + sys.exit(1) + + actionmap = { + "build" : self.call_toolchain_build, + "download" : self.call_toolchain_download, + } + + return actionmap[args.action.name](args.action) + + def call_toolchain_build(self, args): + toolchain = chroot.Toolchain(arch) + + return toolchain.build() + + def call_toolchain_download(self, args): + toolchain = chroot.Toolchain(arch) + + return toolchain.download() def call_build(self, packages): force = True @@ -82,40 +79,103 @@ class Naoki(object): self._build(packages, force=force) - def call_package(self, subaction, **kwargs): - if subaction == "list": - for pkg in self.packages: - print pkg.info_line(long=kwargs["long"]) + def call_package(self, args): + if not args.has_key("action"): + self.cli.help() + sys.exit(1) + + actionmap = { + "info" : self.call_package_info, + "list" : self.call_package_list, + "tree" : self.call_package_tree, + "groups" : self.call_package_groups, + } + + return actionmap[args.action.name](args.action) + + def call_package_info(self, args): + packages = args.packages or backend.get_package_names() + + for package in packages: + package = backend.PackageInfo(package) + if args.long: + print package.fmtstr("""\ +-------------------------------------------------------------------------------- +Name : %(name)s +Version : %(version)s +Release : %(release)s + + %(summary)s + +%(description)s + +Maintainer : %(maintainer)s +License : %(license)s + +Files : %(objects)s +Patches : %(patches)s +--------------------------------------------------------------------------------\ +""") + else: + print package.fmtstr("""\ +-------------------------------------------------------------------------------- +Name : %(name)s +Version : %(version)s +Release : %(release)s + + %(summary)s + +--------------------------------------------------------------------------------\ +""") + + def call_package_list(self, args): + for package in self.package_names: + package = backend.PackageInfo(package) + if args.long: + print package.fmtstr("%(name)-32s | %(version)-15s | %(summary)s") + else: + print package.fmtstr("%(name)s") + + def call_package_tree(self, args): + print "TBD" + + def call_package_groups(self, args): + groups = backend.get_group_names() + if args.wiki: + print "====== All available groups of packages ======" + for group in groups: + print "===== %s =====" % group + for package in backend.get_package_names(): + package = backend.PackageInfo(package) + if not package.group == group: + continue + + print package.fmtstr(" * [[.package:%(name)s|%(name)s]] - %(summary)s") - elif subaction == "info": - packages = [package.find(pkg) for pkg in kwargs.get("package")] - packages.sort() + else: + print "\n".join(groups) - if kwargs["wiki"]: - for pkg in packages: - print pkg.info_wiki() - return - - delimiter = "----------------------------------------------------\n" - - print delimiter.join([pkg.info(long=kwargs["long"]) for pkg in packages]) - - elif subaction == "tree": - print package.deptree(self.packages) - - elif subaction == "groups": - groups = package.groups() - - if kwargs["wiki"]: - print "====== All available groups of packages ======" - for group in groups: - print group.wiki_headline() - for pkg in group.packages: - print pkg.info_wiki(long=False) + def call_source(self, args): + if not args.has_key("action"): + self.cli.help() + sys.exit(1) - return + actionmap = { + "download" : self.call_source_download, + "upload" : self.call_source_upload, + } + + return actionmap[args.action.name](args.action) + + def call_source_download(self, args): + packages = args.packages or backend.get_package_names() + + for package in packages: + package = backend.Package(package, naoki=self) + package.download() - print "\n".join(package.group_names()) + def call_source_upload(self, args): + pass # TODO def _build(self, packages, force=False): requeue = [] @@ -147,5 +207,5 @@ class Naoki(object): build.build() @property - def packages(self): - return package.list() + def package_names(self): + return backend.get_package_names() diff --git a/naoki/backend.py b/naoki/backend.py new file mode 100644 index 000000000..59c8eb50a --- /dev/null +++ b/naoki/backend.py @@ -0,0 +1,291 @@ +#!/usr/bin/python + +import os + +import chroot +import util + +from constants import * + +__cache = { + "package_names" : None, + "group_names" : None, +} + +def get_package_names(toolchain=False): + if not __cache["package_names"]: + names = [] + for repo in get_repositories(toolchain): + names.extend(repo.package_names) + + __cache["package_names"] = sorted(names) + + return __cache["package_names"] + +def get_group_names(): + if not __cache["group_names"]: + groups = [] + for package in get_package_names(): + package = PackageInfo(package) + if not package.group in groups: + groups.append(package.group) + + __cache["group_names"] = sorted(groups) + + return __cache["group_names"] + +def find_package_name(name, toolchain=False): + if name in get_package_names(toolchain): + return name + + for package in get_package_names(toolchain): + if os.path.basename(package) == name: + return package + +def depsolve(packages, recursive=False): + deps = [] + for package in packages: + if not package in deps: + deps.append(package) + + if not recursive or not deps: + return deps + + while True: + length = len(deps) + for dep in deps[:]: + deps.extend(dep.dependencies) + + new_deps = [] + for dep in deps: + if not dep in new_deps: + new_deps.append(dep) + + deps = new_deps + + if length == len(deps): + break + + deps.sort() + return deps + +class PackageInfo(object): + __data = {} + + def __init__(self, name): + self._name = name + + def __repr__(self): + return "" % self.name + + def get_data(self): + if not self.__data.has_key(self.name): + self.__data[self.name] = self.fetch() + + return self.__data[self.name] + + def set_data(self, data): + self.__data[self.name] = data + + _data = property(get_data, set_data) + + def fetch(self): + env = os.environ.copy() + env.update(config.environment) + env["PKGROOT"] = PKGSDIR + output = util.do("make -f %s" % self.filename, shell=True, + cwd=os.path.join(PKGSDIR, self.name), returnOutput=1, env=env) + + ret = {} + for line in output.splitlines(): + a = line.split("=", 1) + if not len(a) == 2: continue + key, val = a + ret[key] = val.strip("\"") + + ret["FINGERPRINT"] = self.fingerprint + + return ret + + def fmtstr(self, s): + return s % self.all + + def getPackage(self, naoki): + return Package(self.name, naoki) + + @property + def all(self): + return { + "description" : self.description, + "filename" : self.filename, + "fingerprint" : self.fingerprint, + "group" : self.group, + "license" : self.license, + "maintainer" : self.maintainer, + "name" : self.name, + "objects" : self.objects, + "patches" : self.patches, + "release" : self.release, + "summary" : self.summary, + "version" : self.version, + } + + def _dependencies(self, s, recursive=False): + c = s + "_CACHE" + if not self._data.has_key(c): + deps = [] + for name in self._data.get(s).split(" "): + name = find_package_name(name) + if name: + deps.append(Dependency(name)) + + self._data.update({c : depsolve(deps, recursive)}) + + return self._data.get(c) + + @property + def dependencies(self): + return self._dependencies("PKG_DEPENDENCIES") + + @property + def dependencies_build(self): + return self._dependencies("PKG_BUILD_DEPENDENCIES") + + @property + def dependencies_all(self): + return depsolve(self.dependencies + self.dependencies_build, recursive=True) + + @property + def description(self): + return self._data.get("PKG_DESCRIPTION") + + @property + def filename(self): + return os.path.join(PKGSDIR, self.name, os.path.basename(self.name)) + ".nm" + + @property + def fingerprint(self): + return "%d" % os.stat(self.filename).st_mtime + + @property + def group(self): + return self._data.get("PKG_GROUP") + + @property + def id(self): + return "%s-%s-%s" % (self.name, self.version, self.release) + + @property + def license(self): + return self._data.get("PKG_LICENSE") + + @property + def maintainer(self): + return self._data.get("PKG_MAINTAINER") + + @property + def name(self): + return self._name + + @property + def objects(self): + return self._data.get("PKG_OBJECTS").split(" ") + + @property + def package_files(self): + return self._data.get("PKG_PACKAGES_FILES").split(" ") + + @property + def patches(self): + return self._data.get("PKG_PATCHES").split(" ") + + @property + def release(self): + return self._data.get("PKG_REL") + + @property + def summary(self): + return self._data.get("PKG_SUMMARY") + + @property + def version(self): + return self._data.get("PKG_VER") + + +class Dependency(PackageInfo): + def __repr__(self): + return "" % self.name + + +class Package(object): + def __init__(self, name, naoki): + self.info = PackageInfo(name) + self.naoki = naoki + + #self.log.debug("Initialized package object %s" % name) + + def build(self): + environment = chroot.Environment(self) + environment.build() + + def download(self): + return "TODO" + files = self.info.objects + #self.log.info("Downloading %s..." % files) + download(self.info.objects) + + def extract(self, dest): + files = [os.path.join(PACKAGESDIR, file) for file in self.info.package_files] + if not files: + return + + self.log.debug("Extracting %s..." % files) + util.do("%s --root=%s %s" % (os.path.join(TOOLSDIR, "decompressor"), + dest, " ".join(files)), shell=True) + + @property + def log(self): + return self.naoki.logging.getBuildLogger(self.info.id) + + +def get_repositories(toolchain=False): + if toolchain: + return Repository("toolchain") + + repos = [] + for repo in os.listdir(PKGSDIR): + if os.path.isdir(os.path.join(PKGSDIR, repo)): + repos.append(repo) + + repos.remove("toolchain") + + return [Repository(repo) for repo in repos] + +class Repository(object): + def __init__(self, name): + self.name = name + + def __repr__(self): + return "" % self.name + + @property + def packages(self): + packages = [] + for package in os.listdir(self.path): + package = PackageInfo(os.path.join(self.name, package)) + packages.append(package) + + return packages + + @property + def package_names(self): + return [package.name for package in self.packages] + + @property + def path(self): + return os.path.join(PKGSDIR, self.name) + +if __name__ == "__main__": + pi = PackageInfo("core/grub") + + print pi.dependencies diff --git a/naoki/chroot.py b/naoki/chroot.py index 0feda2452..a1cacb044 100644 --- a/naoki/chroot.py +++ b/naoki/chroot.py @@ -15,6 +15,7 @@ from logger import getLog class Environment(object): def __init__(self, package): self.package = package + self.naoki = self.package.naoki self.arch = arches.current self.config = config @@ -44,19 +45,8 @@ class Environment(object): ] self.buildroot = "buildroot.%d" % random.randint(0, 1024) - self.log = None - self.__initialized = False - def init(self): - if self.__initialized: - return - self._init() - self.__initialized = True - - def _init(self): - self._setupLogging() - - self.log.info("Setting up environment %s..." % self.chrootPath()) + self.log.debug("Setting up environment %s..." % self.chrootPath()) if os.path.exists(self.chrootPath()): self.clean() @@ -106,7 +96,7 @@ class Environment(object): util.rm(self.chrootPath()) def make(self, target): - file = "/usr/src%s" % self.package.filename[len(BASEDIR):] + file = "/usr/src%s" % self.package.info.filename[len(BASEDIR):] return self.doChroot("make -C %s -f %s %s" % \ (os.path.dirname(file), file, target), shell=True) @@ -152,17 +142,7 @@ class Environment(object): return ret def chrootPath(self, *args): - return os.path.join(BUILDDIR, "environments", self.package.id, *args) - - def _setupLogging(self): - logfile = os.path.join(LOGDIR, self.package.id, "build.log") - if not os.path.exists(os.path.dirname(logfile)): - util.mkdir(os.path.dirname(logfile)) - self.log = logging.getLogger(self.package.id) - fh = logging.FileHandler(logfile) - fh.setFormatter(logging.Formatter("[%(levelname)s] %(message)s")) - fh.setLevel(logging.NOTSET) - self.log.addHandler(fh) + return os.path.join(BUILDDIR, "environments", self.package.info.id, *args) def _setupDev(self): # files in /dev @@ -242,20 +222,14 @@ class Environment(object): util.do(cmd, raiseExc=0, shell=True) def extractAll(self): - packages = self.package.deps + self.package.build_deps - for pkg in config["mandatory_packages"]: - pkg = package.find(pkg) - if not pkg in packages: - packages.append(pkg) - - packages = package.depsolve(packages, recursive=True) + packages = [p.getPackage(self.naoki) \ + for p in self.package.info.dependencies_all] - for pkg in packages: - pkg.extract(self.chrootPath()) + for package in packages: + package.extract(self.chrootPath()) def build(self): self.package.download() - self.init() try: self.make("package") @@ -267,6 +241,10 @@ class Environment(object): if config["cleanup_on_success"]: self.clean() + @property + def log(self): + return self.package.log + class Toolchain(object): def __init__(self, arch): diff --git a/naoki/logger.py b/naoki/logger.py index a73f5da96..47f9944a3 100644 --- a/naoki/logger.py +++ b/naoki/logger.py @@ -2,9 +2,75 @@ import curses import logging +import logging.config +import logging.handlers import sys import time +# fix for python 2.4 logging module bug: +logging.raiseExceptions = 0 + +from constants import * + +class Logging(object): + def __init__(self, naoki): + self.naoki = naoki + + self.setup() + + def setup(self): + self.naoki.log = self.log = logging.getLogger() + + log_ini = config["log_config_file"] + if os.path.exists(log_ini): + logging.config.fileConfig(log_ini) + + if sys.stderr.isatty(): + curses.setupterm() + self.log.handlers[0].setFormatter(_ColorLogFormatter()) + + # Set default configuration + self.quiet(config["quiet"]) + + self.log.handlers[0].setLevel(logging.DEBUG) + logging.getLogger("naoki").propagate = 1 + + if not os.path.isdir(LOGDIR): + os.makedirs(LOGDIR) + fh = logging.handlers.RotatingFileHandler(config["log_file"], + maxBytes=10*1024**2, backupCount=6) + fh.setFormatter(logging.Formatter("[%(levelname)s] %(message)s")) + fh.setLevel(logging.NOTSET) + self.log.addHandler(fh) + + def quiet(self, val): + if val: + self.log.debug("Enabled quiet logging mode") + self.log.handlers[0].setLevel(logging.WARNING) + else: + #self.log.debug("Enabled verbose logging mode") + self.log.handlers[0].setLevel(logging.INFO) + + def _setupBuildLogger(self, logger): + logger.setLevel(logging.DEBUG) + + handler = logging.handlers.RotatingFileHandler( + os.path.join(LOGDIR, logger.name + ".log"), maxBytes=10*1024**2, + backupCount=5) + + formatter = logging.Formatter("[BUILD] %(message)s") + handler.setFormatter(formatter) + + logger.addHandler(handler) + + def getBuildLogger(self, name): + logger = logging.getLogger(name) + if not logger.handlers: + self._setupBuildLogger(logger) + + return logger + + # defaults to module verbose log # does a late binding on log. Forwards all attributes to logger. # works around problem where reconfiguring the logging module means loggers diff --git a/naoki/terminal.py b/naoki/terminal.py index b28292ea9..d39ee8512 100644 --- a/naoki/terminal.py +++ b/naoki/terminal.py @@ -12,6 +12,14 @@ class ParsingError(Exception): pass +class NameSpace(dict): + def __getattr__(self, attr): + try: + return self.__getitem__(attr) + except KeyError: + raise AttributeError + + class Parser(object): def __init__(self, name, arguments=[], parsers=[]): self.name = name @@ -51,11 +59,11 @@ class Parser(object): @property def values(self): - ret = { - "name" : self.name - } + ret = NameSpace( + name=self.name, + ) if self.subparser: - ret["subaction"] = self.subparser.values + ret["action"] = self.subparser.values for argument in self.arguments: ret[argument.name] = argument.value() @@ -156,10 +164,10 @@ class Commandline(object): self.naoki = naoki # Parse the stuff - args = self.__parse() + self.args = self.__parse() # ... afterwards, process global directives - self.__process_global(args) + self.__process_global(self.args) def __process_global(self, args): # Set quiet mode @@ -245,6 +253,9 @@ class Commandline(object): return parser.values + def help(self): + print "PRINTING HELP TEXT" + DEFAULT_COLUMNS = 80 DEFAULT_LINES = 25