From: Michael Tremer Date: Sun, 27 Feb 2011 19:17:48 +0000 (+0100) Subject: Remove legacy build system. X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e4843adb06d9b75e66cf03ea1cf3e54dae0ac383;p=ipfire-3.x.git Remove legacy build system. --- diff --git a/config/architectures.conf b/config/architectures.conf deleted file mode 100644 index c197dd5d6..000000000 --- a/config/architectures.conf +++ /dev/null @@ -1,49 +0,0 @@ -[default] - -; The default architecture -default = i686 - -[i686] - -machine = i686 -personality = linux32 - -cflags = -O2 -fomit-frame-pointer -fasynchronous-unwind-tables -pipe -cxxflags = %(cflags)s - -;[i586] -; -;machine = i586 -;personality = linux32 -; -;cflags = -O2 -fomit-frame-pointer -pipe -;cxxflags = %(cflags)s -; -;[i486] -; -;machine = i486 -;personality = linux32 -; -;cflags = -O2 -fomit-frame-pointer -pipe -;cxxflags = %(cflags)s - -;[alix] -; -; While i586 works fine on a Geode LX, i486 should be a more performant choice -; right now due to the way Geode LX CPU pipeline ans scheduling works. glibc -; i586 assembler optimized routines are measureably slower than the i486 ones on -; a Geode LX. -;machine = i486 -;personality = linux32 -; -;cflags = -Os -march=geode -fno-align-jumps -fno-align-functions -fno-align-labels -fno-align-loops -fomit-frame-pointer -pipe -;cxxflags = %(cflags)s - - -[x86_64] - -machine = x86_64 -personality = linux64 - -cflags = -O2 -fomit-frame-pointer -pipe -cxxflags = %(cflags)s diff --git a/config/etc/certparams b/config/etc/certparams deleted file mode 100644 index 9b25cda60..000000000 --- a/config/etc/certparams +++ /dev/null @@ -1,9 +0,0 @@ -. -. -. -. -. -HOSTNAME -. - - diff --git a/config/etc/group b/config/etc/group deleted file mode 100644 index fc82e802e..000000000 --- a/config/etc/group +++ /dev/null @@ -1,40 +0,0 @@ -root:x:0: -bin:x:1: -sys:x:2: -kmem:x:3: -tty:x:4: -tape:x:5: -daemon:x:6: -floppy:x:7: -disk:x:8: -lp:x:9: -dialout:x:10: -audio:x:11: -video:x:12: -utmp:x:13: -usb:x:14: -cdrom:x:15: -messagebus:x:18: -haldaemon:x:19: -uucp:x:20: -fcron:x:22: -squid:x:23: -www:x:33: -mail:x:34: -ntp:x:38: -ftp:x:45: -vsftpd:x:47: -rsyncd:x:48: -sshd:x:50: -nobody:x:99: -users:x:100: -snort:x:101: -logwatch:x:102: -dnsmasq:x:103: -cron:x:104: -postfix:x:107: -postdrop:x:108: -clamav:x:109: -amavis:x:110: -mldonkey:x:111: -samba:x:1000: diff --git a/config/etc/host.conf b/config/etc/host.conf deleted file mode 100644 index 5cdde5e52..000000000 --- a/config/etc/host.conf +++ /dev/null @@ -1 +0,0 @@ -order hosts,bind diff --git a/config/etc/logrotate.conf b/config/etc/logrotate.conf deleted file mode 100644 index 0f171d644..000000000 --- a/config/etc/logrotate.conf +++ /dev/null @@ -1,93 +0,0 @@ -# rotate log files weekly -weekly - -# keep 52 weeks worth of backlogs -rotate 52 - -# create new (empty) log files after rotating old ones -create - -# uncomment this if you want your log files compressed -compress - -# wtmp -/var/log/wtmp { - weekly - create 0664 root utmp - rotate 1 -} - -/var/log/httpd/access_log /var/log/httpd/error_log /var/log/httpd/ssl_request_log /var/log/httpd/ssl_engine_log { - missingok - sharedscripts - postrotate - /bin/kill -HUP `cat /var/run/httpd.pid 2>/dev/null` 2> /dev/null || true - endscript -} - -/var/log/snort/alert { - weekly - copytruncate - compress - ifempty - missingok - postrotate - /usr/bin/find /var/log/snort -path '/var/log/snort/[0-9]*' -prune -exec /bin/rm -rf {} \; - /usr/bin/find /var/log/snort -name 'snort.log.*' -mtime +28 -exec /bin/rm -rf {} \; - /usr/local/bin/restartsnort - endscript -} - -/var/log/squid/access.log /var/log/squid/user_agent.log /var/log/squid/referer.log { - weekly - copytruncate - ifempty - missingok -} - -/var/log/squid/cache.log { - weekly - rotate 3 - copytruncate - compress - missingok -} - -/var/log/squid/store.log { - weekly - rotate 3 - copytruncate - compress - missingok - postrotate - /bin/chmod -R ugo+rX /var/log/squid - /usr/sbin/squid -k rotate - endscript -} - -/var/log/messages /var/log/boot.log /var/log/dhcpcd.log { - create 664 root syslogd - sharedscripts - ifempty - postrotate - /bin/kill -HUP `cat /var/run/syslogd.pid 2> /dev/null` 2> /dev/null || true - endscript -} - -/var/log/squidGuard/*.log { - weekly - rotate 4 - copytruncate - compress - notifempty - missingok -} - -/var/log/updatexlrator/*.log { - weekly - rotate 4 - copytruncate - compress - notifempty - missingok -} diff --git a/config/etc/modules.conf b/config/etc/modules.conf deleted file mode 100644 index ebf07eb78..000000000 --- a/config/etc/modules.conf +++ /dev/null @@ -1,24 +0,0 @@ -# USB HID Settings -above hid keybdev - -# ISDN Settings -alias char-major-43 hisax -alias char-major-44 hisax -alias char-major-45 hisax -alias ippp0 off -alias ippp1 off - -# PPP Settings -alias char-major-108 ppp_generic -alias /dev/ppp ppp_generic -alias tty-ldisc-3 ppp_async -alias tty-ldisc-13 n_hdlc -alias tty-ldisc-14 ppp_synctty -alias ppp-compress-21 bsd_comp -alias ppp-compress-24 ppp_deflate -alias ppp-compress-26 ppp_deflate -alias net-pf-8 atm -alias net-pf-24 pppoe -alias char-major-144 pppox -alias char-major-166 acm - diff --git a/config/etc/passwd b/config/etc/passwd deleted file mode 100644 index 0ca66730b..000000000 --- a/config/etc/passwd +++ /dev/null @@ -1,27 +0,0 @@ -root:x:0:0:root:/root:/bin/bash -bin:x:1:1:bin:/bin:/bin/false -daemon:x:2:2:daemon:/sbin:/bin/false -mail:x:8:12:mail:/var/spool/mail:/bin/false -messagebus:x:18:18:D-BUS Message Daemon User:/dev/null:/bin/false -haldaemon:x:19:19:HAL Daemon User:/dev/null:/bin/false -fcron:x:22:22:Fcron User:/dev/null:/bin/false -squid:x:23:23:ftp:/var/spool/squid:/bin/false -www:x:33:33:Lighttpd:/srv/web/ipfire:/bin/false -ntp:x:38:38::/etc/ntp:/bin/false -mysql:x:41:41:MySQL Server:/dev/null:/bin/false -ftp:x:45:45:anonymous_user:/home/ftp:/bin/false -vsftpd:x:47:47:vsftpd User:/home/ftp:/bin/false -rsyncd:x:48:48:rsyncd Daemon:/home/rsync:/bin/false -sshd:x:50:50:sshd PrivSep:/var/lib/sshd:/bin/false -nobody:x:99:99:Nobody:/home/nobody:/bin/false -postfix:x:100:100::/var/spool/postfix:/bin/false -snort:x:101:101:ftp:/var/log/snort:/bin/false -logwatch:x:102:102::/var/log/logwatch:/bin/false -dnsmasq:x:103:103::/:/bin/false -cron:x:104:104::/:/bin/false -clamav:x:109:109:Clam AntiVirus:/home/clamav:/bin/false -amavis:x:110:110:Amavisd-new user:/var/amavis: -cyrus:x:111:12:Cyrus user:/usr/cyrus: -filter:x:112:12:Spam user:/home/filter:/bin/false -mldonkey:x:113:111:Mldonkey user:/opt/mldonkey:/bin/false -samba:x:1000:1000:Samba User:/var/empty:/bin/false diff --git a/config/naoki.conf b/config/naoki.conf deleted file mode 100644 index da68fe91b..000000000 --- a/config/naoki.conf +++ /dev/null @@ -1,30 +0,0 @@ - -[architecture] - -config = config/architectures.conf - -arch = i686 - -[distro] - -; Name of the distribution we build. -name = IPFire -sname = ipfire - -; Major version number -epoch = 3 - -; Minor version string -version = %(epoch)s.0-prealpha2 - -; A descriptive slogan -slogan = "Gluttony" - -; The distribution vendor -vendor = The IPFire Team - - -[sources] - -; Where do we get the tarballs from? -download_url = http://source.ipfire.org/source-3.x diff --git a/make.sh b/make.sh deleted file mode 100755 index 0dab4252d..000000000 --- a/make.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/python - -import logging -import sys - -import naoki - -# silence Python 2.6 buggy warnings about Exception.message -if sys.version_info[:2] == (2, 6): - import warnings - warnings.filterwarnings( - action="ignore", - message="BaseException.message has been deprecated as of Python 2.6", - category=DeprecationWarning) - -# Initialize system -n = naoki.Naoki() - -try: - # Run... - n.run() - exitStatus = 0 - -except (SystemExit,): - raise - -except (KeyboardInterrupt,): - exitStatus = 7 - logging.error("Exiting on user interrupt, -C") - -sys.exit(exitStatus) diff --git a/naoki/__init__.py b/naoki/__init__.py deleted file mode 100644 index 11f0cd892..000000000 --- a/naoki/__init__.py +++ /dev/null @@ -1,320 +0,0 @@ -#!/usr/bin/python - -import ConfigParser -import fcntl -import logging -import os.path -import random -import sys -import time - -import architectures -import build -import dependencies -import environ -import generators -import logger -import packages -import repositories -import terminal -import util - -from constants import * - -# Initialize logging -log = logger.Logging() - -class Naoki(object): - def __init__(self): - # Second, parse the command line options - self.cli = terminal.Commandline(self) - - logging.debug("Successfully initialized naoki instance") - for k, v in config.items(): - logging.debug(" %s: %s" % (k, v)) - - def debug(self, state): - """ - Enable or disable debugging mode. - """ - - # Set logging to debug mode - log.debug(state) - - config.debug = state - - def run(self): - args = self.cli.args - - # If there is no action provided, exit - if not args.has_key("action"): - self.cli.help() - return 1 - - if config["nice_level"]: - os.nice(int(config["nice_level"])) - - actionmap = { - "build" : self.call_build, - "package" : self.call_package, - "source" : self.call_source, - "shell" : self.call_shell, - "generate" : self.call_generate, - "batch" : self.call_batch, - } - - return actionmap[args.action.name](args.action) - - def _get_source_repos(self, arch=None): - if not arch: - arches = architectures.Architectures() - arch = arches.get_default() - - return repositories.SourceRepositories(arch=arch) - - def call_build(self, args): - # Source repository - repo = self._get_source_repos() - - config["shell_on_failure"] = args.shell - - package = repo.find_package_by_name(args.package) - if not package: - raise Exception, "Could not find package: %s" % name - - # We found a package... build it: - b = build.Build(package, - ignore_dependency_errors=args.ignore_dependency_errors) - - b.build() - - def call_package(self, args): - if not args.has_key("action"): - self.cli.help() - return 1 - - actionmap = { - "dependencies" : self.call_package_dependencies, - "info" : self.call_package_info, - "list" : self.call_package_list, - "groups" : self.call_package_groups, - "raw" : self.call_package_raw, - "provides" : self.call_package_provides, - } - - return actionmap[args.action.name](args.action) - - def call_package_dependencies(self, args): - repo = self._get_source_repos() - - p = repo.find_package_by_name(args.package) - if not p: - raise Exception, "Could not find package: %s" % args.package - - deps = dependencies.DependencySet() - for dep in p.get_dependencies() + p.get_dependencies("build"): - deps.add_dependency(dep) - - deps.resolve() - - identifiers = set([d.identifier for d in deps.dependencies]) - - for identifier in sorted(identifiers): - print identifier - - def call_package_info(self, args): - # Source repositories - repo = self._get_source_repos() - - for package in repo.packages: - if args.packages: - if not package.name in args.packages: - continue - - if args.long: - print package.fmtstr("""\ --------------------------------------------------------------------------------- -Name : %(PKG_NAME)s -Version : %(PKG_VER)s -Release : %(PKG_REL)s - - %(PKG_SUMMARY)s - -%(PKG_DESCRIPTION)s - -Maintainer : %(PKG_MAINTAINER)s -License : %(PKG_LICENSE)s - -Objects : %(PKG_OBJECTS)s -Patches : %(PKG_PATCHES)s ---------------------------------------------------------------------------------\ -""") - else: - print package.fmtstr("""\ --------------------------------------------------------------------------------- -Name : %(PKG_NAME)s -Version : %(PKG_VER)s -Release : %(PKG_REL)s - - %(PKG_SUMMARY)s - ---------------------------------------------------------------------------------\ -""") - - def call_package_list(self, args): - repo = self._get_source_repos() - - for package in repo.packages: - # Skip unbuilt packages if we want built packages - if args.built and not package.is_built: - continue - - # Skip built packages if we want unbuilt only - if args.unbuilt and package.is_built: - continue - - if args.long: - print package.fmtstr("%(PKG_NAME)-32s | %(PKG_VER)-15s | %(PKG_SUMMARY)s") - else: - print package.name - - def call_package_groups(self, args): - groups = [] - - repo = self._get_source_repos() - - for package in repo.packages: - group = package.group - if not group in groups: - groups.append(group) - - print "\n".join(sorted(groups)) - - def call_package_raw(self, args): - filename = args.package - - if os.path.exists(filename): - p = packages.BinaryPackage(filename) - else: - repo = self._get_source_repos() - - p = repo.find_package_by_name(args.package) - if not p: - raise Exception, "Could not find package: %s" % args.package - - p.print_raw_info() - - def call_package_provides(self, args): - provides = dependencies.Provides(args.provides) - - arch = architectures.Architectures().get_default() - repo = repositories.BinaryRepository(arch) - - for package in repo.find_packages_by_provides(provides): - print package.fmtstr("%(PKG_NAME)s %(PKG_VER)s-%(PKG_REL)s %(PKG_SUMMARY)s") - - def call_source(self, args): - if not args.has_key("action"): - self.cli.help() - sys.exit(1) - - actionmap = { - "download" : self.call_source_download, - "upload" : self.call_source_upload, - } - - return actionmap[args.action.name](args.action) - - def call_source_download(self, args): - repo = self._get_source_repos() - - for package in repo.packages: - if args.packages: - if not package.name in args.packages: - continue - - package.source_download() - - def call_source_upload(self, args): - pass # TODO - - def call_shell(self, args): - # Load architecture set - arches = architectures.Architectures() - - # Choose default architecture - arch = arches.get_default() - - # Load all source packages - repo = repositories.SourceRepositories(arch=arch) - - # Pick the one we need - p = repo.find_package_by_name(args.package) - if not p: - raise Exception, "Could not find package: %s" % args.package - - # Initialize and run the shell - shell = build.PackageShell(p) - - return shell.shell() - - def call_generate(self, args): - if not args.type in ("iso",): - return - - arch = architectures.Architectures().get_default() - - gen = generators.Generator(args.type, arch) - return gen.run() - - def call_batch(self, args): - try: - self._lock = open(LOCK_BATCH, "a+") - except IOError, e: - return 0 - - try: - fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError, e: - print >>sys.stderr, "Batch operations are locked by another process" - return 0 - - actionmap = { - "cron" : self.call_batch_cron, - } - - return actionmap[args.action.name](args.action) - - def call_batch_cron(self, args): - pkgs = [] - candidates = [] - - # Choose architecture - arches = architectures.Architectures() - arch = arches.get_default() - - repo = repositories.SourceRepositories(arch=arch) - for package in repo.packages: - if not package.is_built: - pkgs.append(package) - else: - candidates.append(package) - - # Initialize a job queue - jobs = build.Jobs() - - # Add all unbuilt packages to the job queue - for package in pkgs: - package = build.Build(package) - jobs.add(package) - - # If we have less than ten packages in the queue we add some random - # ones - need_counter = 10 - len(jobs) - if need_counter >= 0: - for candidate in random.sample(candidates, need_counter): - candidate = build.Build(candidate) - jobs.add(candidate) - - while jobs.has_jobs: - jobs.process_next() diff --git a/naoki/architectures.py b/naoki/architectures.py deleted file mode 100644 index ec9b85798..000000000 --- a/naoki/architectures.py +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/python - -import logging -import os.path - -from ConfigParser import ConfigParser, DEFAULTSECT - -from constants import * -from decorators import * - -class Architecture(object): - def __init__(self, name, **settings): - - self._settings = { - "default" : False, - "name" : name, - } - self._settings.update(settings) - - logging.debug("Set up new architecture: %s" % self._settings) - - def __repr__(self): - s = "<%s %s" % (self.__class__.__name__, self._settings["name"]) - if self.default: - s += " (default)" - s += ">" - return s - - @property - def buildable(self): - """ Check if this architecture is buildable on the local host""" - - return True # TODO - - def get_default(self): - return self._settings["default"] - - def set_default(self, value): - self._settings["default"] = value - - default = property(get_default, set_default) - - def __getattr__(self, attr): - try: - return self._settings[attr] - except KeyError: - raise AttributeError, attr - - -@singleton -class Architectures(object): - def __init__(self): - self._architectures = [] - - # Try to read the default architectures - self.read(ARCHES_DEFAULT) - - def read(self, filename): - logging.debug("Reading architectures from %s" % filename) - - if not os.path.exists(filename): - return - - p = ConfigParser() - p.read(filename) - - default = {} - - for arch in p.sections(): - if arch == "default": - default = p.items(arch) - continue - - settings = {} - for key, val in p.items(arch): - settings[key] = val - - a = Architecture(arch, **settings) - self.add_architecture(a) - - for key, val in default: - if key == "default": - self.default = val - - def add_architecture(self, arch): - assert isinstance(arch, Architecture) - - self._architectures.append(arch) - - def get_default(self): - for a in self._architectures: - if a.default: - return a - - raise Exception, "Cannot find default architecture" - - def set_default(self, arch): - if arch is None: - return - - if not arch in [a.name for a in self.all]: - raise Exception, "Cannot set default architecture: Unknown architecture %s" % arch - - logging.debug("Setting default architecture: %s" % arch) - - for a in self._architectures: - if a.name == arch: - a.default = True - else: - a.default = False - - default = property(get_default, set_default) - - def get(self, name): - for arch in self._architectures: - if arch.name == name: - return arch - - raise Exception, "Could not find arch: %s" % name - - @property - def all(self): - return self._architectures[:] - - -if __name__ == "__main__": - arches = Architectures() - - for arch in arches.all: - print "Name: %s" % arch.name - for key, val in arch._settings.items(): - print " %-20s : %s" % (key, val) - - print arches.get("i686") - print arches.default - diff --git a/naoki/build.py b/naoki/build.py deleted file mode 100644 index 81ffac517..000000000 --- a/naoki/build.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/python - -import logging -import uuid - -import dependencies -import environ - -from constants import * -from exception import * - - -class Build(object): - def __init__(self, package, **kwargs): - self.package = package - - # Generate a random, but unique id - self.id = uuid.uuid4() - - # Create dependency set - self.dependency_set = dependencies.DependencySet(arch=self.arch) - - # Add all mandatory packages and build dependencies - deps = [dependencies.Dependency(p) for p in config["mandatory_packages"]] - deps += self.package.get_dependencies("build") - for package in deps: - self.dependency_set.add_dependency(package) - - self.settings = { - "ignore_dependency_errors" : False, - } - - self.settings.update(kwargs) - - def __repr__(self): - return "<%s %s-%s:%s>" % \ - (self.__class__.__name__, self.id, self.package.name, self.arch.name) - - @property - def arch(self): - return self.package.arch - - def build(self, **settings): - self.settings.update(settings) - - logging.info("Building: %s (%s)" % (self.package.id, self.id)) - logging.info("") - logging.info(" %s" % self.package.summary) - logging.info("") - - # Download the source tarballs - self.package.source_download() - - # Resolve the dependencies - try: - self.dependency_set.resolve() - except DependencyResolutionError, e: - if self.settings["ignore_dependency_errors"]: - logging.warning("Ignoring dependency errors: %s" % e) - else: - raise - - e = environ.Build(self.package, build_id="%s" % self.id) - - # Extract all tools - for package in self.dependency_set.packages: - e.extract(package) - - # Build :D - e.build() - - -class PackageShell(Build): - def __init__(self, *args, **kwargs): - Build.__init__(self, *args, **kwargs) - - # Add shell packages to have a little bit more - # comfort in here... - for dependency in config["shell_packages"]: - dependency = dependencies.Dependency(dependency) - self.dependency_set.add_dependency(dependency) - - def shell(self, **settings): - # Resolve the dependencies - try: - self.dependency_set.resolve() - except DependencyResolutionError, e: - if self.settings["ignore_dependency_errors"]: - logging.warning("Ignoring dependency errors: %s" % e) - else: - raise - - e = environ.Build(self.package, build_id="%s" % self.id) - - # Extract all tools - for package in self.dependency_set.packages: - e.extract(package) - - # Download the source tarballs - self.package.source_download() - - # Preparing source... - e.make("prepare") - - # Run the shell - e.shell() diff --git a/naoki/constants.py b/naoki/constants.py deleted file mode 100644 index e1c2f6c5b..000000000 --- a/naoki/constants.py +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/python - -from ConfigParser import ConfigParser, DEFAULTSECT -import math -import os -import socket -import sys - -BASEDIR = os.getcwd() - -BUILDDIR = os.path.join(BASEDIR, "build") -CACHEDIR = os.path.join(BASEDIR, "cache") -CCACHEDIR = os.path.join(BASEDIR, "ccache") -CONFIGDIR = os.path.join(BASEDIR, "config") -DOCDIR = os.path.join(BASEDIR, "doc") -GENDIR = os.path.join(BUILDDIR, "generators") -IMAGESDIR = os.path.join(BUILDDIR, "images") -LOGDIR = os.path.join(BASEDIR, "logs") -PKGSDIR = os.path.join(BASEDIR, "pkgs") -PACKAGESDIR = os.path.join(BUILDDIR, "packages") -REPOSDIR = os.path.join(BUILDDIR, "repositories") -TOOLSDIR = os.path.join(BASEDIR, "tools") - -ARCHES_DEFAULT = os.path.join(CONFIGDIR, "architectures.conf") - -CONFIGFILE = os.path.join(CONFIGDIR, "naoki.conf") - -CHROOT_PATH = "/sbin:/bin:/usr/sbin:/usr/bin" - -LOCK_BATCH = os.path.join(BUILDDIR, ".batch") - -LOG_MARKER = "### LOG MARKER ###" - -DEP_INVALID, DEP_FILE, DEP_LIBRARY, DEP_PACKAGE = range(4) - -def calc_parallelism(): - """ - Calculate how many processes to run - at the same time. - - We take the log10(number of processors) * factor - """ - num = os.sysconf("SC_NPROCESSORS_CONF") - if num == 1: - return 2 - else: - return int(round(math.log10(num) * 26)) - -class Config(object): - _items = { - "toolchain" : False, - "mandatory_packages" : [ - "build-essentials", - ], - "shell_packages" : [ - "/bin/bash", - "less", - "vim", - ], - "nice_level" : 0, - "parallelism" : calc_parallelism(), - # - # Cleanup settings - "cleanup_on_failure" : False, - "cleanup_on_success" : True, - # - # Shell settings - "shell_on_failure" : False, - # - # CLI variables - "debug" : False, - "quiet" : False, - # - # Distro items - "distro_name" : "unknown", - "distro_sname" : "unknown", - "distro_epoch" : "unknown", - "distro_version" : "unknown", - "distro_slogan" : "unknown", - "disto_vendor" : "unknown", - # - # Logging - "log_file" : os.path.join(LOGDIR, "naoki.log"), - # - # Reporting - "error_report_recipient" : None, - "error_report_sender" : "buildsystem@%s" % socket.gethostname(), - "error_report_subject" : "[NAOKI] %(id)s got a build failure", - # - # SMTP - "smtp_server" : None, - "smtp_user" : None, - "smtp_password" : None, - } - - def __init__(self): - self.read([CONFIGFILE, os.path.join(BASEDIR, ".config")]) - - def read(self, files): - parser = ConfigParser() - parser.read(files) - - config = {} - for key, val in parser.items(DEFAULTSECT): - config[key] = val - - for section in parser.sections(): - for key, val in parser.items(section): - config["%s_%s" % (section, key)] = val - - self._items.update(config) - - def items(self): - return self._items.items() - - def __getitem__(self, item): - return self._items[item] - - def __setitem__(self, item, value): - self._items[item] = value - - def __getattr__(self, *args): - return self.__getitem__(*args) - - def __setattr__(self, *args): - return self.__setitem__(*args) - - @property - def environment(self): - ret = { - "HOME" : os.environ.get("HOME", "/root"), - "TERM" : os.environ.get("TERM", ""), - "PS1" : "\u:\w\$ ", - # - "DISTRO_NAME" : self["distro_name"], - "DISTRO_SNAME" : self["distro_sname"], - "DISTRO_EPOCH" : self["distro_epoch"], - "DISTRO_VERSION" : self["distro_version"], - "DISTRO_SLOGAN" : self["distro_slogan"], - "DISTRO_VENDOR" : self["distro_vendor"], - # - "PARALLELISMFLAGS" : "-j%d" % self["parallelism"], - } - - if self["debug"]: - ret["NAOKI_DEBUG"] = "1" - - return ret - - -# Create a globally useable instance of the configuration -config = Config() - diff --git a/naoki/decorators.py b/naoki/decorators.py deleted file mode 100644 index 5b26ace0b..000000000 --- a/naoki/decorators.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/python - -# A decorator to use singleton on a class -# http://en.wikipedia.org/wiki/Singleton_pattern#Python_.28using_decorators.29 -def singleton(cls): - instance_container = [] - def getinstance(): - if not len(instance_container): - instance_container.append(cls()) - return instance_container[0] - return getinstance - diff --git a/naoki/dependencies.py b/naoki/dependencies.py deleted file mode 100644 index 395c70ea4..000000000 --- a/naoki/dependencies.py +++ /dev/null @@ -1,249 +0,0 @@ -#!/usr/bin/python - -import logging -import os -import re - -import architectures -import repositories -import packages - -from constants import * -from exception import * - -class Dependency(object): - def __init__(self, identifier, origin=None): - self.identifier = identifier - self.origin = origin - - def __repr__(self): - s = "<%s %s" % (self.__class__.__name__, self.identifier) - if self.origin: - s += " by %s" % self.origin.name - s += "(%s)" % os.path.basename(self.origin.filename) - s += ">" - return s - - @property - def type(self): - if self.identifier.startswith("/"): - return DEP_FILE - elif ".so" in self.identifier: - return DEP_LIBRARY - elif re.match("^[A-Za-z0-9\-\+_]+$", self.identifier): - return DEP_PACKAGE - - return DEP_INVALID - - def match(self, other): - return self.type == other.type \ - and self.identifier == other.identifier - - __eq__ = match - - -class Provides(Dependency): - pass - - -class DependencySet(object): - def __init__(self, dependencies=[], arch=None): - if not arch: - arches = architectures.Architectures() - arch = arches.get_default() - self.arch = arch - - self.repo = repositories.BinaryRepository(self.arch) - - # initialize package lists - self._dependencies = [] - self._items = [] - - # caches - self.__provides = {} - self.__dependencies = {} - - # add all provided dependencies - for dependency in dependencies: - self.add_dependency(dependency) - - logging.debug("Successfully initialized %s" % self) - - @property - def hash(self): - hash = ["%s" % i for i in self._items] - - return "".join(hash) - - def __repr__(self): - return "<%s>" % (self.__class__.__name__) - - def add_dependency(self, item): - assert isinstance(item, Dependency) - - # If this dependency is already provided by something else - # we skip the add - for p in self.provides: - if p.match(item): - logging.debug("%s matches %s" % (p, item)) - return - - if not item in self.dependencies: - self._dependencies.append(item) - - logging.debug("Added new dependency %s" % item) - - def add_package(self, item): - assert isinstance(item, packages.BinaryPackage) - - if item in self._items: - return - - # Packages conflict - for package in self._items: - if package.name == item.name: - if item > package: - logging.debug("Replacing package %s by %s" % (package, item)) - self._items.remove(package) - break - - self._items.append(item) - logging.debug("Added new package %s" % item) - - def resolve(self): - logging.debug("Resolving %s" % self) - - # Safe for endless loop - counter = 1000 - - while True: - counter -= 1 - if not counter: - logging.debug("Maximum count of dependency loop was reached") - break - - dependency = self.next_unresolved_dependency - if not dependency: - break - - logging.debug("Processing dependency: %s" % dependency.identifier) - - if dependency.type == DEP_PACKAGE: - package = self.repo.find_package_by_name(dependency.identifier) - if package: - # Found a package and add this - self.add_package(package) - continue - - elif dependency.type == DEP_LIBRARY: - package = self.repo.find_package_by_provides(dependency) - if package: - self.add_package(package) - continue - - elif dependency.type == DEP_FILE: - package = self.repo.find_package_by_file(dependency.identifier) - if package: - self.add_package(package) - continue - - logging.warning("Cannot find a package which provides file %s" % \ - dependency.identifier) - - elif dependency.type == DEP_INVALID: - logging.warning("Dropping invalid dependency %s" % dependency.identifier) - continue - - # Found not solution - logging.debug("No match found: %s" % dependency) - - if self.unresolved_dependencies: - #logging.error("Unresolveable dependencies: %s" % self.dependencies) - raise DependencyResolutionError, "%s" % self.unresolved_dependencies - - @property - def unresolved_dependencies(self): - return self.calculate_unresolved_dependencies(next=False) - - @property - def next_unresolved_dependency(self): - d = self.calculate_unresolved_dependencies(next=True) - if d: - return d[0] - - def calculate_unresolved_dependencies(self, next=False): - dependencies = [] - - # Cache provides - provides = self.provides - - for dependency in self._dependencies + self.dependencies: - if dependency.type == DEP_INVALID: - continue - - found = False - for item in self._items: - if item.does_provide(dependency): - found = True - break - - if found: - continue - - #for provide in provides: - # if dependency.match(provide): - # found = True - # break - # - #if found: - # continue - - dependencies.append(dependency) - - # If next is set return only one. - if next: - return dependencies - - return dependencies - - @property - def dependencies(self): - if not self.__dependencies.has_key(self.hash): - self.__dependencies[self.hash] = self.calculate_dependencies() - - return self.__dependencies[self.hash] - - def calculate_dependencies(self): - dependencies = [] - for item in self._items: - dependencies += item.get_dependencies() - - return list(set(dependencies)) - - @property - def packages(self): - return sorted(self._items) - - def calculate_provides(self): - provides = [] - for item in self._items: - provides += item.get_provides() - - return list(set(provides)) - - @property - def provides(self): - if not self.__provides.has_key(self.hash): - self.__provides[self.hash] = self.calculate_provides() - - return self.__provides[self.hash] - - -if __name__ == "__main__": - import architectures - arches = architectures.Architectures() - - ds = DependencySet(arch=arches.get("i686")) - ds.add_dependency(Dependency("/bin/bash")) - ds.resolve() - print ds.packages diff --git a/naoki/environ.py b/naoki/environ.py deleted file mode 100644 index 8f2ecbc2f..000000000 --- a/naoki/environ.py +++ /dev/null @@ -1,375 +0,0 @@ -#!/usr/bin/python - -import grp -import logging -import os -import stat -import time - -import logger -import mail -import util - -from constants import * - -# XXX to be moved to somewhere else -ENVIRONMENT_ARGS = ["PATH", "PWD" ] - -def set(e): - env = {} - - for key in ENVIRONMENT_ARGS: - if os.environ.has_key(key): - env[key] = os.environ[key] - - env.update(config.environment) - env.update(e) - - return env - - -class _Environment(object): - kernel_version = os.uname()[2] - - def __init__(self, arch, **settings): - self.arch = arch - - self._settings = { - "enable_loop" : False, - } - self._settings.update(settings) - - self.logger.debug("Successfully initialized %s" % self) - - # XXX check if already locked - self.prepare() - - def chrootPath(self, *args): - raise NotImplementedError - - def clean(self): - self.logger.debug("Cleaning environment %s" % self) - if os.path.exists(self.chrootPath()): - util.rm(self.chrootPath()) - - def prepare(self): - self.clean() - - self.logger.debug("Preparing environment %s" % self) - - dirs = ( - CACHEDIR, - CCACHEDIR, - IMAGESDIR, - PACKAGESDIR, - "dev", - "dev/pts", - "dev/shm", - "proc", - "root", - "sys", - "tmp", - "usr/src", - "usr/src/cache", - "usr/src/ccache", - "usr/src/images", - "usr/src/packages", - "usr/src/pkgs", - "usr/src/src", - "usr/src/tools", - "var/tmp", - ) - - for dir in dirs: - util.mkdir(self.chrootPath(dir)) - - files = ( - "etc/fstab", - "etc/mtab" - ) - - for file in files: - file = self.chrootPath(file) - dir = os.path.dirname(file) - if not os.path.exists(dir): - util.mkdir(dir) - util.touch(file) - - # Prepare the other stuff - self._prepare_dev() - self._prepare_users() - self._prepare_dns() - - def _prepare_dev(self): - prevMask = os.umask(0000) - - devNodes = [ - (stat.S_IFCHR | 0666, os.makedev(1, 3), "dev/null"), - (stat.S_IFCHR | 0666, os.makedev(1, 7), "dev/full"), - (stat.S_IFCHR | 0666, os.makedev(1, 5), "dev/zero"), - (stat.S_IFCHR | 0666, os.makedev(1, 8), "dev/random"), - (stat.S_IFCHR | 0444, os.makedev(1, 9), "dev/urandom"), - (stat.S_IFCHR | 0666, os.makedev(5, 0), "dev/tty"), - (stat.S_IFCHR | 0600, os.makedev(5, 1), "dev/console") - ] - - if self._settings["enable_loop"]: - for i in range(0, 7): - devNodes.append((stat.S_IFBLK | 0660, os.makedev(7, i), "dev/loop%d" % i)) - - # make device node for el4 and el5 - if self.kernel_version < "2.6.19": - devNodes.append((stat.S_IFCHR | 0666, os.makedev(5, 2), "dev/ptmx")) - - for i in devNodes: - # create node - os.mknod(self.chrootPath(i[2]), i[0], i[1]) - - os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin")) - os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout")) - os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr")) - os.symlink("/proc/self/fd", self.chrootPath("dev", "fd")) - - if self.kernel_version >= "2.6.19": - os.symlink("/dev/pts/ptmx", self.chrootPath("dev", "ptmx")) - - os.umask(prevMask) - - def _prepare_users(self): - f = open(self.chrootPath("etc", "passwd"), "w") - f.write("root:x:0:0:root:/root:/bin/bash\n") - f.write("nobody:x:99:99:Nobody:/:/sbin/nologin\n") - f.close() - - f = open(self.chrootPath("etc", "group"), "w") - f.write("root:x:0:root\n") - f.write("nobody:x:99:\n") - f.close() - - def _prepare_dns(self): - # XXX to be replaced - nameservers = [] - f = open("/etc/resolv.conf") - for line in f.readlines(): - if line.startswith("nameserver"): - nameservers.append(line.split(" ")[-1].strip()) - f.close() - - self.logger.debug("Using nameservers: %s" % nameservers) - - f = open(self.chrootPath("etc", "resolv.conf"), "w") - for nameserver in nameservers: - f.write("nameserver %s" % nameserver) - f.close() - - self.logger.debug("Creating record for localhost") - f = open(self.chrootPath("etc", "hosts"), "w") - f.write("127.0.0.1 localhost\n") - f.close() - - def extract(self, package, *args): - self.logger.info("Extracting %s" % package) - - package.extract(self.chrootPath(*args)) - - def variables(self): - env = set({ - "HOME" : "/root", - "PATH" : "/sbin:/bin:/usr/sbin:/usr/bin", - "BASEDIR" : "/usr/src", - "PKGROOT" : "/usr/src/pkgs", - "TARGET" : "%s-ipfire-linux-gnu" % self.arch.machine, - "TARGET_MACHINE" : self.arch.machine, - "CHROOT" : "1", # XXX to be removed - "CFLAGS" : self.arch.cflags, - "CXXFLAGS" : self.arch.cxxflags, - "PKG_ARCH" : self.arch.name, - }) - - # Settings for ccache - env.update({ - "PATH" : "/usr/ccache/bin:%s" % env["PATH"], - "CCACHE_COMPILERCHECK" : "none", - "CCACHE_COMPRESS" : "1", - "CCACHE_DIR" : "/usr/src/ccache", - }) - - return env - - def doChroot(self, command, shell=True, *args, **kwargs): - ret = None - try: - env = self.variables() - - if kwargs.has_key("env"): - env.update(kwargs.pop("env")) - - self._mountall() - - if not kwargs.has_key("chrootPath"): - kwargs["chrootPath"] = self.chrootPath() - - ret = util.do(command, - personality=self.arch.personality, - shell=shell, - env=env, - logger=self.logger, - *args, **kwargs) - - finally: - self._umountall() - - return ret - - def _mountall(self): - self.logger.debug("Mounting environment") - for cmd in self.mountCmds: - util.do(cmd, shell=True) - - def _umountall(self): - self.logger.debug("Umounting environment") - for cmd in self.umountCmds: - util.do(cmd, raiseExc=0, shell=True) - - @property - def umountCmds(self): - return ( - "umount -n %s" % self.chrootPath("proc"), - "umount -n %s" % self.chrootPath("sys"), - "umount -n %s" % self.chrootPath("usr", "src", "cache"), - "umount -n %s" % self.chrootPath("usr", "src", "ccache"), - "umount -n %s" % self.chrootPath("usr", "src", "images"), - "umount -n %s" % self.chrootPath("usr", "src", "packages"), - "umount -n %s" % self.chrootPath("usr", "src", "pkgs"), - "umount -n %s" % self.chrootPath("usr", "src", "src"), - "umount -n %s" % self.chrootPath("usr", "src", "tools"), - "umount -n %s" % self.chrootPath("dev", "pts"), - "umount -n %s" % self.chrootPath("dev", "shm") - ) - - @property - def mountCmds(self): - ret = [ - "mount -n -t proc naoki_chroot_proc %s" % self.chrootPath("proc"), - "mount -n -t sysfs naoki_chroot_sysfs %s" % self.chrootPath("sys"), - "mount -n --bind %s %s" % (CACHEDIR, self.chrootPath("usr", "src", "cache")), - "mount -n --bind %s %s" % (CCACHEDIR, self.chrootPath("usr", "src", "ccache")), - "mount -n --bind %s %s" % (IMAGESDIR, self.chrootPath("usr", "src", "images")), - "mount -n --bind %s %s" % (PACKAGESDIR, self.chrootPath("usr", "src", "packages")), - "mount -n --bind %s %s" % (PKGSDIR, self.chrootPath("usr", "src", "pkgs")), - "mount -n --bind %s %s" % (os.path.join(BASEDIR, "src"), self.chrootPath("usr", "src", "src")), - "mount -n --bind %s %s" % (TOOLSDIR, self.chrootPath("usr", "src", "tools")), - ] - - mountopt = "gid=%d,mode=0620,ptmxmode=0666" % grp.getgrnam("tty").gr_gid - if self.kernel_version >= "2.6.29": - mountopt += ",newinstance" - - ret.extend([ - "mount -n -t devpts -o %s naoki_chroot_devpts %s" % (mountopt, self.chrootPath("dev", "pts")), - "mount -n -t tmpfs naoki_chroot_shmfs %s" % self.chrootPath("dev", "shm")]) - - return ret - - @property - def logger(self): - return logging.getLogger() - - -class Build(_Environment): - def __init__(self, package, build_id=None): - self.package = package - self.build_id = build_id - - _Environment.__init__(self, self.package.arch) - - @property - def logger(self): - if not hasattr(self, "_logger"): - logfile = os.path.join(LOGDIR, self.package.repository.name, - self.package.name, self.package.id + ".log") - - # Ensure that logging directory exists - util.mkdir(os.path.dirname(logfile)) - - self._logger = logging.getLogger(self.package.id) - - fh = logging.FileHandler(logfile) - fh.setFormatter(logging.Formatter("%(message)s")) - fh.setLevel(logging.NOTSET) - - self._logger.addHandler(fh) - - return self._logger - - def chrootPath(self, *args): - return os.path.join(BUILDDIR, "environments", self.package.id, *args) - - @property - def buildroot(self): - if not hasattr(self, "__buildroot"): - self.__buildroot = "buildroot.%s" % util.random_string() - - return self.__buildroot - - def variables(self): - v = _Environment.variables(self) - v.update({ - "BUILDROOT" : "/%s" % self.buildroot, - "BUILD_ID" : self.build_id or "", - }) - return v - - def build(self, *args, **kwargs): - # Save starting time - time_start = time.time() - - try: - self.make("package") - except: - if config["shell_on_failure"]: - self.shell() - return - - if config["cleanup_on_failure"]: - self.clean() - - # Send email report about an error - mail.report_error(self.package) - raise - - # Log time that was needed to build the package - self.logger.info("Build time: %.2fs" % (time.time() - time_start)) - - if config["cleanup_on_success"]: - self.clean() - - def make(self, target): - file = "/usr/src%s" % self.package.filename[len(BASEDIR):] - - return self.doChroot("make -C %s -f %s %s" % \ - (os.path.dirname(file), file, target), shell=True) - - def shell(self, args=[]): - command = "%s chroot %s /usr/src/tools/chroot-shell %s" % \ - (self.arch.personality, self.chrootPath(), " ".join(args)) - - for key, val in self.variables().items(): - command = "%s=\"%s\" " % (key, val) + command - - if self.package.source_dir: - command = "SOURCE_DIR=%s %s" % (self.package.source_dir, command) - - logging.debug("Shell command: %s" % command) - - try: - self._mountall() - - shell = os.system(command) - return os.WEXITSTATUS(shell) - - finally: - self._umountall() - - # Clean up the environment - self.clean() diff --git a/naoki/exception.py b/naoki/exception.py deleted file mode 100644 index 703b6b5fe..000000000 --- a/naoki/exception.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/python - -class Error(Exception): - "base class for our errors." - def __init__(self, msg, status=None): - Exception.__init__(self) - self.msg = msg - self.resultcode = 1 - if status is not None: - self.resultcode = status - - def __str__(self): - return self.msg - - -class BuildRootLocked(Error): - "build root in use by another process." - def __init__(self, msg): - Error.__init__(self, msg) - self.msg = msg - self.resultcode = 60 - - -class commandTimeoutExpired(Error): - def __init__(self, msg): - Error.__init__(self, msg) - self.msg = msg - self.resultcode = 10 - - -class DownloadError(Error): - pass - - -class DependencyResolutionError(Error): - pass diff --git a/naoki/generators.py b/naoki/generators.py deleted file mode 100644 index d3c995bcf..000000000 --- a/naoki/generators.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/python - -import logging -import os - -import dependencies -import repositories -import util - -from constants import * -from environ import _Environment - - -class Generator(_Environment): - def __init__(self, type, arch=None): - _Environment.__init__(self, arch, enable_loop=True) - - self.type = type - - self.repos = repositories.SourceRepositories(self.arch) - - for r in self.repos.all: - if not r.completely_built: - raise Exception, "The repo is not built completely: %s" % r - - self.build_deps = dependencies.DependencySet(arch=self.arch) - deps = [ - "basesystem", - "dracut", - "e2fsprogs", - "kernel", - "squashfs-tools", - "syslinux", - "util-linux-ng", - "zerofree", - "/sbin/dmsetup", - "/usr/bin/mkisofs", - ] - for dep in deps: - dep = dependencies.Dependency(dep) - self.build_deps.add_dependency(dep) - - self.installer_deps = dependencies.DependencySet(arch=self.arch) - deps = [ - "basesystem", - "installer", - # TODO needs to be replaced - "xorg-x11-drv-ati", - "xorg-x11-drv-evdev", - "xorg-x11-drv-intel", - "xorg-x11-drv-keyboard", - "xorg-x11-drv-mouse", - "xorg-x11-drv-nv", - "xorg-x11-drv-synaptics", - "xorg-x11-drv-vesa", - "xorg-x11-drv-vmware", - ] - for dep in deps: - dep = dependencies.Dependency(dep) - self.installer_deps.add_dependency(dep) - - def chrootPath(self, *args): - return os.path.join(GENDIR, self.arch.name, *args) - - @property - def logger(self): - return logging.getLogger() - - def run(self): - # Unpacking packages we need in this environment - logging.info("Resolving dependencies...") - self.build_deps.resolve() - self.installer_deps.resolve() - - for package in self.build_deps.packages: - self.extract(package) - - util.mkdir(self.chrootPath("installer")) - for package in self.installer_deps.packages: - self.extract(package, "installer") - - util.mkdir(self.chrootPath("packages")) - for p in self.repos.packages: - for bin in p.binary_files: - os.link(os.path.join(PACKAGESDIR, self.arch.name, bin), - self.chrootPath("packages", bin)) - - self.doChroot("/usr/src/tools/generator %s" % self.type, shell=True) diff --git a/naoki/io.py b/naoki/io.py deleted file mode 100644 index 6f06caa4a..000000000 --- a/naoki/io.py +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/python - -import grp -import os -import pwd -import stat -import time - -def ftype(mode): - if stat.S_ISBLK(mode): - return "b" - elif stat.S_ISCHR(mode): - return "c" - elif stat.S_ISDIR(mode): - return "d" - elif stat.S_ISREG(mode): - return "-" - elif stat.S_ISFIFO(mode): - return "p" - elif stat.S_ISLINK(mode): - return "l" - elif stat.S_ISSOCK(mode): - return "s" - return "?" - -def rwx(mode): - ret = "" - if mode & stat.S_IRUSR: - ret += "r" - else: - ret += "-" - - if mode & stat.S_IWUSR: - ret += "w" - else: - ret += "-" - - if mode & stat.S_IXUSR: - ret += "x" - else: - ret += "-" - - return ret - -def fmode(mode): - ret = ftype(mode) - ret += rwx((mode & 0700) << 0) - ret += rwx((mode & 0070) << 3) - ret += rwx((mode & 0007) << 6) - return ret - -class CpioError(Exception): - pass - - -class CpioEntry(object): - def __init__(self, hdr, archive, offset): - self.archive = archive - self.hdr = hdr - - self.offset = offset + 110 + self.namesize - self.offset += (4 - (self.offset % 4)) % 4 - self.current = 0 - - self.closed = False - - if len(self.hdr) < 110: - raise CpioError("Header too short.") - - if not self.hdr.startswith("070701") and not self.hdr.startswith("070702"): - raise CpioError("Invalid header: %s" % self.hdr[:6]) - - def close(self): - self.closed = True - - def flush(self): - pass # noop - - def read(self, size=None): - """Read data from the entry. - - Keyword arguments: - size -- Number of bytes to read (default: whole entry) - """ - if self.closed: - raise ValueError("Read operation on closed file.") - - self.archive.file.seek(self.offset + self.current, os.SEEK_SET) - - if size and size < self.size - self.current: - ret = self.archive.file.read(size) - else: - ret = self.archive.file.read(self.size - self.current) - self.current += len(ret) - return ret - - def seek(self, offset, whence=0): - """Move to new position within an entry. - - Keyword arguments: - offset -- Byte count - whence -- Describes how offset is used. - 0: From beginning of file - 1: Forwards from current position - 2: Backwards from current position - Other values are ignored. - """ - if self.closed: - raise ValueError("Seek operation on closed file.") - - if whence == os.SEEK_SET: - self.current = offset - elif whence == os.SEEK_REL: - self.current += offset - elif whence == os.SEEK_END: - self.current -= offset - - self.current = min(max(0, self.current), self.size) - - def tell(self): - """Get current position within an entry""" - if self.closed: - raise ValueError("Tell operation on closed file.") - return self.current - - def __repr__(self): - return "" % (self.name, self.checksum,) - - @property - def checksum(self): - return int(self.hdr[102:110], 16) - - @property - def devmajor(self): - return int(self.hdr[62:70], 16) - - @property - def devminor(self): - return int(self.hdr[70:78], 16) - - @property - def gid(self): - return int(self.hdr[30:38], 16) - - @property - def inode(self): - return int(self.hdr[6:14], 16) - - @property - def mode(self): - return int(self.hdr[14:22], 16) - - @property - def mtime(self): - return int(self.hdr[46:54], 16) - - @property - def name(self): - end = 110 + self.namesize - 1 - return self.hdr[110:end] - - @property - def namesize(self): - return int(self.hdr[94:102], 16) - - @property - def nlinks(self): - return int(self.hdr[38:46], 16) - - @property - def rdevmajor(self): - return int(self.hdr[78:86], 16) - - @property - def rdevminor(self): - return int(self.hdr[86:94], 16) - - @property - def size(self): - return int(self.hdr[54:62], 16) - - @property - def uid(self): - return int(self.hdr[22:30], 16) - - -class CpioArchive(object): - _entries = [] - file = None - - def __init__(self, filename): - - self.filename = filename - self.file = open(self.filename, "r") - self.__readfile() - - self.closed = False - - def close(self): - if self.closed: - return - self.closed = True - - self.file.close() - - def __readfile(self): - if not self.file: - raise CpioError("File was not yet opened.") - - self._entries = [] - sposition = self.file.tell() - hdr = self.file.read(110) - while hdr: - namelen = int(hdr[94:102], 16) # Length of the name - hdr += self.file.read(namelen) - ce = CpioEntry(hdr, self, sposition) - if ce.name == "TRAILER!!!": - return - self._entries.append(ce) - - self.file.seek((4 - (self.file.tell()-sposition) % 4) % 4, os.SEEK_CUR) - self.file.seek(ce.size, os.SEEK_CUR) - self.file.seek((4 - (self.file.tell()-sposition) % 4) % 4, os.SEEK_CUR) - - sposition = self.file.tell() - hdr = self.file.read(110) - else: - raise CpioError("Premature end of headers.") - - @property - def entries(self): - return sorted(self._entries) - - @property - def size(self): - return os.path.getsize(self.filename) - - def ls(self): - for x in self.entries: - print x.name - - def ll(self): - for x in self.entries: - print "%s %s %s %s %9d %s %s" % \ - (fmode(x.mode), - x.nlinks, - pwd.getpwuid(x.uid)[0], - grp.getgrgid(x.gid)[0], - x.size, - time.strftime("%Y-%m-%d %H:%M", time.localtime(x.mtime)), - x.name,) - - def get(self, item): - for x in self.entries: - if x.name == item: - return x - raise KeyError("No such file or directory.") - - def __getitem__(self, item): - x = self.get(item) - x.seek(0) - return x.read() diff --git a/naoki/logger.py b/naoki/logger.py deleted file mode 100644 index 17a409e4c..000000000 --- a/naoki/logger.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/python - -import curses -import logging -import logging.handlers -import sys -import time - -# fix for python 2.4 logging module bug: -logging.raiseExceptions = 0 - -from constants import * - -class Logging(object): - def __init__(self): - self.log = logging.getLogger() - self.log.setLevel(logging.NOTSET) - - # Initialize console - ch = self.console - ch.setLevel(logging.INFO) - - if sys.stderr.isatty(): - curses.setupterm() - ch.setFormatter(_ColorLogFormatter()) - else: - ch.setFormatter(logging.Formatter("[%(levelname)s] %(message)s")) - self.log.addHandler(ch) - - # Initialize log file - if not os.path.isdir(LOGDIR): - os.makedirs(LOGDIR) - fh = logging.handlers.RotatingFileHandler(config["log_file"], - maxBytes=10*1024**2, backupCount=6) - fh.setFormatter(logging.Formatter("[%(levelname)s] %(message)s")) - fh.setLevel(logging.DEBUG) - self.log.addHandler(fh) - - def debug(self, val): - if val: - self.console.setLevel(logging.DEBUG) - self.log.debug("Enabled debug logging mode") - else: - self.log.debug("Disabled debug logging mode") - self.console.setLevel(logging.INFO) - - @property - def console(self): - console = self.log.handlers[0] - - assert isinstance(console, logging.StreamHandler) - - return console - - -# defaults to module verbose log -# does a late binding on log. Forwards all attributes to logger. -# works around problem where reconfiguring the logging module means loggers -# configured before reconfig dont output. -class getLog(object): - def __init__(self, name=None, prefix="", *args, **kargs): - if name is None: - frame = sys._getframe(1) - name = frame.f_globals["__name__"] - - self.name = prefix + name - - def __getattr__(self, name): - logger = logging.getLogger(self.name) - return getattr(logger, name) - - -# Borrowed from tornado -class _ColorLogFormatter(logging.Formatter): - def __init__(self, *args, **kwargs): - logging.Formatter.__init__(self, *args, **kwargs) - fg_color = curses.tigetstr("setaf") or curses.tigetstr("setf") or "" - self._colors = { - logging.DEBUG: curses.tparm(fg_color, 4), # Blue - logging.INFO: curses.tparm(fg_color, 2), # Green - logging.WARNING: curses.tparm(fg_color, 3), # Yellow - logging.ERROR: curses.tparm(fg_color, 1), # Red - } - self._normal = curses.tigetstr("sgr0") - - def format(self, record): - try: - record.message = record.getMessage() - except Exception, e: - record.message = "Bad message (%r): %r" % (e, record.__dict__) - record.asctime = time.strftime( - "%H:%M:%S", self.converter(record.created)) - prefix = " %(levelname)-7s" % record.__dict__ - color = self._colors.get(record.levelno, self._normal) - formatted = color + prefix + self._normal + " " + record.message - if record.exc_info: - if not record.exc_text: - record.exc_text = self.formatException(record.exc_info) - if record.exc_text: - formatted = formatted.rstrip() + "\n" + record.exc_text - return formatted.replace("\n", "\n ") diff --git a/naoki/mail.py b/naoki/mail.py deleted file mode 100644 index fdd723539..000000000 --- a/naoki/mail.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/python - -import email.mime.multipart -import email.mime.text -import logging -import os -import smtplib - -from constants import * - -def report_error(package): - # Do not send a report if no recipient is configured - if not config["error_report_recipient"]: - return - - try: - connection = smtplib.SMTP(config["smtp_server"]) - #connection.set_debuglevel(1) - - if config["smtp_user"] and config["smtp_password"]: - connection.login(config["smtp_user"], config["smtp_password"]) - - except smtplib.SMTPConnectError, e: - logging.error("Could not establish a connection to the smtp server: %s" % e) - return - except smtplib.SMTPAuthenticationError, e: - logging.error("Could not successfully login to the smtp server: %s" % e) - return - - msg = email.mime.multipart.MIMEMultipart() - msg["From"] = config["error_report_sender"] - msg["To"] = config["error_report_recipient"] - msg["Subject"] = config["error_report_subject"] % package.all - msg.preamble = 'You will not see this in a MIME-aware mail reader.\n' - - body = """\ -The package %(name)s had a difficulty to build itself. -This email will give you a short report about the error. - -Package information: - Name : %(name)s - %(summary)s - Version : %(version)s - Release : %(release)s - - This package in maintained by %(maintainer)s. - - -A detailed logfile is attached. - -Sincerely, - Naoki - """ % { - "name" : package.name, - "summary" : package.summary, - "version" : package.version, - "release" : package.release, - "maintainer" : package.maintainer, - } - - msg.attach(email.mime.text.MIMEText(body)) - - # Read log and append it to mail - loglines = [] - if os.path.exists(package.logfile): - f = open(package.logfile) - line = f.readline() - while line: - line = line.rstrip() - if line.endswith(LOG_MARKER): - # Reset log - loglines = [] - - loglines.append(line) - line = f.readline() - - f.close() - - if not loglines: - loglines = ["Logfile wasn't found."] - - log = email.mime.text.MIMEText("\n".join(loglines), _subtype="plain") - log.add_header('Content-Disposition', 'attachment', - filename="%s.log" % package.id) - msg.attach(log) - - try: - connection.sendmail(config["error_report_sender"], - config["error_report_recipient"], msg.as_string()) - except Exception, e: - logging.error("Could not send error report: %s: %s" % (e.__class__.__name__, e)) - return - - connection.quit() diff --git a/naoki/packages.py b/naoki/packages.py deleted file mode 100644 index 0c66ef5cf..000000000 --- a/naoki/packages.py +++ /dev/null @@ -1,319 +0,0 @@ -#!/usr/bin/python - -import logging -import os -import re - -import urlgrabber -import urlgrabber.progress - -import architectures -import dependencies -import environ -import io -import util - -from constants import * -from exception import * - -def version_compare_epoch(e1, e2): - return cmp(e1, e2) - -def version_compare_version(v1, v2): - return cmp(v1, v2) - -def version_compare_release(r1, r2): - return cmp(r1, r2) - -def version_compare((e1, v1, r1), (e2, v2, r2)): - - ret = version_compare_epoch(e1, e2) - if not ret == 0: - return ret - - ret = version_compare_version(v1, v2) - if not ret == 0: - return ret - - return version_compare_release(r1, r2) - - -class Package(object): - def __repr__(self): - return "<%s %s:%s>" % \ - (self.__class__.__name__, self.name, self.arch.name) - - #@property - #def arch(self): - # raise NotImplementedError - - @property - def epoch(self): - epoch = self._info.get("PKG_EPOCH", None) - if not epoch: - epoch = 0 - return int(epoch) - - @property - def name(self): - return self._info["PKG_NAME"] - - @property - def group(self): - return self._info["PKG_GROUP"] - - @property - def id(self): - return "%s-%s-%s.%s" % \ - (self.name, self.version, self.release, self.arch.name) - - @property - def release(self): - return int(self._info["PKG_REL"]) - - @property - def version(self): - return self._info["PKG_VER"] - - def print_raw_info(self): - for k in sorted(self._info.keys()): - print "%s=\"%s\"" % (k, self._info.get(k)) - - def fmtstr(self, s): - return s % self._info - - -class SourcePackage(Package): - def __init__(self, filename, repo, arch): - self.arch = arch - self.filename = filename - self.repository = repo - - self.init() - - logging.debug("Successfully initialized %s" % self) - - def init(self): - self._info = {} - - env = environ.set({ - "PKG_ARCH" : self.arch.name, - "PKGROOT" : PKGSDIR, - }) - - output = util.do("make -f %s" % os.path.basename(self.filename), - shell=True, - cwd=os.path.dirname(self.filename), - returnOutput=1, - env=env) - - for line in output.splitlines(): - if not line: - continue - - m = re.match(r"^(\w+)=(.*)$", line) - if m is None: - continue - - key, val = m.groups() - self._info[key] = val.strip("\"") - - # XXX should return a dependencyset - def get_dependencies(self, type=""): - type2var = { - "" : "PKG_DEPENDENCIES", - "build" : "PKG_BUILD_DEPENDENCIES", - } - - type = type2var[type] - - return [dependencies.Dependency(d, origin=self) for d in self._info[type].split()] - - @property - def source_dir(self): - return self._info.get("SOURCE_DIR", "") - - @property - def summary(self): - return self._info["PKG_SUMMARY"] - - @property - def binary_files(self): - return self._info.get("PKG_PACKAGES_FILES").split() - - @property - def is_built(self): - for file in self.binary_files: - file = os.path.join(PACKAGESDIR, self.arch.name, file) - if not os.path.exists(file): - return False - - return True - - @property - def source_files(self): - return self._info.get("PKG_OBJECTS").split() - - def source_download(self): - g = urlgrabber.grabber.URLGrabber( - prefix = config["sources_download_url"], - progress_obj = urlgrabber.progress.TextMeter(), - quote=0, - ) - - for file in self.source_files: - file = os.path.join(CACHEDIR, file) - - if os.path.exists(file): - continue - - util.mkdir(CACHEDIR) - - try: - g.urlgrab(os.path.basename(file), filename=file) - except urlgrabber.grabber.URLGrabError,e : - raise DownloadError, "%s %s" % (os.path.basename(file), e) - - -class BinaryPackage(Package): - def __init__(self, filename): - self.filename = filename - - self.init() - logging.debug("Successfully initialized %s" % self) - - def __repr__(self): - return "<%s %s:%s-%s-%s>" % \ - (self.__class__.__name__, self.epoch, self.name, self.version, - self.release) - - def __cmp__(self, other): - # Packages are equal - if self.id == other.id: - return 0 - - # if packages differ names return in alphabetical order - if not self.name == other.name: - return cmp(self.name, other.name) - - ret = version_compare((self.epoch, self.version, self.release), - (other.epoch, other.version, other.release)) - - if ret == 0: - logging.debug("%s is equal to %s" % (self, other)) - elif ret < 0: - logging.debug("%s is more recent than %s" % (other, self)) - elif ret > 0: - logging.debug("%s is more recent than %s" % (self, other)) - - return ret - - def _readfile(self, name): - try: - f = io.CpioArchive(self.filename) - except ValueError, e: - raise Exception, "Could not open package '%s' correctly: %s" \ - % (self.filename, e) - - # If file is not available, return None - ret = None - if name in [e.name for e in f.entries]: - ret = f.get(name).read() - - f.close() - return ret - - def init(self): - self._info = {} - self._filelist = [] - - info = self._readfile("info") - - for line in info.splitlines(): - if not line: - continue - - m = re.match(r"^(\w+)=(.*)$", line) - if m is None: - continue - - key, val = m.groups() - self._info[key] = val.strip("\"") - - self.__dependencies = self.get_dependencies() - self.__provides = self.get_provides() - - def extract(self, path): - logging.debug("Extracting %s to %s" % (self, path)) - - cmd = os.path.join(TOOLSDIR, "decompressor") - cmd += " --root=%s %s" % (path, self.filename) - - util.do(cmd, shell=True, logger=logging.getLogger()) - - @property - def arch(self): - arches = architectures.Architectures() - - arch = self._info.get("PKG_ARCH", None) - if arch: - arch = arches.get(arch) - else: - arch = arches.get_default() - - return arch - - @property - def name(self): - return self._info["PKG_NAME"] - - def get_dependencies(self): - if hasattr(self, "__dependencies"): - return self.__dependencies - - objects = self._info.get("PKG_DEPS", "").split() - - # Compatibility to older package format - objects += self._info.get("PKG_REQUIRES", "").split() - - return [dependencies.Dependency(o, origin=self) for o in objects] - - def get_provides(self): - if hasattr(self, "__provides"): - return self.__provides - - return [dependencies.Provides(p, origin=self) \ - for p in self._info.get("PKG_PROVIDES", "").split()] - - @property - def filelist(self): - if not self._filelist: - - filelist = self._readfile("filelist") - if not filelist: - return [] - - for file in filelist.splitlines(): - if not file or not file.startswith("/"): - continue - - self._filelist.append(file) - - return self._filelist - - def does_provide(self, what): - if what.type == DEP_PACKAGE: - return what.identifier == self.name - - elif what.type == DEP_FILE: - return what.identifier in self.filelist - - elif what.type in (DEP_LIBRARY,): - for provide in self.get_provides(): - if provide.match(what): - return True - - return False - - # For invalid dependencies alsways return false - return False diff --git a/naoki/repositories.py b/naoki/repositories.py deleted file mode 100644 index 3eacc0fe8..000000000 --- a/naoki/repositories.py +++ /dev/null @@ -1,230 +0,0 @@ -#!/usr/bin/python - -import logging -import operator -import os - -import architectures -import packages - -from constants import * -from decorators import * - -class Repository(object): - def __init__(self, arch): - assert isinstance(arch, architectures.Architecture) - self.arch = arch - - logging.debug("Successfully initialized %s" % self) - - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.path) - - @property - def path(self): - raise NotImplementedError - - -class BinaryRepository(Repository): - _cache = {} - - @property - def path(self): - return os.path.join(PACKAGESDIR, self.arch.name) - - def find_package_by_name(self, name): - pkgs = self.find_packages_by_name(name) - - if pkgs: - return pkgs[0] - - def find_packages_by_name(self, name): - pkgs = [] - - for package in self.all: - if package.name == name: - pkgs.append(package) - - return sorted(pkgs, reverse=True) - - @property - def all(self): - if not self._cache.has_key(self.arch.name): - l = [] - - for package in os.listdir(self.path): - # Construct filename - package = os.path.join(self.path, package) - - # Create package instance - package = packages.BinaryPackage(package) - - # Append package to list - l.append(package) - - self._cache[self.arch.name] = l - - return self._cache[self.arch.name] - - def find_package_by_file(self, filename): - pkgs = self.find_packages_by_file(filename) - - if pkgs: - return pkgs[0] - - def find_packages_by_file(self, filename): - pkgs = [] - - for package in self.all: - if filename in package.filelist: - pkgs.append(package) - - return sorted(pkgs, reverse=True) - - def find_package_by_provides(self, provides): - pkgs = self.find_packages_by_provides(provides) - - if pkgs: - return pkgs[0] - - def find_packages_by_provides(self, provides): - pkgs = [] - - for package in self.all: - if package.does_provide(provides): - pkgs.append(package) - - return sorted(pkgs, reverse=True) - - -class SourceRepository(Repository): - def __init__(self, name, arch): - self.name = name - - Repository.__init__(self, arch) - - def __iter__(self): - pkgs = [os.path.join(self.path, p, p + ".nm") for p in self.packages] - - args = { - "arch" : self.arch, - "repo" : self, - } - - return PackageIterator(pkgs, packages.SourcePackage, args) - - def find_package_by_name(self, name, fast=False): - if fast: - filename = os.path.join(self.path, name, name + ".nm") - if os.path.exists(filename): - return packages.SourcePackage(filename, repo=self, arch=self.arch) - - else: - for package in self: - if package.name == name: - return package - - @property - def path(self): - return os.path.join(PKGSDIR, self.name) - - @property - def packages(self): - return sorted(os.listdir(self.path)) - - @property - def completely_built(self): - for p in self: - if not p.is_built: - return False - - return True - - -class PackageIterator(object): - def __init__(self, paks, package_cls=None, cls_args={}): - self.packages = paks - self.package_cls = package_cls - self.package_cls_args = cls_args - - self.__i = 0 - - def __iter__(self): - return self - - def next(self): - if self.__i >= len(self.packages): - raise StopIteration - - pkgs = self.packages[self.__i] - self.__i += 1 - - if self.package_cls: - package = self.package_cls(pkgs, **self.package_cls_args) - - return package - - -class SourceRepositories(object): - def __init__(self, arch): - assert isinstance(arch, architectures.Architecture) - self.arch = arch - - self._repositories = [] - - self.find_all() - - def find_all(self): - for repo in sorted(os.listdir(PKGSDIR)): - # Skip all non-directories - if not os.path.isdir(os.path.join(PKGSDIR, repo)): - continue - - logging.debug("Found source repository: %s" % repo) - - repo = SourceRepository(repo, self.arch) - self._repositories.append(repo) - - @property - def all(self): - return self._repositories[:] - - def find_package_by_name(self, name): - for fast in (True, False): - for repo in self._repositories: - package = repo.find_package_by_name(name, fast=fast) - if package: - return package - - @property - def packages(self): - #pkgs = [] - #for repository in self._repositories: - # pkgs += [p for p in repository] - # - #return pkgs - - for repository in self._repositories: - for p in repository: - yield p - - -if __name__ == "__main__": - arches = architectures.Architectures() - - r = SourceRepositories(arches.get("i686")) - - for repo in r.all: - print "%s" % repo.name - - #for pack in repo: - # print " %s" % pack.name - - #for package in ("gcc", "screen", "iptables", "system-release", "glibc", "nonexistant"): - # print "Searching for %s" % package, - # print r.find_package_by_name(package) - - b = BinaryRepository(arches.get("i686")) - - print b.find_packages_by_name("aiccu") - print b.find_package_by_name("aiccu") diff --git a/naoki/terminal.py b/naoki/terminal.py deleted file mode 100644 index 7d9f1facc..000000000 --- a/naoki/terminal.py +++ /dev/null @@ -1,425 +0,0 @@ -#!/usr/bin/python - -import fcntl -import os -import struct -import sys -import termios - -import architectures -from constants import * - -arches = architectures.Architectures() - -class ParsingError(Exception): - pass - - -class NameSpace(dict): - def __getattr__(self, attr): - try: - return self.__getitem__(attr) - except KeyError: - raise AttributeError - - -class Parser(object): - def __init__(self, name, arguments=[], parsers=[], **kwargs): - self.name = name - self.arguments = arguments - self.parsers = parsers - - self._help = kwargs.get("help", "No help available") - - self.subparser = None - - def __repr__(self): - return "" % self.name - - def check(self, args): - if not args: - return False - - return self.name == args[0] - - def parse(self, args): - if args and self.name == args[0]: - args = args[1:] - - for argument in self.arguments: - args = argument.parse(args) - - if args and self.parsers: - for parser in self.parsers: - if not parser.check(args): - continue - - self.subparser = parser - break - - if self.subparser: - args = self.subparser.parse(args) - - return args - - @property - def values(self): - ret = NameSpace( - name=self.name, - ) - if self.subparser: - ret["action"] = self.subparser.values - - for argument in self.arguments: - ret[argument.name] = argument.value() - - return ret - - @property - def help_line(self): - ret = "" - for argument in self.arguments: - ret += " %s" % argument.help_line - - if self.parsers: - ret += " " - - return ret - - def help(self, indent=0): - ret = self.name - - help_line = self.help_line - if help_line: - ret += self.help_line + "\n" - if self._help: - ret += "\n " + self._help + "\n" - else: - ret += " - " + self._help + "\n" - - if self.arguments: - ret += "\n" - for argument in sorted(self.arguments): - ret += " %15s | %s\n" % (", ".join(argument.args), argument.help) - ret += "\n" - - if self.parsers: - ret += "\n" - for parser in self.parsers: - ret += parser.help(indent=indent + 4) - - indent_string = " " * 4 - return indent_string.join(["%s\n" % line for line in ret.split("\n")]) - - -class _Argument(object): - DEFAULT_HELP = "No help available" - - def __init__(self, name, args, **kwargs): - self.name = name - self.args = sorted(args, reverse=True, key=str.__len__) - self.help = kwargs.get("help", self.DEFAULT_HELP) - - self._parsed = False - self._parsed_args = [] - - def __cmp__(self, other): - return cmp(self.name, other.name) - - def parse(self, args): - raise NotImplementedError - - def value(self): - raise NotImplementedError - - @property - def help_line(self): - raise NotImplementedError - - -class Argument(_Argument): - def __init__(self, name, **kwargs): - _Argument.__init__(self, name, [], **kwargs) - - def parse(self, args): - self._parsed = True - - if len(args) >= 1: - self._parsed_args = args[:1] - - return args[1:] - - def value(self): - if self._parsed_args: - return self._parsed_args[0] - - return [] - - @property - def help_line(self): - return self.name - - -class Option(_Argument): - def parse(self, args): - self._parsed = True - - new_args = [] - for arg in args: - if arg in self.args: - self._parsed_args.append(arg) - else: - new_args.append(arg) - - return new_args - - def value(self): - for arg in self._parsed_args: - if arg in self.args: - return True - - return False - - @property - def help_line(self): - return "[%s]" % ", ".join(self.args) - - -class Choice(_Argument): - def __init__(self, *args, **kwargs): - _Argument.__init__(self, *args, **kwargs) - - self.choices = kwargs.get("choices", []) - self.choices.sort() - - self.help += " [%s]" % ", ".join(self.choices) - - def parse(self, args): - self._parsed = True - - new_args = [] - next_arg = False - for arg in args: - if next_arg: - if self.choices and not arg in self.choices: - raise ParsingError, "'%s' is not an allowed choice" % arg - - self._parsed_args.append(arg) - next_arg = False - continue - - if arg in self.args: - self._parsed_args.append(arg) - next_arg = True - else: - new_args.append(arg) - - return new_args - - def value(self): - if self._parsed_args: - return self._parsed_args[-1] - - return None - - @property - def help_line(self): - return "[%s %s]" % (", ".join(self.args), self.name.upper()) - - -class List(_Argument): - def __init__(self, name, **kwargs): - _Argument.__init__(self, name, [], **kwargs) - - def parse(self, args): - self._parsed = True - self._parsed_args = args - - return [] - - def value(self): - return self._parsed_args - - @property - def help_line(self): - name = self.name[:-1] + " " - return "[" + name * 2 + "...]" - - -class Commandline(object): - def __init__(self, naoki): - self.naoki = naoki - - # Parse the stuff - self.args = self.__parse() - - # ... afterwards, process global directives - self.__process_global(self.args) - - def __process_global(self, args): - # Set debugging mode - self.naoki.debug(args.debug) - - # Set architecture - arches.set_default(args.arch) - - def __parse(self): - parser = Parser(sys.argv[0], - help="Global help", - arguments=[ - Option("help", ["-h", "--help"], help="Show help text"), - Option("quiet", ["-q", "--quiet"], help="Set quiet mode"), - Option("debug", ["-d", "--debug"], help="Set debugging mode"), - Choice("arch", ["-a", "--arch"], help="Set architecture", - choices=[arch.name for arch in arches.all]), - ], - parsers=[ - # Build - Parser("build", - help="Primary build command", - arguments=[ - Option("all", ["--all"], help="Build all packages"), - Option("ignore_dependency_errors", ["-i", "--ignore-dependency-errors"], - help="Ignore dependency errors."), - Option("shell", ["-s", "--shell"], help="Enter a shell after unsuccessfull build."), - Argument("package", help="Name of a package to build."), - ]), - - # Package - Parser("package", - parsers=[ - Parser("dependencies", - help="Show package dependencies.", - arguments=[ - Argument("package", help="Name of the package."), - ]), - Parser("info", - help="Show detailed information about given packages", - arguments=[ - Option("long", ["-l", "--long"], help="Show long list of information"), - Option("machine", ["--machine"], help="Output in machine parseable format"), - List("packages"), - ]), - Parser("list", - help="Show package list", - arguments=[ - Option("long", ["-l", "--long"], help="Show list with lots of information"), - Option("unbuilt", ["-u", "--unbuilt"], help="Do only show unbuilt packages"), - Option("built", ["-b", "--built"], help="Do only show already built packages"), - ]), - Parser("groups", - help="Show package groups", - ), - Parser("raw", - help="Show package information in a machine parseable format.", - arguments=[ - Argument("package", help="Name of the package."), - ]), - Parser("provides", - help="Find packages which provide a specific thing.", - arguments=[ - Argument("provides", help="Pattern to search for."), - ]), - ]), - - # Source - Parser("source", - help="Handle source tarballs", - parsers=[ - Parser("download", - help="Download source tarballs", - arguments=[ - List("packages"), - ]), - Parser("upload", - help="Upload source tarballs", - arguments=[ - List("packages"), - ]), - ]), - - # Check - Parser("check", - help="Check commands", - parsers=[ - Parser("host", help="Check if host fullfills prerequisites"), - ]), - - # Batch - Parser("batch", - help="Batch command - use with caution", - parsers=[ - Parser("cron", help="Command that gets called by cron"), - ]), - - # Shell - Parser("shell", - help="Shell environment", - arguments=[ - Argument("package", help="Package to process."), - ]), - - # Generator - Parser("generate", - help="Generator command", - arguments=[ - Argument("type", help="Type of image to generate"), - ]), - ]) - - self.parser = parser - - args = parser.parse(sys.argv[1:]) - - if args: - raise ParsingError, "Unknown argument(s) passed: %s" % args - - return parser.values - - def help(self): - print >>sys.stderr, self.parser.help(), - - -DEFAULT_COLUMNS = 80 -DEFAULT_LINES = 25 - -def get_size(): - """ - returns the size of the terminal - - tupel: lines, columns - """ - def ioctl_GWINSZ(fd): - try: - cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234")) - except: - return None - - return cr - - cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) - if not cr: - try: - fd = os.open(os.ctermid(), os.O_RDONLY) - cr = ioctl_GWINSZ(fd) - os.close(fd) - except: - pass - - if not cr: - try: - cr = (os.environ['LINES'], os.environ['COLUMNS']) - except: - cr = (DEFAULT_LINES, DEFAULT_COLUMNS) - - return int(cr[1]), int(cr[0]) - -def get_lines(): - return get_size()[0] - -def get_columns(): - return get_size()[1] - - -if __name__ == "__main__": - cl = Commandline(naoki=None) diff --git a/naoki/urlgrabber/__init__.py b/naoki/urlgrabber/__init__.py deleted file mode 100644 index 7bcd9d554..000000000 --- a/naoki/urlgrabber/__init__.py +++ /dev/null @@ -1,53 +0,0 @@ -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -# Copyright 2002-2006 Michael D. Stenner, Ryan Tomayko - -# $Id: __init__.py,v 1.20 2006/09/22 00:58:55 mstenner Exp $ - -"""A high-level cross-protocol url-grabber. - -Using urlgrabber, data can be fetched in three basic ways: - - urlgrab(url) copy the file to the local filesystem - urlopen(url) open the remote file and return a file object - (like urllib2.urlopen) - urlread(url) return the contents of the file as a string - -When using these functions (or methods), urlgrabber supports the -following features: - - * identical behavior for http://, ftp://, and file:// urls - * http keepalive - faster downloads of many files by using - only a single connection - * byte ranges - fetch only a portion of the file - * reget - for a urlgrab, resume a partial download - * progress meters - the ability to report download progress - automatically, even when using urlopen! - * throttling - restrict bandwidth usage - * retries - automatically retry a download if it fails. The - number of retries and failure types are configurable. - * authenticated server access for http and ftp - * proxy support - support for authenticated http and ftp proxies - * mirror groups - treat a list of mirrors as a single source, - automatically switching mirrors if there is a failure. -""" - -__version__ = '3.1.0' -__date__ = '2006/09/21' -__author__ = 'Michael D. Stenner , ' \ - 'Ryan Tomayko ' -__url__ = 'http://linux.duke.edu/projects/urlgrabber/' - -from grabber import urlgrab, urlopen, urlread diff --git a/naoki/urlgrabber/byterange.py b/naoki/urlgrabber/byterange.py deleted file mode 100644 index 001b4e32d..000000000 --- a/naoki/urlgrabber/byterange.py +++ /dev/null @@ -1,463 +0,0 @@ -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, -# Boston, MA 02111-1307 USA - -# This file is part of urlgrabber, a high-level cross-protocol url-grabber -# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko - -# $Id: byterange.py,v 1.12 2006/07/20 20:15:58 mstenner Exp $ - -import os -import stat -import urllib -import urllib2 -import rfc822 - -DEBUG = None - -try: - from cStringIO import StringIO -except ImportError, msg: - from StringIO import StringIO - -class RangeError(IOError): - """Error raised when an unsatisfiable range is requested.""" - pass - -class HTTPRangeHandler(urllib2.BaseHandler): - """Handler that enables HTTP Range headers. - - This was extremely simple. The Range header is a HTTP feature to - begin with so all this class does is tell urllib2 that the - "206 Partial Content" reponse from the HTTP server is what we - expected. - - Example: - import urllib2 - import byterange - - range_handler = range.HTTPRangeHandler() - opener = urllib2.build_opener(range_handler) - - # install it - urllib2.install_opener(opener) - - # create Request and set Range header - req = urllib2.Request('http://www.python.org/') - req.header['Range'] = 'bytes=30-50' - f = urllib2.urlopen(req) - """ - - def http_error_206(self, req, fp, code, msg, hdrs): - # 206 Partial Content Response - r = urllib.addinfourl(fp, hdrs, req.get_full_url()) - r.code = code - r.msg = msg - return r - - def http_error_416(self, req, fp, code, msg, hdrs): - # HTTP's Range Not Satisfiable error - raise RangeError('Requested Range Not Satisfiable') - -class HTTPSRangeHandler(HTTPRangeHandler): - """ Range Header support for HTTPS. """ - - def https_error_206(self, req, fp, code, msg, hdrs): - return self.http_error_206(req, fp, code, msg, hdrs) - - def https_error_416(self, req, fp, code, msg, hdrs): - self.https_error_416(req, fp, code, msg, hdrs) - -class RangeableFileObject: - """File object wrapper to enable raw range handling. - This was implemented primarilary for handling range - specifications for file:// urls. This object effectively makes - a file object look like it consists only of a range of bytes in - the stream. - - Examples: - # expose 10 bytes, starting at byte position 20, from - # /etc/aliases. - >>> fo = RangeableFileObject(file('/etc/passwd', 'r'), (20,30)) - # seek seeks within the range (to position 23 in this case) - >>> fo.seek(3) - # tell tells where your at _within the range_ (position 3 in - # this case) - >>> fo.tell() - # read EOFs if an attempt is made to read past the last - # byte in the range. the following will return only 7 bytes. - >>> fo.read(30) - """ - - def __init__(self, fo, rangetup): - """Create a RangeableFileObject. - fo -- a file like object. only the read() method need be - supported but supporting an optimized seek() is - preferable. - rangetup -- a (firstbyte,lastbyte) tuple specifying the range - to work over. - The file object provided is assumed to be at byte offset 0. - """ - self.fo = fo - (self.firstbyte, self.lastbyte) = range_tuple_normalize(rangetup) - self.realpos = 0 - self._do_seek(self.firstbyte) - - def __getattr__(self, name): - """This effectively allows us to wrap at the instance level. - Any attribute not found in _this_ object will be searched for - in self.fo. This includes methods.""" - if hasattr(self.fo, name): - return getattr(self.fo, name) - raise AttributeError, name - - def tell(self): - """Return the position within the range. - This is different from fo.seek in that position 0 is the - first byte position of the range tuple. For example, if - this object was created with a range tuple of (500,899), - tell() will return 0 when at byte position 500 of the file. - """ - return (self.realpos - self.firstbyte) - - def seek(self,offset,whence=0): - """Seek within the byte range. - Positioning is identical to that described under tell(). - """ - assert whence in (0, 1, 2) - if whence == 0: # absolute seek - realoffset = self.firstbyte + offset - elif whence == 1: # relative seek - realoffset = self.realpos + offset - elif whence == 2: # absolute from end of file - # XXX: are we raising the right Error here? - raise IOError('seek from end of file not supported.') - - # do not allow seek past lastbyte in range - if self.lastbyte and (realoffset >= self.lastbyte): - realoffset = self.lastbyte - - self._do_seek(realoffset - self.realpos) - - def read(self, size=-1): - """Read within the range. - This method will limit the size read based on the range. - """ - size = self._calc_read_size(size) - rslt = self.fo.read(size) - self.realpos += len(rslt) - return rslt - - def readline(self, size=-1): - """Read lines within the range. - This method will limit the size read based on the range. - """ - size = self._calc_read_size(size) - rslt = self.fo.readline(size) - self.realpos += len(rslt) - return rslt - - def _calc_read_size(self, size): - """Handles calculating the amount of data to read based on - the range. - """ - if self.lastbyte: - if size > -1: - if ((self.realpos + size) >= self.lastbyte): - size = (self.lastbyte - self.realpos) - else: - size = (self.lastbyte - self.realpos) - return size - - def _do_seek(self,offset): - """Seek based on whether wrapped object supports seek(). - offset is relative to the current position (self.realpos). - """ - assert offset >= 0 - if not hasattr(self.fo, 'seek'): - self._poor_mans_seek(offset) - else: - self.fo.seek(self.realpos + offset) - self.realpos+= offset - - def _poor_mans_seek(self,offset): - """Seek by calling the wrapped file objects read() method. - This is used for file like objects that do not have native - seek support. The wrapped objects read() method is called - to manually seek to the desired position. - offset -- read this number of bytes from the wrapped - file object. - raise RangeError if we encounter EOF before reaching the - specified offset. - """ - pos = 0 - bufsize = 1024 - while pos < offset: - if (pos + bufsize) > offset: - bufsize = offset - pos - buf = self.fo.read(bufsize) - if len(buf) != bufsize: - raise RangeError('Requested Range Not Satisfiable') - pos+= bufsize - -class FileRangeHandler(urllib2.FileHandler): - """FileHandler subclass that adds Range support. - This class handles Range headers exactly like an HTTP - server would. - """ - def open_local_file(self, req): - import mimetypes - import mimetools - host = req.get_host() - file = req.get_selector() - localfile = urllib.url2pathname(file) - stats = os.stat(localfile) - size = stats[stat.ST_SIZE] - modified = rfc822.formatdate(stats[stat.ST_MTIME]) - mtype = mimetypes.guess_type(file)[0] - if host: - host, port = urllib.splitport(host) - if port or socket.gethostbyname(host) not in self.get_names(): - raise urllib2.URLError('file not on local host') - fo = open(localfile,'rb') - brange = req.headers.get('Range',None) - brange = range_header_to_tuple(brange) - assert brange != () - if brange: - (fb,lb) = brange - if lb == '': lb = size - if fb < 0 or fb > size or lb > size: - raise RangeError('Requested Range Not Satisfiable') - size = (lb - fb) - fo = RangeableFileObject(fo, (fb,lb)) - headers = mimetools.Message(StringIO( - 'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' % - (mtype or 'text/plain', size, modified))) - return urllib.addinfourl(fo, headers, 'file:'+file) - - -# FTP Range Support -# Unfortunately, a large amount of base FTP code had to be copied -# from urllib and urllib2 in order to insert the FTP REST command. -# Code modifications for range support have been commented as -# follows: -# -- range support modifications start/end here - -from urllib import splitport, splituser, splitpasswd, splitattr, \ - unquote, addclosehook, addinfourl -import ftplib -import socket -import sys -import ftplib -import mimetypes -import mimetools - -class FTPRangeHandler(urllib2.FTPHandler): - def ftp_open(self, req): - host = req.get_host() - if not host: - raise IOError, ('ftp error', 'no host given') - host, port = splitport(host) - if port is None: - port = ftplib.FTP_PORT - - # username/password handling - user, host = splituser(host) - if user: - user, passwd = splitpasswd(user) - else: - passwd = None - host = unquote(host) - user = unquote(user or '') - passwd = unquote(passwd or '') - - try: - host = socket.gethostbyname(host) - except socket.error, msg: - raise urllib2.URLError(msg) - path, attrs = splitattr(req.get_selector()) - dirs = path.split('/') - dirs = map(unquote, dirs) - dirs, file = dirs[:-1], dirs[-1] - if dirs and not dirs[0]: - dirs = dirs[1:] - try: - fw = self.connect_ftp(user, passwd, host, port, dirs) - type = file and 'I' or 'D' - for attr in attrs: - attr, value = splitattr(attr) - if attr.lower() == 'type' and \ - value in ('a', 'A', 'i', 'I', 'd', 'D'): - type = value.upper() - - # -- range support modifications start here - rest = None - range_tup = range_header_to_tuple(req.headers.get('Range',None)) - assert range_tup != () - if range_tup: - (fb,lb) = range_tup - if fb > 0: rest = fb - # -- range support modifications end here - - fp, retrlen = fw.retrfile(file, type, rest) - - # -- range support modifications start here - if range_tup: - (fb,lb) = range_tup - if lb == '': - if retrlen is None or retrlen == 0: - raise RangeError('Requested Range Not Satisfiable due to unobtainable file length.') - lb = retrlen - retrlen = lb - fb - if retrlen < 0: - # beginning of range is larger than file - raise RangeError('Requested Range Not Satisfiable') - else: - retrlen = lb - fb - fp = RangeableFileObject(fp, (0,retrlen)) - # -- range support modifications end here - - headers = "" - mtype = mimetypes.guess_type(req.get_full_url())[0] - if mtype: - headers += "Content-Type: %s\n" % mtype - if retrlen is not None and retrlen >= 0: - headers += "Content-Length: %d\n" % retrlen - sf = StringIO(headers) - headers = mimetools.Message(sf) - return addinfourl(fp, headers, req.get_full_url()) - except ftplib.all_errors, msg: - raise IOError, ('ftp error', msg), sys.exc_info()[2] - - def connect_ftp(self, user, passwd, host, port, dirs): - fw = ftpwrapper(user, passwd, host, port, dirs) - return fw - -class ftpwrapper(urllib.ftpwrapper): - # range support note: - # this ftpwrapper code is copied directly from - # urllib. The only enhancement is to add the rest - # argument and pass it on to ftp.ntransfercmd - def retrfile(self, file, type, rest=None): - self.endtransfer() - if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1 - else: cmd = 'TYPE ' + type; isdir = 0 - try: - self.ftp.voidcmd(cmd) - except ftplib.all_errors: - self.init() - self.ftp.voidcmd(cmd) - conn = None - if file and not isdir: - # Use nlst to see if the file exists at all - try: - self.ftp.nlst(file) - except ftplib.error_perm, reason: - raise IOError, ('ftp error', reason), sys.exc_info()[2] - # Restore the transfer mode! - self.ftp.voidcmd(cmd) - # Try to retrieve as a file - try: - cmd = 'RETR ' + file - conn = self.ftp.ntransfercmd(cmd, rest) - except ftplib.error_perm, reason: - if str(reason)[:3] == '501': - # workaround for REST not supported error - fp, retrlen = self.retrfile(file, type) - fp = RangeableFileObject(fp, (rest,'')) - return (fp, retrlen) - elif str(reason)[:3] != '550': - raise IOError, ('ftp error', reason), sys.exc_info()[2] - if not conn: - # Set transfer mode to ASCII! - self.ftp.voidcmd('TYPE A') - # Try a directory listing - if file: cmd = 'LIST ' + file - else: cmd = 'LIST' - conn = self.ftp.ntransfercmd(cmd) - self.busy = 1 - # Pass back both a suitably decorated object and a retrieval length - return (addclosehook(conn[0].makefile('rb'), - self.endtransfer), conn[1]) - - -#################################################################### -# Range Tuple Functions -# XXX: These range tuple functions might go better in a class. - -_rangere = None -def range_header_to_tuple(range_header): - """Get a (firstbyte,lastbyte) tuple from a Range header value. - - Range headers have the form "bytes=-". This - function pulls the firstbyte and lastbyte values and returns - a (firstbyte,lastbyte) tuple. If lastbyte is not specified in - the header value, it is returned as an empty string in the - tuple. - - Return None if range_header is None - Return () if range_header does not conform to the range spec - pattern. - - """ - global _rangere - if range_header is None: return None - if _rangere is None: - import re - _rangere = re.compile(r'^bytes=(\d{1,})-(\d*)') - match = _rangere.match(range_header) - if match: - tup = range_tuple_normalize(match.group(1,2)) - if tup and tup[1]: - tup = (tup[0],tup[1]+1) - return tup - return () - -def range_tuple_to_header(range_tup): - """Convert a range tuple to a Range header value. - Return a string of the form "bytes=-" or None - if no range is needed. - """ - if range_tup is None: return None - range_tup = range_tuple_normalize(range_tup) - if range_tup: - if range_tup[1]: - range_tup = (range_tup[0],range_tup[1] - 1) - return 'bytes=%s-%s' % range_tup - -def range_tuple_normalize(range_tup): - """Normalize a (first_byte,last_byte) range tuple. - Return a tuple whose first element is guaranteed to be an int - and whose second element will be '' (meaning: the last byte) or - an int. Finally, return None if the normalized tuple == (0,'') - as that is equivelant to retrieving the entire file. - """ - if range_tup is None: return None - # handle first byte - fb = range_tup[0] - if fb in (None,''): fb = 0 - else: fb = int(fb) - # handle last byte - try: lb = range_tup[1] - except IndexError: lb = '' - else: - if lb is None: lb = '' - elif lb != '': lb = int(lb) - # check if range is over the entire file - if (fb,lb) == (0,''): return None - # check that the range is valid - if lb < fb: raise RangeError('Invalid byte range: %s-%s' % (fb,lb)) - return (fb,lb) - diff --git a/naoki/urlgrabber/grabber.py b/naoki/urlgrabber/grabber.py deleted file mode 100644 index 20e7899b2..000000000 --- a/naoki/urlgrabber/grabber.py +++ /dev/null @@ -1,1477 +0,0 @@ -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, -# Boston, MA 02111-1307 USA - -# This file is part of urlgrabber, a high-level cross-protocol url-grabber -# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko - -"""A high-level cross-protocol url-grabber. - -GENERAL ARGUMENTS (kwargs) - - Where possible, the module-level default is indicated, and legal - values are provided. - - copy_local = 0 [0|1] - - ignored except for file:// urls, in which case it specifies - whether urlgrab should still make a copy of the file, or simply - point to the existing copy. The module level default for this - option is 0. - - close_connection = 0 [0|1] - - tells URLGrabber to close the connection after a file has been - transfered. This is ignored unless the download happens with the - http keepalive handler (keepalive=1). Otherwise, the connection - is left open for further use. The module level default for this - option is 0 (keepalive connections will not be closed). - - keepalive = 1 [0|1] - - specifies whether keepalive should be used for HTTP/1.1 servers - that support it. The module level default for this option is 1 - (keepalive is enabled). - - progress_obj = None - - a class instance that supports the following methods: - po.start(filename, url, basename, length, text) - # length will be None if unknown - po.update(read) # read == bytes read so far - po.end() - - text = None - - specifies an alternativ text item in the beginning of the progress - bar line. If not given, the basename of the file is used. - - throttle = 1.0 - - a number - if it's an int, it's the bytes/second throttle limit. - If it's a float, it is first multiplied by bandwidth. If throttle - == 0, throttling is disabled. If None, the module-level default - (which can be set on default_grabber.throttle) is used. See - BANDWIDTH THROTTLING for more information. - - timeout = None - - a positive float expressing the number of seconds to wait for socket - operations. If the value is None or 0.0, socket operations will block - forever. Setting this option causes urlgrabber to call the settimeout - method on the Socket object used for the request. See the Python - documentation on settimeout for more information. - http://www.python.org/doc/current/lib/socket-objects.html - - bandwidth = 0 - - the nominal max bandwidth in bytes/second. If throttle is a float - and bandwidth == 0, throttling is disabled. If None, the - module-level default (which can be set on - default_grabber.bandwidth) is used. See BANDWIDTH THROTTLING for - more information. - - range = None - - a tuple of the form (first_byte, last_byte) describing a byte - range to retrieve. Either or both of the values may set to - None. If first_byte is None, byte offset 0 is assumed. If - last_byte is None, the last byte available is assumed. Note that - the range specification is python-like in that (0,10) will yeild - the first 10 bytes of the file. - - If set to None, no range will be used. - - reget = None [None|'simple'|'check_timestamp'] - - whether to attempt to reget a partially-downloaded file. Reget - only applies to .urlgrab and (obviously) only if there is a - partially downloaded file. Reget has two modes: - - 'simple' -- the local file will always be trusted. If there - are 100 bytes in the local file, then the download will always - begin 100 bytes into the requested file. - - 'check_timestamp' -- the timestamp of the server file will be - compared to the timestamp of the local file. ONLY if the - local file is newer than or the same age as the server file - will reget be used. If the server file is newer, or the - timestamp is not returned, the entire file will be fetched. - - NOTE: urlgrabber can do very little to verify that the partial - file on disk is identical to the beginning of the remote file. - You may want to either employ a custom "checkfunc" or simply avoid - using reget in situations where corruption is a concern. - - user_agent = 'urlgrabber/VERSION' - - a string, usually of the form 'AGENT/VERSION' that is provided to - HTTP servers in the User-agent header. The module level default - for this option is "urlgrabber/VERSION". - - http_headers = None - - a tuple of 2-tuples, each containing a header and value. These - will be used for http and https requests only. For example, you - can do - http_headers = (('Pragma', 'no-cache'),) - - ftp_headers = None - - this is just like http_headers, but will be used for ftp requests. - - proxies = None - - a dictionary that maps protocol schemes to proxy hosts. For - example, to use a proxy server on host "foo" port 3128 for http - and https URLs: - proxies={ 'http' : 'http://foo:3128', 'https' : 'http://foo:3128' } - note that proxy authentication information may be provided using - normal URL constructs: - proxies={ 'http' : 'http://user:host@foo:3128' } - Lastly, if proxies is None, the default environment settings will - be used. - - prefix = None - - a url prefix that will be prepended to all requested urls. For - example: - g = URLGrabber(prefix='http://foo.com/mirror/') - g.urlgrab('some/file.txt') - ## this will fetch 'http://foo.com/mirror/some/file.txt' - This option exists primarily to allow identical behavior to - MirrorGroup (and derived) instances. Note: a '/' will be inserted - if necessary, so you cannot specify a prefix that ends with a - partial file or directory name. - - opener = None - - Overrides the default urllib2.OpenerDirector provided to urllib2 - when making requests. This option exists so that the urllib2 - handler chain may be customized. Note that the range, reget, - proxy, and keepalive features require that custom handlers be - provided to urllib2 in order to function properly. If an opener - option is provided, no attempt is made by urlgrabber to ensure - chain integrity. You are responsible for ensuring that any - extension handlers are present if said features are required. - - data = None - - Only relevant for the HTTP family (and ignored for other - protocols), this allows HTTP POSTs. When the data kwarg is - present (and not None), an HTTP request will automatically become - a POST rather than GET. This is done by direct passthrough to - urllib2. If you use this, you may also want to set the - 'Content-length' and 'Content-type' headers with the http_headers - option. Note that python 2.2 handles the case of these - badly and if you do not use the proper case (shown here), your - values will be overridden with the defaults. - - -RETRY RELATED ARGUMENTS - - retry = None - - the number of times to retry the grab before bailing. If this is - zero, it will retry forever. This was intentional... really, it - was :). If this value is not supplied or is supplied but is None - retrying does not occur. - - retrycodes = [-1,2,4,5,6,7] - - a sequence of errorcodes (values of e.errno) for which it should - retry. See the doc on URLGrabError for more details on this. You - might consider modifying a copy of the default codes rather than - building yours from scratch so that if the list is extended in the - future (or one code is split into two) you can still enjoy the - benefits of the default list. You can do that with something like - this: - - retrycodes = urlgrabber.grabber.URLGrabberOptions().retrycodes - if 12 not in retrycodes: - retrycodes.append(12) - - checkfunc = None - - a function to do additional checks. This defaults to None, which - means no additional checking. The function should simply return - on a successful check. It should raise URLGrabError on an - unsuccessful check. Raising of any other exception will be - considered immediate failure and no retries will occur. - - If it raises URLGrabError, the error code will determine the retry - behavior. Negative error numbers are reserved for use by these - passed in functions, so you can use many negative numbers for - different types of failure. By default, -1 results in a retry, - but this can be customized with retrycodes. - - If you simply pass in a function, it will be given exactly one - argument: a CallbackObject instance with the .url attribute - defined and either .filename (for urlgrab) or .data (for urlread). - For urlgrab, .filename is the name of the local file. For - urlread, .data is the actual string data. If you need other - arguments passed to the callback (program state of some sort), you - can do so like this: - - checkfunc=(function, ('arg1', 2), {'kwarg': 3}) - - if the downloaded file has filename /tmp/stuff, then this will - result in this call (for urlgrab): - - function(obj, 'arg1', 2, kwarg=3) - # obj.filename = '/tmp/stuff' - # obj.url = 'http://foo.com/stuff' - - NOTE: both the "args" tuple and "kwargs" dict must be present if - you use this syntax, but either (or both) can be empty. - - failure_callback = None - - The callback that gets called during retries when an attempt to - fetch a file fails. The syntax for specifying the callback is - identical to checkfunc, except for the attributes defined in the - CallbackObject instance. The attributes for failure_callback are: - - exception = the raised exception - url = the url we're trying to fetch - tries = the number of tries so far (including this one) - retry = the value of the retry option - - The callback is present primarily to inform the calling program of - the failure, but if it raises an exception (including the one it's - passed) that exception will NOT be caught and will therefore cause - future retries to be aborted. - - The callback is called for EVERY failure, including the last one. - On the last try, the callback can raise an alternate exception, - but it cannot (without severe trickiness) prevent the exception - from being raised. - - interrupt_callback = None - - This callback is called if KeyboardInterrupt is received at any - point in the transfer. Basically, this callback can have three - impacts on the fetch process based on the way it exits: - - 1) raise no exception: the current fetch will be aborted, but - any further retries will still take place - - 2) raise a URLGrabError: if you're using a MirrorGroup, then - this will prompt a failover to the next mirror according to - the behavior of the MirrorGroup subclass. It is recommended - that you raise URLGrabError with code 15, 'user abort'. If - you are NOT using a MirrorGroup subclass, then this is the - same as (3). - - 3) raise some other exception (such as KeyboardInterrupt), which - will not be caught at either the grabber or mirror levels. - That is, it will be raised up all the way to the caller. - - This callback is very similar to failure_callback. They are - passed the same arguments, so you could use the same function for - both. - - urlparser = URLParser() - - The URLParser class handles pre-processing of URLs, including - auth-handling for user/pass encoded in http urls, file handing - (that is, filenames not sent as a URL), and URL quoting. If you - want to override any of this behavior, you can pass in a - replacement instance. See also the 'quote' option. - - quote = None - - Whether or not to quote the path portion of a url. - quote = 1 -> quote the URLs (they're not quoted yet) - quote = 0 -> do not quote them (they're already quoted) - quote = None -> guess what to do - - This option only affects proper urls like 'file:///etc/passwd'; it - does not affect 'raw' filenames like '/etc/passwd'. The latter - will always be quoted as they are converted to URLs. Also, only - the path part of a url is quoted. If you need more fine-grained - control, you should probably subclass URLParser and pass it in via - the 'urlparser' option. - -BANDWIDTH THROTTLING - - urlgrabber supports throttling via two values: throttle and - bandwidth Between the two, you can either specify and absolute - throttle threshold or specify a theshold as a fraction of maximum - available bandwidth. - - throttle is a number - if it's an int, it's the bytes/second - throttle limit. If it's a float, it is first multiplied by - bandwidth. If throttle == 0, throttling is disabled. If None, the - module-level default (which can be set with set_throttle) is used. - - bandwidth is the nominal max bandwidth in bytes/second. If throttle - is a float and bandwidth == 0, throttling is disabled. If None, the - module-level default (which can be set with set_bandwidth) is used. - - THROTTLING EXAMPLES: - - Lets say you have a 100 Mbps connection. This is (about) 10^8 bits - per second, or 12,500,000 Bytes per second. You have a number of - throttling options: - - *) set_bandwidth(12500000); set_throttle(0.5) # throttle is a float - - This will limit urlgrab to use half of your available bandwidth. - - *) set_throttle(6250000) # throttle is an int - - This will also limit urlgrab to use half of your available - bandwidth, regardless of what bandwidth is set to. - - *) set_throttle(6250000); set_throttle(1.0) # float - - Use half your bandwidth - - *) set_throttle(6250000); set_throttle(2.0) # float - - Use up to 12,500,000 Bytes per second (your nominal max bandwidth) - - *) set_throttle(6250000); set_throttle(0) # throttle = 0 - - Disable throttling - this is more efficient than a very large - throttle setting. - - *) set_throttle(0); set_throttle(1.0) # throttle is float, bandwidth = 0 - - Disable throttling - this is the default when the module is loaded. - - SUGGESTED AUTHOR IMPLEMENTATION (THROTTLING) - - While this is flexible, it's not extremely obvious to the user. I - suggest you implement a float throttle as a percent to make the - distinction between absolute and relative throttling very explicit. - - Also, you may want to convert the units to something more convenient - than bytes/second, such as kbps or kB/s, etc. - -""" - -# $Id: grabber.py,v 1.48 2006/09/22 00:58:05 mstenner Exp $ - -import os -import os.path -import sys -import urlparse -import rfc822 -import time -import string -import urllib -import urllib2 -from stat import * # S_* and ST_* - -######################################################################## -# MODULE INITIALIZATION -######################################################################## -try: - exec('from ' + (__name__.split('.'))[0] + ' import __version__') -except: - __version__ = '???' - -import sslfactory - -auth_handler = urllib2.HTTPBasicAuthHandler( \ - urllib2.HTTPPasswordMgrWithDefaultRealm()) - -try: - from i18n import _ -except ImportError, msg: - def _(st): return st - -try: - from httplib import HTTPException -except ImportError, msg: - HTTPException = None - -try: - # This is a convenient way to make keepalive optional. - # Just rename the module so it can't be imported. - import keepalive - from keepalive import HTTPHandler, HTTPSHandler - have_keepalive = True -except ImportError, msg: - have_keepalive = False - -try: - # add in range support conditionally too - import byterange - from byterange import HTTPRangeHandler, HTTPSRangeHandler, \ - FileRangeHandler, FTPRangeHandler, range_tuple_normalize, \ - range_tuple_to_header, RangeError -except ImportError, msg: - range_handlers = () - RangeError = None - have_range = 0 -else: - range_handlers = (HTTPRangeHandler(), HTTPSRangeHandler(), - FileRangeHandler(), FTPRangeHandler()) - have_range = 1 - - -# check whether socket timeout support is available (Python >= 2.3) -import socket -try: - TimeoutError = socket.timeout - have_socket_timeout = True -except AttributeError: - TimeoutError = None - have_socket_timeout = False - -######################################################################## -# functions for debugging output. These functions are here because they -# are also part of the module initialization. -DEBUG = None -def set_logger(DBOBJ): - """Set the DEBUG object. This is called by _init_default_logger when - the environment variable URLGRABBER_DEBUG is set, but can also be - called by a calling program. Basically, if the calling program uses - the logging module and would like to incorporate urlgrabber logging, - then it can do so this way. It's probably not necessary as most - internal logging is only for debugging purposes. - - The passed-in object should be a logging.Logger instance. It will - be pushed into the keepalive and byterange modules if they're - being used. The mirror module pulls this object in on import, so - you will need to manually push into it. In fact, you may find it - tidier to simply push your logging object (or objects) into each - of these modules independently. - """ - - global DEBUG - DEBUG = DBOBJ - if have_keepalive and keepalive.DEBUG is None: - keepalive.DEBUG = DBOBJ - if have_range and byterange.DEBUG is None: - byterange.DEBUG = DBOBJ - if sslfactory.DEBUG is None: - sslfactory.DEBUG = DBOBJ - -def _init_default_logger(): - '''Examines the environment variable URLGRABBER_DEBUG and creates - a logging object (logging.logger) based on the contents. It takes - the form - - URLGRABBER_DEBUG=level,filename - - where "level" can be either an integer or a log level from the - logging module (DEBUG, INFO, etc). If the integer is zero or - less, logging will be disabled. Filename is the filename where - logs will be sent. If it is "-", then stdout will be used. If - the filename is empty or missing, stderr will be used. If the - variable cannot be processed or the logging module cannot be - imported (python < 2.3) then logging will be disabled. Here are - some examples: - - URLGRABBER_DEBUG=1,debug.txt # log everything to debug.txt - URLGRABBER_DEBUG=WARNING,- # log warning and higher to stdout - URLGRABBER_DEBUG=INFO # log info and higher to stderr - - This funtion is called during module initialization. It is not - intended to be called from outside. The only reason it is a - function at all is to keep the module-level namespace tidy and to - collect the code into a nice block.''' - - try: - dbinfo = os.environ['URLGRABBER_DEBUG'].split(',') - import logging - level = logging._levelNames.get(dbinfo[0], int(dbinfo[0])) - if level < 1: raise ValueError() - - formatter = logging.Formatter('%(asctime)s %(message)s') - if len(dbinfo) > 1: filename = dbinfo[1] - else: filename = '' - if filename == '': handler = logging.StreamHandler(sys.stderr) - elif filename == '-': handler = logging.StreamHandler(sys.stdout) - else: handler = logging.FileHandler(filename) - handler.setFormatter(formatter) - DBOBJ = logging.getLogger('urlgrabber') - DBOBJ.addHandler(handler) - DBOBJ.setLevel(level) - except (KeyError, ImportError, ValueError): - DBOBJ = None - set_logger(DBOBJ) - -_init_default_logger() -######################################################################## -# END MODULE INITIALIZATION -######################################################################## - - - -class URLGrabError(IOError): - """ - URLGrabError error codes: - - URLGrabber error codes (0 -- 255) - 0 - everything looks good (you should never see this) - 1 - malformed url - 2 - local file doesn't exist - 3 - request for non-file local file (dir, etc) - 4 - IOError on fetch - 5 - OSError on fetch - 6 - no content length header when we expected one - 7 - HTTPException - 8 - Exceeded read limit (for urlread) - 9 - Requested byte range not satisfiable. - 10 - Byte range requested, but range support unavailable - 11 - Illegal reget mode - 12 - Socket timeout - 13 - malformed proxy url - 14 - HTTPError (includes .code and .exception attributes) - 15 - user abort - - MirrorGroup error codes (256 -- 511) - 256 - No more mirrors left to try - - Custom (non-builtin) classes derived from MirrorGroup (512 -- 767) - [ this range reserved for application-specific error codes ] - - Retry codes (< 0) - -1 - retry the download, unknown reason - - Note: to test which group a code is in, you can simply do integer - division by 256: e.errno / 256 - - Negative codes are reserved for use by functions passed in to - retrygrab with checkfunc. The value -1 is built in as a generic - retry code and is already included in the retrycodes list. - Therefore, you can create a custom check function that simply - returns -1 and the fetch will be re-tried. For more customized - retries, you can use other negative number and include them in - retry-codes. This is nice for outputting useful messages about - what failed. - - You can use these error codes like so: - try: urlgrab(url) - except URLGrabError, e: - if e.errno == 3: ... - # or - print e.strerror - # or simply - print e #### print '[Errno %i] %s' % (e.errno, e.strerror) - """ - pass - -class CallbackObject: - """Container for returned callback data. - - This is currently a dummy class into which urlgrabber can stuff - information for passing to callbacks. This way, the prototype for - all callbacks is the same, regardless of the data that will be - passed back. Any function that accepts a callback function as an - argument SHOULD document what it will define in this object. - - It is possible that this class will have some greater - functionality in the future. - """ - def __init__(self, **kwargs): - self.__dict__.update(kwargs) - -def urlgrab(url, filename=None, **kwargs): - """grab the file at and make a local copy at - If filename is none, the basename of the url is used. - urlgrab returns the filename of the local file, which may be different - from the passed-in filename if the copy_local kwarg == 0. - - See module documentation for a description of possible kwargs. - """ - return default_grabber.urlgrab(url, filename, **kwargs) - -def urlopen(url, **kwargs): - """open the url and return a file object - If a progress object or throttle specifications exist, then - a special file object will be returned that supports them. - The file object can be treated like any other file object. - - See module documentation for a description of possible kwargs. - """ - return default_grabber.urlopen(url, **kwargs) - -def urlread(url, limit=None, **kwargs): - """read the url into a string, up to 'limit' bytes - If the limit is exceeded, an exception will be thrown. Note that urlread - is NOT intended to be used as a way of saying "I want the first N bytes" - but rather 'read the whole file into memory, but don't use too much' - - See module documentation for a description of possible kwargs. - """ - return default_grabber.urlread(url, limit, **kwargs) - - -class URLParser: - """Process the URLs before passing them to urllib2. - - This class does several things: - - * add any prefix - * translate a "raw" file to a proper file: url - * handle any http or https auth that's encoded within the url - * quote the url - - Only the "parse" method is called directly, and it calls sub-methods. - - An instance of this class is held in the options object, which - means that it's easy to change the behavior by sub-classing and - passing the replacement in. It need only have a method like: - - url, parts = urlparser.parse(url, opts) - """ - - def parse(self, url, opts): - """parse the url and return the (modified) url and its parts - - Note: a raw file WILL be quoted when it's converted to a URL. - However, other urls (ones which come with a proper scheme) may - or may not be quoted according to opts.quote - - opts.quote = 1 --> quote it - opts.quote = 0 --> do not quote it - opts.quote = None --> guess - """ - quote = opts.quote - - if opts.prefix: - url = self.add_prefix(url, opts.prefix) - - parts = urlparse.urlparse(url) - (scheme, host, path, parm, query, frag) = parts - - if not scheme or (len(scheme) == 1 and scheme in string.letters): - # if a scheme isn't specified, we guess that it's "file:" - if url[0] not in '/\\': url = os.path.abspath(url) - url = 'file:' + urllib.pathname2url(url) - parts = urlparse.urlparse(url) - quote = 0 # pathname2url quotes, so we won't do it again - - if scheme in ['http', 'https']: - parts = self.process_http(parts) - - if quote is None: - quote = self.guess_should_quote(parts) - if quote: - parts = self.quote(parts) - - url = urlparse.urlunparse(parts) - return url, parts - - def add_prefix(self, url, prefix): - if prefix[-1] == '/' or url[0] == '/': - url = prefix + url - else: - url = prefix + '/' + url - return url - - def process_http(self, parts): - (scheme, host, path, parm, query, frag) = parts - - if '@' in host and auth_handler: - try: - user_pass, host = host.split('@', 1) - if ':' in user_pass: - user, password = user_pass.split(':', 1) - except ValueError, e: - raise URLGrabError(1, _('Bad URL: %s') % url) - if DEBUG: DEBUG.info('adding HTTP auth: %s, %s', user, password) - auth_handler.add_password(None, host, user, password) - - return (scheme, host, path, parm, query, frag) - - def quote(self, parts): - """quote the URL - - This method quotes ONLY the path part. If you need to quote - other parts, you should override this and pass in your derived - class. The other alternative is to quote other parts before - passing into urlgrabber. - """ - (scheme, host, path, parm, query, frag) = parts - path = urllib.quote(path) - return (scheme, host, path, parm, query, frag) - - hexvals = '0123456789ABCDEF' - def guess_should_quote(self, parts): - """ - Guess whether we should quote a path. This amounts to - guessing whether it's already quoted. - - find ' ' -> 1 - find '%' -> 1 - find '%XX' -> 0 - else -> 1 - """ - (scheme, host, path, parm, query, frag) = parts - if ' ' in path: - return 1 - ind = string.find(path, '%') - if ind > -1: - while ind > -1: - if len(path) < ind+3: - return 1 - code = path[ind+1:ind+3].upper() - if code[0] not in self.hexvals or \ - code[1] not in self.hexvals: - return 1 - ind = string.find(path, '%', ind+1) - return 0 - return 1 - -class URLGrabberOptions: - """Class to ease kwargs handling.""" - - def __init__(self, delegate=None, **kwargs): - """Initialize URLGrabberOptions object. - Set default values for all options and then update options specified - in kwargs. - """ - self.delegate = delegate - if delegate is None: - self._set_defaults() - self._set_attributes(**kwargs) - - def __getattr__(self, name): - if self.delegate and hasattr(self.delegate, name): - return getattr(self.delegate, name) - raise AttributeError, name - - def raw_throttle(self): - """Calculate raw throttle value from throttle and bandwidth - values. - """ - if self.throttle <= 0: - return 0 - elif type(self.throttle) == type(0): - return float(self.throttle) - else: # throttle is a float - return self.bandwidth * self.throttle - - def derive(self, **kwargs): - """Create a derived URLGrabberOptions instance. - This method creates a new instance and overrides the - options specified in kwargs. - """ - return URLGrabberOptions(delegate=self, **kwargs) - - def _set_attributes(self, **kwargs): - """Update object attributes with those provided in kwargs.""" - self.__dict__.update(kwargs) - if have_range and kwargs.has_key('range'): - # normalize the supplied range value - self.range = range_tuple_normalize(self.range) - if not self.reget in [None, 'simple', 'check_timestamp']: - raise URLGrabError(11, _('Illegal reget mode: %s') \ - % (self.reget, )) - - def _set_defaults(self): - """Set all options to their default values. - When adding new options, make sure a default is - provided here. - """ - self.progress_obj = None - self.throttle = 1.0 - self.bandwidth = 0 - self.retry = None - self.retrycodes = [-1,2,4,5,6,7] - self.checkfunc = None - self.copy_local = 0 - self.close_connection = 0 - self.range = None - self.user_agent = 'urlgrabber/%s' % __version__ - self.keepalive = 1 - self.proxies = None - self.reget = None - self.failure_callback = None - self.interrupt_callback = None - self.prefix = None - self.opener = None - self.cache_openers = True - self.timeout = None - self.text = None - self.http_headers = None - self.ftp_headers = None - self.data = None - self.urlparser = URLParser() - self.quote = None - self.ssl_ca_cert = None - self.ssl_context = None - -class URLGrabber: - """Provides easy opening of URLs with a variety of options. - - All options are specified as kwargs. Options may be specified when - the class is created and may be overridden on a per request basis. - - New objects inherit default values from default_grabber. - """ - - def __init__(self, **kwargs): - self.opts = URLGrabberOptions(**kwargs) - - def _retry(self, opts, func, *args): - tries = 0 - while 1: - # there are only two ways out of this loop. The second has - # several "sub-ways" - # 1) via the return in the "try" block - # 2) by some exception being raised - # a) an excepton is raised that we don't "except" - # b) a callback raises ANY exception - # c) we're not retry-ing or have run out of retries - # d) the URLGrabError code is not in retrycodes - # beware of infinite loops :) - tries = tries + 1 - exception = None - retrycode = None - callback = None - if DEBUG: DEBUG.info('attempt %i/%s: %s', - tries, opts.retry, args[0]) - try: - r = apply(func, (opts,) + args, {}) - if DEBUG: DEBUG.info('success') - return r - except URLGrabError, e: - exception = e - callback = opts.failure_callback - retrycode = e.errno - except KeyboardInterrupt, e: - exception = e - callback = opts.interrupt_callback - - if DEBUG: DEBUG.info('exception: %s', exception) - if callback: - if DEBUG: DEBUG.info('calling callback: %s', callback) - cb_func, cb_args, cb_kwargs = self._make_callback(callback) - obj = CallbackObject(exception=exception, url=args[0], - tries=tries, retry=opts.retry) - cb_func(obj, *cb_args, **cb_kwargs) - - if (opts.retry is None) or (tries == opts.retry): - if DEBUG: DEBUG.info('retries exceeded, re-raising') - raise - - if (retrycode is not None) and (retrycode not in opts.retrycodes): - if DEBUG: DEBUG.info('retrycode (%i) not in list %s, re-raising', - retrycode, opts.retrycodes) - raise - - def urlopen(self, url, **kwargs): - """open the url and return a file object - If a progress object or throttle value specified when this - object was created, then a special file object will be - returned that supports them. The file object can be treated - like any other file object. - """ - opts = self.opts.derive(**kwargs) - (url,parts) = opts.urlparser.parse(url, opts) - def retryfunc(opts, url): - return URLGrabberFileObject(url, filename=None, opts=opts) - return self._retry(opts, retryfunc, url) - - def urlgrab(self, url, filename=None, **kwargs): - """grab the file at and make a local copy at - If filename is none, the basename of the url is used. - urlgrab returns the filename of the local file, which may be - different from the passed-in filename if copy_local == 0. - """ - opts = self.opts.derive(**kwargs) - (url,parts) = opts.urlparser.parse(url, opts) - (scheme, host, path, parm, query, frag) = parts - if filename is None: - filename = os.path.basename( urllib.unquote(path) ) - if scheme == 'file' and not opts.copy_local: - # just return the name of the local file - don't make a - # copy currently - path = urllib.url2pathname(path) - if host: - path = os.path.normpath('//' + host + path) - if not os.path.exists(path): - raise URLGrabError(2, - _('Local file does not exist: %s') % (path, )) - elif not os.path.isfile(path): - raise URLGrabError(3, - _('Not a normal file: %s') % (path, )) - elif not opts.range: - return path - - def retryfunc(opts, url, filename): - fo = URLGrabberFileObject(url, filename, opts) - try: - fo._do_grab() - if not opts.checkfunc is None: - cb_func, cb_args, cb_kwargs = \ - self._make_callback(opts.checkfunc) - obj = CallbackObject() - obj.filename = filename - obj.url = url - apply(cb_func, (obj, )+cb_args, cb_kwargs) - finally: - fo.close() - return filename - - return self._retry(opts, retryfunc, url, filename) - - def urlread(self, url, limit=None, **kwargs): - """read the url into a string, up to 'limit' bytes - If the limit is exceeded, an exception will be thrown. Note - that urlread is NOT intended to be used as a way of saying - "I want the first N bytes" but rather 'read the whole file - into memory, but don't use too much' - """ - opts = self.opts.derive(**kwargs) - (url,parts) = opts.urlparser.parse(url, opts) - if limit is not None: - limit = limit + 1 - - def retryfunc(opts, url, limit): - fo = URLGrabberFileObject(url, filename=None, opts=opts) - s = '' - try: - # this is an unfortunate thing. Some file-like objects - # have a default "limit" of None, while the built-in (real) - # file objects have -1. They each break the other, so for - # now, we just force the default if necessary. - if limit is None: s = fo.read() - else: s = fo.read(limit) - - if not opts.checkfunc is None: - cb_func, cb_args, cb_kwargs = \ - self._make_callback(opts.checkfunc) - obj = CallbackObject() - obj.data = s - obj.url = url - apply(cb_func, (obj, )+cb_args, cb_kwargs) - finally: - fo.close() - return s - - s = self._retry(opts, retryfunc, url, limit) - if limit and len(s) > limit: - raise URLGrabError(8, - _('Exceeded limit (%i): %s') % (limit, url)) - return s - - def _make_callback(self, callback_obj): - if callable(callback_obj): - return callback_obj, (), {} - else: - return callback_obj - -# create the default URLGrabber used by urlXXX functions. -# NOTE: actual defaults are set in URLGrabberOptions -default_grabber = URLGrabber() - -class URLGrabberFileObject: - """This is a file-object wrapper that supports progress objects - and throttling. - - This exists to solve the following problem: lets say you want to - drop-in replace a normal open with urlopen. You want to use a - progress meter and/or throttling, but how do you do that without - rewriting your code? Answer: urlopen will return a wrapped file - object that does the progress meter and-or throttling internally. - """ - - def __init__(self, url, filename, opts): - self.url = url - self.filename = filename - self.opts = opts - self.fo = None - self._rbuf = '' - self._rbufsize = 1024*8 - self._ttime = time.time() - self._tsize = 0 - self._amount_read = 0 - self._opener = None - self._do_open() - - def __getattr__(self, name): - """This effectively allows us to wrap at the instance level. - Any attribute not found in _this_ object will be searched for - in self.fo. This includes methods.""" - if hasattr(self.fo, name): - return getattr(self.fo, name) - raise AttributeError, name - - def _get_opener(self): - """Build a urllib2 OpenerDirector based on request options.""" - if self.opts.opener: - return self.opts.opener - elif self._opener is None: - handlers = [] - need_keepalive_handler = (have_keepalive and self.opts.keepalive) - need_range_handler = (range_handlers and \ - (self.opts.range or self.opts.reget)) - # if you specify a ProxyHandler when creating the opener - # it _must_ come before all other handlers in the list or urllib2 - # chokes. - if self.opts.proxies: - handlers.append( CachedProxyHandler(self.opts.proxies) ) - - # ------------------------------------------------------- - # OK, these next few lines are a serious kludge to get - # around what I think is a bug in python 2.2's - # urllib2. The basic idea is that default handlers - # get applied first. If you override one (like a - # proxy handler), then the default gets pulled, but - # the replacement goes on the end. In the case of - # proxies, this means the normal handler picks it up - # first and the proxy isn't used. Now, this probably - # only happened with ftp or non-keepalive http, so not - # many folks saw it. The simple approach to fixing it - # is just to make sure you override the other - # conflicting defaults as well. I would LOVE to see - # these go way or be dealt with more elegantly. The - # problem isn't there after 2.2. -MDS 2005/02/24 - if not need_keepalive_handler: - handlers.append( urllib2.HTTPHandler() ) - if not need_range_handler: - handlers.append( urllib2.FTPHandler() ) - # ------------------------------------------------------- - - ssl_factory = sslfactory.get_factory(self.opts.ssl_ca_cert, - self.opts.ssl_context) - - if need_keepalive_handler: - handlers.append(HTTPHandler()) - handlers.append(HTTPSHandler(ssl_factory)) - if need_range_handler: - handlers.extend( range_handlers ) - handlers.append( auth_handler ) - if self.opts.cache_openers: - self._opener = CachedOpenerDirector(ssl_factory, *handlers) - else: - self._opener = ssl_factory.create_opener(*handlers) - # OK, I don't like to do this, but otherwise, we end up with - # TWO user-agent headers. - self._opener.addheaders = [] - return self._opener - - def _do_open(self): - opener = self._get_opener() - - req = urllib2.Request(self.url, self.opts.data) # build request object - self._add_headers(req) # add misc headers that we need - self._build_range(req) # take care of reget and byterange stuff - - fo, hdr = self._make_request(req, opener) - if self.reget_time and self.opts.reget == 'check_timestamp': - # do this if we have a local file with known timestamp AND - # we're in check_timestamp reget mode. - fetch_again = 0 - try: - modified_tuple = hdr.getdate_tz('last-modified') - modified_stamp = rfc822.mktime_tz(modified_tuple) - if modified_stamp > self.reget_time: fetch_again = 1 - except (TypeError,): - fetch_again = 1 - - if fetch_again: - # the server version is newer than the (incomplete) local - # version, so we should abandon the version we're getting - # and fetch the whole thing again. - fo.close() - self.opts.reget = None - del req.headers['Range'] - self._build_range(req) - fo, hdr = self._make_request(req, opener) - - (scheme, host, path, parm, query, frag) = urlparse.urlparse(self.url) - path = urllib.unquote(path) - if not (self.opts.progress_obj or self.opts.raw_throttle() \ - or self.opts.timeout): - # if we're not using the progress_obj, throttling, or timeout - # we can get a performance boost by going directly to - # the underlying fileobject for reads. - self.read = fo.read - if hasattr(fo, 'readline'): - self.readline = fo.readline - elif self.opts.progress_obj: - try: - length = int(hdr['Content-Length']) - length = length + self._amount_read # Account for regets - except (KeyError, ValueError, TypeError): - length = None - - self.opts.progress_obj.start(str(self.filename), - urllib.unquote(self.url), - os.path.basename(path), - length, text=self.opts.text) - self.opts.progress_obj.update(0) - (self.fo, self.hdr) = (fo, hdr) - - def _add_headers(self, req): - if self.opts.user_agent: - req.add_header('User-agent', self.opts.user_agent) - try: req_type = req.get_type() - except ValueError: req_type = None - if self.opts.http_headers and req_type in ('http', 'https'): - for h, v in self.opts.http_headers: - req.add_header(h, v) - if self.opts.ftp_headers and req_type == 'ftp': - for h, v in self.opts.ftp_headers: - req.add_header(h, v) - - def _build_range(self, req): - self.reget_time = None - self.append = 0 - reget_length = 0 - rt = None - if have_range and self.opts.reget and type(self.filename) == type(''): - # we have reget turned on and we're dumping to a file - try: - s = os.stat(self.filename) - except OSError: - pass - else: - self.reget_time = s[ST_MTIME] - reget_length = s[ST_SIZE] - - # Set initial length when regetting - self._amount_read = reget_length - - rt = reget_length, '' - self.append = 1 - - if self.opts.range: - if not have_range: - raise URLGrabError(10, _('Byte range requested but range '\ - 'support unavailable')) - rt = self.opts.range - if rt[0]: rt = (rt[0] + reget_length, rt[1]) - - if rt: - header = range_tuple_to_header(rt) - if header: req.add_header('Range', header) - - def _make_request(self, req, opener): - try: - if have_socket_timeout and self.opts.timeout: - old_to = socket.getdefaulttimeout() - socket.setdefaulttimeout(self.opts.timeout) - try: - fo = opener.open(req) - finally: - socket.setdefaulttimeout(old_to) - else: - fo = opener.open(req) - hdr = fo.info() - except ValueError, e: - raise URLGrabError(1, _('Bad URL: %s') % (e, )) - except RangeError, e: - raise URLGrabError(9, str(e)) - except urllib2.HTTPError, e: - new_e = URLGrabError(14, str(e)) - new_e.code = e.code - new_e.exception = e - raise new_e - except IOError, e: - if hasattr(e, 'reason') and have_socket_timeout and \ - isinstance(e.reason, TimeoutError): - raise URLGrabError(12, _('Timeout: %s') % (e, )) - else: - raise URLGrabError(4, _('IOError: %s') % (e, )) - except OSError, e: - raise URLGrabError(5, _('OSError: %s') % (e, )) - except HTTPException, e: - raise URLGrabError(7, _('HTTP Exception (%s): %s') % \ - (e.__class__.__name__, e)) - else: - return (fo, hdr) - - def _do_grab(self): - """dump the file to self.filename.""" - if self.append: new_fo = open(self.filename, 'ab') - else: new_fo = open(self.filename, 'wb') - bs = 1024*8 - size = 0 - - block = self.read(bs) - size = size + len(block) - while block: - new_fo.write(block) - block = self.read(bs) - size = size + len(block) - - new_fo.close() - try: - modified_tuple = self.hdr.getdate_tz('last-modified') - modified_stamp = rfc822.mktime_tz(modified_tuple) - os.utime(self.filename, (modified_stamp, modified_stamp)) - except (TypeError,), e: pass - - return size - - def _fill_buffer(self, amt=None): - """fill the buffer to contain at least 'amt' bytes by reading - from the underlying file object. If amt is None, then it will - read until it gets nothing more. It updates the progress meter - and throttles after every self._rbufsize bytes.""" - # the _rbuf test is only in this first 'if' for speed. It's not - # logically necessary - if self._rbuf and not amt is None: - L = len(self._rbuf) - if amt > L: - amt = amt - L - else: - return - - # if we've made it here, then we don't have enough in the buffer - # and we need to read more. - - buf = [self._rbuf] - bufsize = len(self._rbuf) - while amt is None or amt: - # first, delay if necessary for throttling reasons - if self.opts.raw_throttle(): - diff = self._tsize/self.opts.raw_throttle() - \ - (time.time() - self._ttime) - if diff > 0: time.sleep(diff) - self._ttime = time.time() - - # now read some data, up to self._rbufsize - if amt is None: readamount = self._rbufsize - else: readamount = min(amt, self._rbufsize) - try: - new = self.fo.read(readamount) - except socket.error, e: - raise URLGrabError(4, _('Socket Error: %s') % (e, )) - except TimeoutError, e: - raise URLGrabError(12, _('Timeout: %s') % (e, )) - except IOError, e: - raise URLGrabError(4, _('IOError: %s') %(e,)) - newsize = len(new) - if not newsize: break # no more to read - - if amt: amt = amt - newsize - buf.append(new) - bufsize = bufsize + newsize - self._tsize = newsize - self._amount_read = self._amount_read + newsize - if self.opts.progress_obj: - self.opts.progress_obj.update(self._amount_read) - - self._rbuf = string.join(buf, '') - return - - def read(self, amt=None): - self._fill_buffer(amt) - if amt is None: - s, self._rbuf = self._rbuf, '' - else: - s, self._rbuf = self._rbuf[:amt], self._rbuf[amt:] - return s - - def readline(self, limit=-1): - i = string.find(self._rbuf, '\n') - while i < 0 and not (0 < limit <= len(self._rbuf)): - L = len(self._rbuf) - self._fill_buffer(L + self._rbufsize) - if not len(self._rbuf) > L: break - i = string.find(self._rbuf, '\n', L) - - if i < 0: i = len(self._rbuf) - else: i = i+1 - if 0 <= limit < len(self._rbuf): i = limit - - s, self._rbuf = self._rbuf[:i], self._rbuf[i:] - return s - - def close(self): - if self.opts.progress_obj: - self.opts.progress_obj.end(self._amount_read) - self.fo.close() - if self.opts.close_connection: - try: self.fo.close_connection() - except: pass - -_handler_cache = [] -def CachedOpenerDirector(ssl_factory = None, *handlers): - for (cached_handlers, opener) in _handler_cache: - if cached_handlers == handlers: - for handler in opener.handlers: - handler.add_parent(opener) - return opener - if not ssl_factory: - ssl_factory = sslfactory.get_factory() - opener = ssl_factory.create_opener(*handlers) - _handler_cache.append( (handlers, opener) ) - return opener - -_proxy_cache = [] -def CachedProxyHandler(proxies): - for (pdict, handler) in _proxy_cache: - if pdict == proxies: - if DEBUG: DEBUG.debug('re-using proxy settings: %s', proxies) - break - else: - for k, v in proxies.items(): - utype, url = urllib.splittype(v) - host, other = urllib.splithost(url) - if (utype is None) or (host is None): - raise URLGrabError(13, _('Bad proxy URL: %s') % v) - - if DEBUG: DEBUG.info('creating new proxy handler: %s', proxies) - handler = urllib2.ProxyHandler(proxies) - _proxy_cache.append( (proxies, handler) ) - return handler - -##################################################################### -# DEPRECATED FUNCTIONS -def set_throttle(new_throttle): - """Deprecated. Use: default_grabber.throttle = new_throttle""" - default_grabber.throttle = new_throttle - -def set_bandwidth(new_bandwidth): - """Deprecated. Use: default_grabber.bandwidth = new_bandwidth""" - default_grabber.bandwidth = new_bandwidth - -def set_progress_obj(new_progress_obj): - """Deprecated. Use: default_grabber.progress_obj = new_progress_obj""" - default_grabber.progress_obj = new_progress_obj - -def set_user_agent(new_user_agent): - """Deprecated. Use: default_grabber.user_agent = new_user_agent""" - default_grabber.user_agent = new_user_agent - -def retrygrab(url, filename=None, copy_local=0, close_connection=0, - progress_obj=None, throttle=None, bandwidth=None, - numtries=3, retrycodes=[-1,2,4,5,6,7], checkfunc=None): - """Deprecated. Use: urlgrab() with the retry arg instead""" - kwargs = {'copy_local' : copy_local, - 'close_connection' : close_connection, - 'progress_obj' : progress_obj, - 'throttle' : throttle, - 'bandwidth' : bandwidth, - 'retry' : numtries, - 'retrycodes' : retrycodes, - 'checkfunc' : checkfunc - } - return urlgrab(url, filename, **kwargs) - - -##################################################################### -# TESTING -def _main_test(): - import sys - try: url, filename = sys.argv[1:3] - except ValueError: - print 'usage:', sys.argv[0], \ - ' [copy_local=0|1] [close_connection=0|1]' - sys.exit() - - kwargs = {} - for a in sys.argv[3:]: - k, v = string.split(a, '=', 1) - kwargs[k] = int(v) - - set_throttle(1.0) - set_bandwidth(32 * 1024) - print "throttle: %s, throttle bandwidth: %s B/s" % (default_grabber.throttle, - default_grabber.bandwidth) - - try: from progress import text_progress_meter - except ImportError, e: pass - else: kwargs['progress_obj'] = text_progress_meter() - - try: name = apply(urlgrab, (url, filename), kwargs) - except URLGrabError, e: print e - else: print 'LOCAL FILE:', name - - -def _retry_test(): - import sys - try: url, filename = sys.argv[1:3] - except ValueError: - print 'usage:', sys.argv[0], \ - ' [copy_local=0|1] [close_connection=0|1]' - sys.exit() - - kwargs = {} - for a in sys.argv[3:]: - k, v = string.split(a, '=', 1) - kwargs[k] = int(v) - - try: from progress import text_progress_meter - except ImportError, e: pass - else: kwargs['progress_obj'] = text_progress_meter() - - def cfunc(filename, hello, there='foo'): - print hello, there - import random - rnum = random.random() - if rnum < .5: - print 'forcing retry' - raise URLGrabError(-1, 'forcing retry') - if rnum < .75: - print 'forcing failure' - raise URLGrabError(-2, 'forcing immediate failure') - print 'success' - return - - kwargs['checkfunc'] = (cfunc, ('hello',), {'there':'there'}) - try: name = apply(retrygrab, (url, filename), kwargs) - except URLGrabError, e: print e - else: print 'LOCAL FILE:', name - -def _file_object_test(filename=None): - import random, cStringIO, sys - if filename is None: - filename = __file__ - print 'using file "%s" for comparisons' % filename - fo = open(filename) - s_input = fo.read() - fo.close() - - for testfunc in [_test_file_object_smallread, - _test_file_object_readall, - _test_file_object_readline, - _test_file_object_readlines]: - fo_input = cStringIO.StringIO(s_input) - fo_output = cStringIO.StringIO() - wrapper = URLGrabberFileObject(fo_input, None, 0) - print 'testing %-30s ' % testfunc.__name__, - testfunc(wrapper, fo_output) - s_output = fo_output.getvalue() - if s_output == s_input: print 'passed' - else: print 'FAILED' - -def _test_file_object_smallread(wrapper, fo_output): - while 1: - s = wrapper.read(23) - fo_output.write(s) - if not s: return - -def _test_file_object_readall(wrapper, fo_output): - s = wrapper.read() - fo_output.write(s) - -def _test_file_object_readline(wrapper, fo_output): - while 1: - s = wrapper.readline() - fo_output.write(s) - if not s: return - -def _test_file_object_readlines(wrapper, fo_output): - li = wrapper.readlines() - fo_output.write(string.join(li, '')) - -if __name__ == '__main__': - _main_test() - _retry_test() - _file_object_test('test') diff --git a/naoki/urlgrabber/keepalive.py b/naoki/urlgrabber/keepalive.py deleted file mode 100644 index 71393e2b8..000000000 --- a/naoki/urlgrabber/keepalive.py +++ /dev/null @@ -1,617 +0,0 @@ -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, -# Boston, MA 02111-1307 USA - -# This file is part of urlgrabber, a high-level cross-protocol url-grabber -# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko - -"""An HTTP handler for urllib2 that supports HTTP 1.1 and keepalive. - ->>> import urllib2 ->>> from keepalive import HTTPHandler ->>> keepalive_handler = HTTPHandler() ->>> opener = urllib2.build_opener(keepalive_handler) ->>> urllib2.install_opener(opener) ->>> ->>> fo = urllib2.urlopen('http://www.python.org') - -If a connection to a given host is requested, and all of the existing -connections are still in use, another connection will be opened. If -the handler tries to use an existing connection but it fails in some -way, it will be closed and removed from the pool. - -To remove the handler, simply re-run build_opener with no arguments, and -install that opener. - -You can explicitly close connections by using the close_connection() -method of the returned file-like object (described below) or you can -use the handler methods: - - close_connection(host) - close_all() - open_connections() - -NOTE: using the close_connection and close_all methods of the handler -should be done with care when using multiple threads. - * there is nothing that prevents another thread from creating new - connections immediately after connections are closed - * no checks are done to prevent in-use connections from being closed - ->>> keepalive_handler.close_all() - -EXTRA ATTRIBUTES AND METHODS - - Upon a status of 200, the object returned has a few additional - attributes and methods, which should not be used if you want to - remain consistent with the normal urllib2-returned objects: - - close_connection() - close the connection to the host - readlines() - you know, readlines() - status - the return status (ie 404) - reason - english translation of status (ie 'File not found') - - If you want the best of both worlds, use this inside an - AttributeError-catching try: - - >>> try: status = fo.status - >>> except AttributeError: status = None - - Unfortunately, these are ONLY there if status == 200, so it's not - easy to distinguish between non-200 responses. The reason is that - urllib2 tries to do clever things with error codes 301, 302, 401, - and 407, and it wraps the object upon return. - - For python versions earlier than 2.4, you can avoid this fancy error - handling by setting the module-level global HANDLE_ERRORS to zero. - You see, prior to 2.4, it's the HTTP Handler's job to determine what - to handle specially, and what to just pass up. HANDLE_ERRORS == 0 - means "pass everything up". In python 2.4, however, this job no - longer belongs to the HTTP Handler and is now done by a NEW handler, - HTTPErrorProcessor. Here's the bottom line: - - python version < 2.4 - HANDLE_ERRORS == 1 (default) pass up 200, treat the rest as - errors - HANDLE_ERRORS == 0 pass everything up, error processing is - left to the calling code - python version >= 2.4 - HANDLE_ERRORS == 1 pass up 200, treat the rest as errors - HANDLE_ERRORS == 0 (default) pass everything up, let the - other handlers (specifically, - HTTPErrorProcessor) decide what to do - - In practice, setting the variable either way makes little difference - in python 2.4, so for the most consistent behavior across versions, - you probably just want to use the defaults, which will give you - exceptions on errors. - -""" - -# $Id: keepalive.py,v 1.16 2006/09/22 00:58:05 mstenner Exp $ - -import urllib2 -import httplib -import socket -import thread - -DEBUG = None - -import sslfactory - -import sys -if sys.version_info < (2, 4): HANDLE_ERRORS = 1 -else: HANDLE_ERRORS = 0 - -class ConnectionManager: - """ - The connection manager must be able to: - * keep track of all existing - """ - def __init__(self): - self._lock = thread.allocate_lock() - self._hostmap = {} # map hosts to a list of connections - self._connmap = {} # map connections to host - self._readymap = {} # map connection to ready state - - def add(self, host, connection, ready): - self._lock.acquire() - try: - if not self._hostmap.has_key(host): self._hostmap[host] = [] - self._hostmap[host].append(connection) - self._connmap[connection] = host - self._readymap[connection] = ready - finally: - self._lock.release() - - def remove(self, connection): - self._lock.acquire() - try: - try: - host = self._connmap[connection] - except KeyError: - pass - else: - del self._connmap[connection] - del self._readymap[connection] - self._hostmap[host].remove(connection) - if not self._hostmap[host]: del self._hostmap[host] - finally: - self._lock.release() - - def set_ready(self, connection, ready): - try: self._readymap[connection] = ready - except KeyError: pass - - def get_ready_conn(self, host): - conn = None - self._lock.acquire() - try: - if self._hostmap.has_key(host): - for c in self._hostmap[host]: - if self._readymap[c]: - self._readymap[c] = 0 - conn = c - break - finally: - self._lock.release() - return conn - - def get_all(self, host=None): - if host: - return list(self._hostmap.get(host, [])) - else: - return dict(self._hostmap) - -class KeepAliveHandler: - def __init__(self): - self._cm = ConnectionManager() - - #### Connection Management - def open_connections(self): - """return a list of connected hosts and the number of connections - to each. [('foo.com:80', 2), ('bar.org', 1)]""" - return [(host, len(li)) for (host, li) in self._cm.get_all().items()] - - def close_connection(self, host): - """close connection(s) to - host is the host:port spec, as in 'www.cnn.com:8080' as passed in. - no error occurs if there is no connection to that host.""" - for h in self._cm.get_all(host): - self._cm.remove(h) - h.close() - - def close_all(self): - """close all open connections""" - for host, conns in self._cm.get_all().items(): - for h in conns: - self._cm.remove(h) - h.close() - - def _request_closed(self, request, host, connection): - """tells us that this request is now closed and the the - connection is ready for another request""" - self._cm.set_ready(connection, 1) - - def _remove_connection(self, host, connection, close=0): - if close: connection.close() - self._cm.remove(connection) - - #### Transaction Execution - def do_open(self, req): - host = req.get_host() - if not host: - raise urllib2.URLError('no host given') - - try: - h = self._cm.get_ready_conn(host) - while h: - r = self._reuse_connection(h, req, host) - - # if this response is non-None, then it worked and we're - # done. Break out, skipping the else block. - if r: break - - # connection is bad - possibly closed by server - # discard it and ask for the next free connection - h.close() - self._cm.remove(h) - h = self._cm.get_ready_conn(host) - else: - # no (working) free connections were found. Create a new one. - h = self._get_connection(host) - if DEBUG: DEBUG.info("creating new connection to %s (%d)", - host, id(h)) - self._cm.add(host, h, 0) - self._start_transaction(h, req) - r = h.getresponse() - except (socket.error, httplib.HTTPException), err: - raise urllib2.URLError(err) - - # if not a persistent connection, don't try to reuse it - if r.will_close: self._cm.remove(h) - - if DEBUG: DEBUG.info("STATUS: %s, %s", r.status, r.reason) - r._handler = self - r._host = host - r._url = req.get_full_url() - r._connection = h - r.code = r.status - r.headers = r.msg - r.msg = r.reason - - if r.status == 200 or not HANDLE_ERRORS: - return r - else: - return self.parent.error('http', req, r, - r.status, r.msg, r.headers) - - def _reuse_connection(self, h, req, host): - """start the transaction with a re-used connection - return a response object (r) upon success or None on failure. - This DOES not close or remove bad connections in cases where - it returns. However, if an unexpected exception occurs, it - will close and remove the connection before re-raising. - """ - try: - self._start_transaction(h, req) - r = h.getresponse() - # note: just because we got something back doesn't mean it - # worked. We'll check the version below, too. - except (socket.error, httplib.HTTPException): - r = None - except: - # adding this block just in case we've missed - # something we will still raise the exception, but - # lets try and close the connection and remove it - # first. We previously got into a nasty loop - # where an exception was uncaught, and so the - # connection stayed open. On the next try, the - # same exception was raised, etc. The tradeoff is - # that it's now possible this call will raise - # a DIFFERENT exception - if DEBUG: DEBUG.error("unexpected exception - closing " + \ - "connection to %s (%d)", host, id(h)) - self._cm.remove(h) - h.close() - raise - - if r is None or r.version == 9: - # httplib falls back to assuming HTTP 0.9 if it gets a - # bad header back. This is most likely to happen if - # the socket has been closed by the server since we - # last used the connection. - if DEBUG: DEBUG.info("failed to re-use connection to %s (%d)", - host, id(h)) - r = None - else: - if DEBUG: DEBUG.info("re-using connection to %s (%d)", host, id(h)) - - return r - - def _start_transaction(self, h, req): - try: - if req.has_data(): - data = req.get_data() - h.putrequest('POST', req.get_selector()) - if not req.headers.has_key('Content-type'): - h.putheader('Content-type', - 'application/x-www-form-urlencoded') - if not req.headers.has_key('Content-length'): - h.putheader('Content-length', '%d' % len(data)) - else: - h.putrequest('GET', req.get_selector()) - except (socket.error, httplib.HTTPException), err: - raise urllib2.URLError(err) - - for args in self.parent.addheaders: - h.putheader(*args) - for k, v in req.headers.items(): - h.putheader(k, v) - h.endheaders() - if req.has_data(): - h.send(data) - - def _get_connection(self, host): - return NotImplementedError - -class HTTPHandler(KeepAliveHandler, urllib2.HTTPHandler): - def __init__(self): - KeepAliveHandler.__init__(self) - - def http_open(self, req): - return self.do_open(req) - - def _get_connection(self, host): - return HTTPConnection(host) - -class HTTPSHandler(KeepAliveHandler, urllib2.HTTPSHandler): - def __init__(self, ssl_factory=None): - KeepAliveHandler.__init__(self) - if not ssl_factory: - ssl_factory = sslfactory.get_factory() - self._ssl_factory = ssl_factory - - def https_open(self, req): - return self.do_open(req) - - def _get_connection(self, host): - return self._ssl_factory.get_https_connection(host) - -class HTTPResponse(httplib.HTTPResponse): - # we need to subclass HTTPResponse in order to - # 1) add readline() and readlines() methods - # 2) add close_connection() methods - # 3) add info() and geturl() methods - - # in order to add readline(), read must be modified to deal with a - # buffer. example: readline must read a buffer and then spit back - # one line at a time. The only real alternative is to read one - # BYTE at a time (ick). Once something has been read, it can't be - # put back (ok, maybe it can, but that's even uglier than this), - # so if you THEN do a normal read, you must first take stuff from - # the buffer. - - # the read method wraps the original to accomodate buffering, - # although read() never adds to the buffer. - # Both readline and readlines have been stolen with almost no - # modification from socket.py - - - def __init__(self, sock, debuglevel=0, strict=0, method=None): - if method: # the httplib in python 2.3 uses the method arg - httplib.HTTPResponse.__init__(self, sock, debuglevel, method) - else: # 2.2 doesn't - httplib.HTTPResponse.__init__(self, sock, debuglevel) - self.fileno = sock.fileno - self.code = None - self._rbuf = '' - self._rbufsize = 8096 - self._handler = None # inserted by the handler later - self._host = None # (same) - self._url = None # (same) - self._connection = None # (same) - - _raw_read = httplib.HTTPResponse.read - - def close(self): - if self.fp: - self.fp.close() - self.fp = None - if self._handler: - self._handler._request_closed(self, self._host, - self._connection) - - def close_connection(self): - self._handler._remove_connection(self._host, self._connection, close=1) - self.close() - - def info(self): - return self.headers - - def geturl(self): - return self._url - - def read(self, amt=None): - # the _rbuf test is only in this first if for speed. It's not - # logically necessary - if self._rbuf and not amt is None: - L = len(self._rbuf) - if amt > L: - amt -= L - else: - s = self._rbuf[:amt] - self._rbuf = self._rbuf[amt:] - return s - - s = self._rbuf + self._raw_read(amt) - self._rbuf = '' - return s - - def readline(self, limit=-1): - data = "" - i = self._rbuf.find('\n') - while i < 0 and not (0 < limit <= len(self._rbuf)): - new = self._raw_read(self._rbufsize) - if not new: break - i = new.find('\n') - if i >= 0: i = i + len(self._rbuf) - self._rbuf = self._rbuf + new - if i < 0: i = len(self._rbuf) - else: i = i+1 - if 0 <= limit < len(self._rbuf): i = limit - data, self._rbuf = self._rbuf[:i], self._rbuf[i:] - return data - - def readlines(self, sizehint = 0): - total = 0 - list = [] - while 1: - line = self.readline() - if not line: break - list.append(line) - total += len(line) - if sizehint and total >= sizehint: - break - return list - - -class HTTPConnection(httplib.HTTPConnection): - # use the modified response class - response_class = HTTPResponse - -class HTTPSConnection(httplib.HTTPSConnection): - response_class = HTTPResponse - -######################################################################### -##### TEST FUNCTIONS -######################################################################### - -def error_handler(url): - global HANDLE_ERRORS - orig = HANDLE_ERRORS - keepalive_handler = HTTPHandler() - opener = urllib2.build_opener(keepalive_handler) - urllib2.install_opener(opener) - pos = {0: 'off', 1: 'on'} - for i in (0, 1): - print " fancy error handling %s (HANDLE_ERRORS = %i)" % (pos[i], i) - HANDLE_ERRORS = i - try: - fo = urllib2.urlopen(url) - foo = fo.read() - fo.close() - try: status, reason = fo.status, fo.reason - except AttributeError: status, reason = None, None - except IOError, e: - print " EXCEPTION: %s" % e - raise - else: - print " status = %s, reason = %s" % (status, reason) - HANDLE_ERRORS = orig - hosts = keepalive_handler.open_connections() - print "open connections:", hosts - keepalive_handler.close_all() - -def continuity(url): - import md5 - format = '%25s: %s' - - # first fetch the file with the normal http handler - opener = urllib2.build_opener() - urllib2.install_opener(opener) - fo = urllib2.urlopen(url) - foo = fo.read() - fo.close() - m = md5.new(foo) - print format % ('normal urllib', m.hexdigest()) - - # now install the keepalive handler and try again - opener = urllib2.build_opener(HTTPHandler()) - urllib2.install_opener(opener) - - fo = urllib2.urlopen(url) - foo = fo.read() - fo.close() - m = md5.new(foo) - print format % ('keepalive read', m.hexdigest()) - - fo = urllib2.urlopen(url) - foo = '' - while 1: - f = fo.readline() - if f: foo = foo + f - else: break - fo.close() - m = md5.new(foo) - print format % ('keepalive readline', m.hexdigest()) - -def comp(N, url): - print ' making %i connections to:\n %s' % (N, url) - - sys.stdout.write(' first using the normal urllib handlers') - # first use normal opener - opener = urllib2.build_opener() - urllib2.install_opener(opener) - t1 = fetch(N, url) - print ' TIME: %.3f s' % t1 - - sys.stdout.write(' now using the keepalive handler ') - # now install the keepalive handler and try again - opener = urllib2.build_opener(HTTPHandler()) - urllib2.install_opener(opener) - t2 = fetch(N, url) - print ' TIME: %.3f s' % t2 - print ' improvement factor: %.2f' % (t1/t2, ) - -def fetch(N, url, delay=0): - import time - lens = [] - starttime = time.time() - for i in range(N): - if delay and i > 0: time.sleep(delay) - fo = urllib2.urlopen(url) - foo = fo.read() - fo.close() - lens.append(len(foo)) - diff = time.time() - starttime - - j = 0 - for i in lens[1:]: - j = j + 1 - if not i == lens[0]: - print "WARNING: inconsistent length on read %i: %i" % (j, i) - - return diff - -def test_timeout(url): - global DEBUG - dbbackup = DEBUG - class FakeLogger: - def debug(self, msg, *args): print msg % args - info = warning = error = debug - DEBUG = FakeLogger() - print " fetching the file to establish a connection" - fo = urllib2.urlopen(url) - data1 = fo.read() - fo.close() - - i = 20 - print " waiting %i seconds for the server to close the connection" % i - while i > 0: - sys.stdout.write('\r %2i' % i) - sys.stdout.flush() - time.sleep(1) - i -= 1 - sys.stderr.write('\r') - - print " fetching the file a second time" - fo = urllib2.urlopen(url) - data2 = fo.read() - fo.close() - - if data1 == data2: - print ' data are identical' - else: - print ' ERROR: DATA DIFFER' - - DEBUG = dbbackup - - -def test(url, N=10): - print "checking error hander (do this on a non-200)" - try: error_handler(url) - except IOError, e: - print "exiting - exception will prevent further tests" - sys.exit() - print - print "performing continuity test (making sure stuff isn't corrupted)" - continuity(url) - print - print "performing speed comparison" - comp(N, url) - print - print "performing dropped-connection check" - test_timeout(url) - -if __name__ == '__main__': - import time - import sys - try: - N = int(sys.argv[1]) - url = sys.argv[2] - except: - print "%s " % sys.argv[0] - else: - test(url, N) diff --git a/naoki/urlgrabber/mirror.py b/naoki/urlgrabber/mirror.py deleted file mode 100644 index 9664c6b5c..000000000 --- a/naoki/urlgrabber/mirror.py +++ /dev/null @@ -1,458 +0,0 @@ -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, -# Boston, MA 02111-1307 USA - -# This file is part of urlgrabber, a high-level cross-protocol url-grabber -# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko - -"""Module for downloading files from a pool of mirrors - -DESCRIPTION - - This module provides support for downloading files from a pool of - mirrors with configurable failover policies. To a large extent, the - failover policy is chosen by using different classes derived from - the main class, MirrorGroup. - - Instances of MirrorGroup (and cousins) act very much like URLGrabber - instances in that they have urlread, urlgrab, and urlopen methods. - They can therefore, be used in very similar ways. - - from urlgrabber.grabber import URLGrabber - from urlgrabber.mirror import MirrorGroup - gr = URLGrabber() - mg = MirrorGroup(gr, ['http://foo.com/some/directory/', - 'http://bar.org/maybe/somewhere/else/', - 'ftp://baz.net/some/other/place/entirely/'] - mg.urlgrab('relative/path.zip') - - The assumption is that all mirrors are identical AFTER the base urls - specified, so that any mirror can be used to fetch any file. - -FAILOVER - - The failover mechanism is designed to be customized by subclassing - from MirrorGroup to change the details of the behavior. In general, - the classes maintain a master mirror list and a "current mirror" - index. When a download is initiated, a copy of this list and index - is created for that download only. The specific failover policy - depends on the class used, and so is documented in the class - documentation. Note that ANY behavior of the class can be - overridden, so any failover policy at all is possible (although - you may need to change the interface in extreme cases). - -CUSTOMIZATION - - Most customization of a MirrorGroup object is done at instantiation - time (or via subclassing). There are four major types of - customization: - - 1) Pass in a custom urlgrabber - The passed in urlgrabber will be - used (by default... see #2) for the grabs, so options to it - apply for the url-fetching - - 2) Custom mirror list - Mirror lists can simply be a list of - stings mirrors (as shown in the example above) but each can - also be a dict, allowing for more options. For example, the - first mirror in the list above could also have been: - - {'mirror': 'http://foo.com/some/directory/', - 'grabber': , - 'kwargs': { }} - - All mirrors are converted to this format internally. If - 'grabber' is omitted, the default grabber will be used. If - kwargs are omitted, then (duh) they will not be used. - - 3) Pass keyword arguments when instantiating the mirror group. - See, for example, the failure_callback argument. - - 4) Finally, any kwargs passed in for the specific file (to the - urlgrab method, for example) will be folded in. The options - passed into the grabber's urlXXX methods will override any - options specified in a custom mirror dict. - -""" - -# $Id: mirror.py,v 1.14 2006/02/22 18:26:46 mstenner Exp $ - -import random -import thread # needed for locking to make this threadsafe - -from grabber import URLGrabError, CallbackObject, DEBUG - -try: - from i18n import _ -except ImportError, msg: - def _(st): return st - -class GrabRequest: - """This is a dummy class used to hold information about the specific - request. For example, a single file. By maintaining this information - separately, we can accomplish two things: - - 1) make it a little easier to be threadsafe - 2) have request-specific parameters - """ - pass - -class MirrorGroup: - """Base Mirror class - - Instances of this class are built with a grabber object and a list - of mirrors. Then all calls to urlXXX should be passed relative urls. - The requested file will be searched for on the first mirror. If the - grabber raises an exception (possibly after some retries) then that - mirror will be removed from the list, and the next will be attempted. - If all mirrors are exhausted, then an exception will be raised. - - MirrorGroup has the following failover policy: - - * downloads begin with the first mirror - - * by default (see default_action below) a failure (after retries) - causes it to increment the local AND master indices. Also, - the current mirror is removed from the local list (but NOT the - master list - the mirror can potentially be used for other - files) - - * if the local list is ever exhausted, a URLGrabError will be - raised (errno=256, no more mirrors) - - OPTIONS - - In addition to the required arguments "grabber" and "mirrors", - MirrorGroup also takes the following optional arguments: - - default_action - - A dict that describes the actions to be taken upon failure - (after retries). default_action can contain any of the - following keys (shown here with their default values): - - default_action = {'increment': 1, - 'increment_master': 1, - 'remove': 1, - 'remove_master': 0, - 'fail': 0} - - In this context, 'increment' means "use the next mirror" and - 'remove' means "never use this mirror again". The two - 'master' values refer to the instance-level mirror list (used - for all files), whereas the non-master values refer to the - current download only. - - The 'fail' option will cause immediate failure by re-raising - the exception and no further attempts to get the current - download. - - This dict can be set at instantiation time, - mg = MirrorGroup(grabber, mirrors, default_action={'fail':1}) - at method-execution time (only applies to current fetch), - filename = mg.urlgrab(url, default_action={'increment': 0}) - or by returning an action dict from the failure_callback - return {'fail':0} - in increasing precedence. - - If all three of these were done, the net result would be: - {'increment': 0, # set in method - 'increment_master': 1, # class default - 'remove': 1, # class default - 'remove_master': 0, # class default - 'fail': 0} # set at instantiation, reset - # from callback - - failure_callback - - this is a callback that will be called when a mirror "fails", - meaning the grabber raises some URLGrabError. If this is a - tuple, it is interpreted to be of the form (cb, args, kwargs) - where cb is the actual callable object (function, method, - etc). Otherwise, it is assumed to be the callable object - itself. The callback will be passed a grabber.CallbackObject - instance along with args and kwargs (if present). The following - attributes are defined withing the instance: - - obj.exception = < exception that was raised > - obj.mirror = < the mirror that was tried > - obj.relative_url = < url relative to the mirror > - obj.url = < full url that failed > - # .url is just the combination of .mirror - # and .relative_url - - The failure callback can return an action dict, as described - above. - - Like default_action, the failure_callback can be set at - instantiation time or when the urlXXX method is called. In - the latter case, it applies only for that fetch. - - The callback can re-raise the exception quite easily. For - example, this is a perfectly adequate callback function: - - def callback(obj): raise obj.exception - - WARNING: do not save the exception object (or the - CallbackObject instance). As they contain stack frame - references, they can lead to circular references. - - Notes: - * The behavior can be customized by deriving and overriding the - 'CONFIGURATION METHODS' - * The 'grabber' instance is kept as a reference, not copied. - Therefore, the grabber instance can be modified externally - and changes will take effect immediately. - """ - - # notes on thread-safety: - - # A GrabRequest should never be shared by multiple threads because - # it's never saved inside the MG object and never returned outside it. - # therefore, it should be safe to access/modify grabrequest data - # without a lock. However, accessing the mirrors and _next attributes - # of the MG itself must be done when locked to prevent (for example) - # removal of the wrong mirror. - - ############################################################## - # CONFIGURATION METHODS - intended to be overridden to - # customize behavior - def __init__(self, grabber, mirrors, **kwargs): - """Initialize the MirrorGroup object. - - REQUIRED ARGUMENTS - - grabber - URLGrabber instance - mirrors - a list of mirrors - - OPTIONAL ARGUMENTS - - failure_callback - callback to be used when a mirror fails - default_action - dict of failure actions - - See the module-level and class level documentation for more - details. - """ - - # OVERRIDE IDEAS: - # shuffle the list to randomize order - self.grabber = grabber - self.mirrors = self._parse_mirrors(mirrors) - self._next = 0 - self._lock = thread.allocate_lock() - self.default_action = None - self._process_kwargs(kwargs) - - # if these values are found in **kwargs passed to one of the urlXXX - # methods, they will be stripped before getting passed on to the - # grabber - options = ['default_action', 'failure_callback'] - - def _process_kwargs(self, kwargs): - self.failure_callback = kwargs.get('failure_callback') - self.default_action = kwargs.get('default_action') - - def _parse_mirrors(self, mirrors): - parsed_mirrors = [] - for m in mirrors: - if type(m) == type(''): m = {'mirror': m} - parsed_mirrors.append(m) - return parsed_mirrors - - def _load_gr(self, gr): - # OVERRIDE IDEAS: - # shuffle gr list - self._lock.acquire() - gr.mirrors = list(self.mirrors) - gr._next = self._next - self._lock.release() - - def _get_mirror(self, gr): - # OVERRIDE IDEAS: - # return a random mirror so that multiple mirrors get used - # even without failures. - if not gr.mirrors: - raise URLGrabError(256, _('No more mirrors to try.')) - return gr.mirrors[gr._next] - - def _failure(self, gr, cb_obj): - # OVERRIDE IDEAS: - # inspect the error - remove=1 for 404, remove=2 for connection - # refused, etc. (this can also be done via - # the callback) - cb = gr.kw.get('failure_callback') or self.failure_callback - if cb: - if type(cb) == type( () ): - cb, args, kwargs = cb - else: - args, kwargs = (), {} - action = cb(cb_obj, *args, **kwargs) or {} - else: - action = {} - # XXXX - decide - there are two ways to do this - # the first is action-overriding as a whole - use the entire action - # or fall back on module level defaults - #action = action or gr.kw.get('default_action') or self.default_action - # the other is to fall through for each element in the action dict - a = dict(self.default_action or {}) - a.update(gr.kw.get('default_action', {})) - a.update(action) - action = a - self.increment_mirror(gr, action) - if action and action.get('fail', 0): raise - - def increment_mirror(self, gr, action={}): - """Tell the mirror object increment the mirror index - - This increments the mirror index, which amounts to telling the - mirror object to use a different mirror (for this and future - downloads). - - This is a SEMI-public method. It will be called internally, - and you may never need to call it. However, it is provided - (and is made public) so that the calling program can increment - the mirror choice for methods like urlopen. For example, with - urlopen, there's no good way for the mirror group to know that - an error occurs mid-download (it's already returned and given - you the file object). - - remove --- can have several values - 0 do not remove the mirror from the list - 1 remove the mirror for this download only - 2 remove the mirror permanently - - beware of remove=0 as it can lead to infinite loops - """ - badmirror = gr.mirrors[gr._next] - - self._lock.acquire() - try: - ind = self.mirrors.index(badmirror) - except ValueError: - pass - else: - if action.get('remove_master', 0): - del self.mirrors[ind] - elif self._next == ind and action.get('increment_master', 1): - self._next += 1 - if self._next >= len(self.mirrors): self._next = 0 - self._lock.release() - - if action.get('remove', 1): - del gr.mirrors[gr._next] - elif action.get('increment', 1): - gr._next += 1 - if gr._next >= len(gr.mirrors): gr._next = 0 - - if DEBUG: - grm = [m['mirror'] for m in gr.mirrors] - DEBUG.info('GR mirrors: [%s] %i', ' '.join(grm), gr._next) - selfm = [m['mirror'] for m in self.mirrors] - DEBUG.info('MAIN mirrors: [%s] %i', ' '.join(selfm), self._next) - - ##################################################################### - # NON-CONFIGURATION METHODS - # these methods are designed to be largely workhorse methods that - # are not intended to be overridden. That doesn't mean you can't; - # if you want to, feel free, but most things can be done by - # by overriding the configuration methods :) - - def _join_url(self, base_url, rel_url): - if base_url.endswith('/') or rel_url.startswith('/'): - return base_url + rel_url - else: - return base_url + '/' + rel_url - - def _mirror_try(self, func, url, kw): - gr = GrabRequest() - gr.func = func - gr.url = url - gr.kw = dict(kw) - self._load_gr(gr) - - for k in self.options: - try: del kw[k] - except KeyError: pass - - while 1: - mirrorchoice = self._get_mirror(gr) - fullurl = self._join_url(mirrorchoice['mirror'], gr.url) - kwargs = dict(mirrorchoice.get('kwargs', {})) - kwargs.update(kw) - grabber = mirrorchoice.get('grabber') or self.grabber - func_ref = getattr(grabber, func) - if DEBUG: DEBUG.info('MIRROR: trying %s -> %s', url, fullurl) - try: - return func_ref( *(fullurl,), **kwargs ) - except URLGrabError, e: - if DEBUG: DEBUG.info('MIRROR: failed') - obj = CallbackObject() - obj.exception = e - obj.mirror = mirrorchoice['mirror'] - obj.relative_url = gr.url - obj.url = fullurl - self._failure(gr, obj) - - def urlgrab(self, url, filename=None, **kwargs): - kw = dict(kwargs) - kw['filename'] = filename - func = 'urlgrab' - return self._mirror_try(func, url, kw) - - def urlopen(self, url, **kwargs): - kw = dict(kwargs) - func = 'urlopen' - return self._mirror_try(func, url, kw) - - def urlread(self, url, limit=None, **kwargs): - kw = dict(kwargs) - kw['limit'] = limit - func = 'urlread' - return self._mirror_try(func, url, kw) - - -class MGRandomStart(MirrorGroup): - """A mirror group that starts at a random mirror in the list. - - This behavior of this class is identical to MirrorGroup, except that - it starts at a random location in the mirror list. - """ - - def __init__(self, grabber, mirrors, **kwargs): - """Initialize the object - - The arguments for intialization are the same as for MirrorGroup - """ - MirrorGroup.__init__(self, grabber, mirrors, **kwargs) - self._next = random.randrange(len(mirrors)) - -class MGRandomOrder(MirrorGroup): - """A mirror group that uses mirrors in a random order. - - This behavior of this class is identical to MirrorGroup, except that - it uses the mirrors in a random order. Note that the order is set at - initialization time and fixed thereafter. That is, it does not pick a - random mirror after each failure. - """ - - def __init__(self, grabber, mirrors, **kwargs): - """Initialize the object - - The arguments for intialization are the same as for MirrorGroup - """ - MirrorGroup.__init__(self, grabber, mirrors, **kwargs) - random.shuffle(self.mirrors) - -if __name__ == '__main__': - pass diff --git a/naoki/urlgrabber/progress.py b/naoki/urlgrabber/progress.py deleted file mode 100644 index 02db524e7..000000000 --- a/naoki/urlgrabber/progress.py +++ /dev/null @@ -1,530 +0,0 @@ -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, -# Boston, MA 02111-1307 USA - -# This file is part of urlgrabber, a high-level cross-protocol url-grabber -# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko - -# $Id: progress.py,v 1.7 2005/08/19 21:59:07 mstenner Exp $ - -import sys -import time -import math -import thread - -class BaseMeter: - def __init__(self): - self.update_period = 0.3 # seconds - - self.filename = None - self.url = None - self.basename = None - self.text = None - self.size = None - self.start_time = None - self.last_amount_read = 0 - self.last_update_time = None - self.re = RateEstimator() - - def start(self, filename=None, url=None, basename=None, - size=None, now=None, text=None): - self.filename = filename - self.url = url - self.basename = basename - self.text = text - - #size = None ######### TESTING - self.size = size - if not size is None: self.fsize = format_number(size) + 'B' - - if now is None: now = time.time() - self.start_time = now - self.re.start(size, now) - self.last_amount_read = 0 - self.last_update_time = now - self._do_start(now) - - def _do_start(self, now=None): - pass - - def update(self, amount_read, now=None): - # for a real gui, you probably want to override and put a call - # to your mainloop iteration function here - if now is None: now = time.time() - if (now >= self.last_update_time + self.update_period) or \ - not self.last_update_time: - self.re.update(amount_read, now) - self.last_amount_read = amount_read - self.last_update_time = now - self._do_update(amount_read, now) - - def _do_update(self, amount_read, now=None): - pass - - def end(self, amount_read, now=None): - if now is None: now = time.time() - self.re.update(amount_read, now) - self.last_amount_read = amount_read - self.last_update_time = now - self._do_end(amount_read, now) - - def _do_end(self, amount_read, now=None): - pass - -class TextMeter(BaseMeter): - def __init__(self, fo=sys.stderr): - BaseMeter.__init__(self) - self.fo = fo - - def _do_update(self, amount_read, now=None): - etime = self.re.elapsed_time() - fetime = format_time(etime) - fread = format_number(amount_read) - #self.size = None - if self.text is not None: - text = self.text - else: - text = self.basename - if self.size is None: - out = '\r%-60.60s %5sB %s ' % \ - (text, fread, fetime) - else: - rtime = self.re.remaining_time() - frtime = format_time(rtime) - frac = self.re.fraction_read() - bar = '='*int(25 * frac) - - out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s ETA ' % \ - (text, frac*100, bar, fread, frtime) - - self.fo.write(out) - self.fo.flush() - - def _do_end(self, amount_read, now=None): - total_time = format_time(self.re.elapsed_time()) - total_size = format_number(amount_read) - if self.text is not None: - text = self.text - else: - text = self.basename - if self.size is None: - out = '\r%-60.60s %5sB %s ' % \ - (text, total_size, total_time) - else: - bar = '='*25 - out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s ' % \ - (text, 100, bar, total_size, total_time) - self.fo.write(out + '\n') - self.fo.flush() - -text_progress_meter = TextMeter - -class MultiFileHelper(BaseMeter): - def __init__(self, master): - BaseMeter.__init__(self) - self.master = master - - def _do_start(self, now): - self.master.start_meter(self, now) - - def _do_update(self, amount_read, now): - # elapsed time since last update - self.master.update_meter(self, now) - - def _do_end(self, amount_read, now): - self.ftotal_time = format_time(now - self.start_time) - self.ftotal_size = format_number(self.last_amount_read) - self.master.end_meter(self, now) - - def failure(self, message, now=None): - self.master.failure_meter(self, message, now) - - def message(self, message): - self.master.message_meter(self, message) - -class MultiFileMeter: - helperclass = MultiFileHelper - def __init__(self): - self.meters = [] - self.in_progress_meters = [] - self._lock = thread.allocate_lock() - self.update_period = 0.3 # seconds - - self.numfiles = None - self.finished_files = 0 - self.failed_files = 0 - self.open_files = 0 - self.total_size = None - self.failed_size = 0 - self.start_time = None - self.finished_file_size = 0 - self.last_update_time = None - self.re = RateEstimator() - - def start(self, numfiles=None, total_size=None, now=None): - if now is None: now = time.time() - self.numfiles = numfiles - self.finished_files = 0 - self.failed_files = 0 - self.open_files = 0 - self.total_size = total_size - self.failed_size = 0 - self.start_time = now - self.finished_file_size = 0 - self.last_update_time = now - self.re.start(total_size, now) - self._do_start(now) - - def _do_start(self, now): - pass - - def end(self, now=None): - if now is None: now = time.time() - self._do_end(now) - - def _do_end(self, now): - pass - - def lock(self): self._lock.acquire() - def unlock(self): self._lock.release() - - ########################################################### - # child meter creation and destruction - def newMeter(self): - newmeter = self.helperclass(self) - self.meters.append(newmeter) - return newmeter - - def removeMeter(self, meter): - self.meters.remove(meter) - - ########################################################### - # child functions - these should only be called by helpers - def start_meter(self, meter, now): - if not meter in self.meters: - raise ValueError('attempt to use orphaned meter') - self._lock.acquire() - try: - if not meter in self.in_progress_meters: - self.in_progress_meters.append(meter) - self.open_files += 1 - finally: - self._lock.release() - self._do_start_meter(meter, now) - - def _do_start_meter(self, meter, now): - pass - - def update_meter(self, meter, now): - if not meter in self.meters: - raise ValueError('attempt to use orphaned meter') - if (now >= self.last_update_time + self.update_period) or \ - not self.last_update_time: - self.re.update(self._amount_read(), now) - self.last_update_time = now - self._do_update_meter(meter, now) - - def _do_update_meter(self, meter, now): - pass - - def end_meter(self, meter, now): - if not meter in self.meters: - raise ValueError('attempt to use orphaned meter') - self._lock.acquire() - try: - try: self.in_progress_meters.remove(meter) - except ValueError: pass - self.open_files -= 1 - self.finished_files += 1 - self.finished_file_size += meter.last_amount_read - finally: - self._lock.release() - self._do_end_meter(meter, now) - - def _do_end_meter(self, meter, now): - pass - - def failure_meter(self, meter, message, now): - if not meter in self.meters: - raise ValueError('attempt to use orphaned meter') - self._lock.acquire() - try: - try: self.in_progress_meters.remove(meter) - except ValueError: pass - self.open_files -= 1 - self.failed_files += 1 - if meter.size and self.failed_size is not None: - self.failed_size += meter.size - else: - self.failed_size = None - finally: - self._lock.release() - self._do_failure_meter(meter, message, now) - - def _do_failure_meter(self, meter, message, now): - pass - - def message_meter(self, meter, message): - pass - - ######################################################## - # internal functions - def _amount_read(self): - tot = self.finished_file_size - for m in self.in_progress_meters: - tot += m.last_amount_read - return tot - - -class TextMultiFileMeter(MultiFileMeter): - def __init__(self, fo=sys.stderr): - self.fo = fo - MultiFileMeter.__init__(self) - - # files: ###/### ###% data: ######/###### ###% time: ##:##:##/##:##:## - def _do_update_meter(self, meter, now): - self._lock.acquire() - try: - format = "files: %3i/%-3i %3i%% data: %6.6s/%-6.6s %3i%% " \ - "time: %8.8s/%8.8s" - df = self.finished_files - tf = self.numfiles or 1 - pf = 100 * float(df)/tf + 0.49 - dd = self.re.last_amount_read - td = self.total_size - pd = 100 * (self.re.fraction_read() or 0) + 0.49 - dt = self.re.elapsed_time() - rt = self.re.remaining_time() - if rt is None: tt = None - else: tt = dt + rt - - fdd = format_number(dd) + 'B' - ftd = format_number(td) + 'B' - fdt = format_time(dt, 1) - ftt = format_time(tt, 1) - - out = '%-79.79s' % (format % (df, tf, pf, fdd, ftd, pd, fdt, ftt)) - self.fo.write('\r' + out) - self.fo.flush() - finally: - self._lock.release() - - def _do_end_meter(self, meter, now): - self._lock.acquire() - try: - format = "%-30.30s %6.6s %8.8s %9.9s" - fn = meter.basename - size = meter.last_amount_read - fsize = format_number(size) + 'B' - et = meter.re.elapsed_time() - fet = format_time(et, 1) - frate = format_number(size / et) + 'B/s' - - out = '%-79.79s' % (format % (fn, fsize, fet, frate)) - self.fo.write('\r' + out + '\n') - finally: - self._lock.release() - self._do_update_meter(meter, now) - - def _do_failure_meter(self, meter, message, now): - self._lock.acquire() - try: - format = "%-30.30s %6.6s %s" - fn = meter.basename - if type(message) in (type(''), type(u'')): - message = message.splitlines() - if not message: message = [''] - out = '%-79s' % (format % (fn, 'FAILED', message[0] or '')) - self.fo.write('\r' + out + '\n') - for m in message[1:]: self.fo.write(' ' + m + '\n') - self._lock.release() - finally: - self._do_update_meter(meter, now) - - def message_meter(self, meter, message): - self._lock.acquire() - try: - pass - finally: - self._lock.release() - - def _do_end(self, now): - self._do_update_meter(None, now) - self._lock.acquire() - try: - self.fo.write('\n') - self.fo.flush() - finally: - self._lock.release() - -###################################################################### -# support classes and functions - -class RateEstimator: - def __init__(self, timescale=5.0): - self.timescale = timescale - - def start(self, total=None, now=None): - if now is None: now = time.time() - self.total = total - self.start_time = now - self.last_update_time = now - self.last_amount_read = 0 - self.ave_rate = None - - def update(self, amount_read, now=None): - if now is None: now = time.time() - if amount_read == 0: - # if we just started this file, all bets are off - self.last_update_time = now - self.last_amount_read = 0 - self.ave_rate = None - return - - #print 'times', now, self.last_update_time - time_diff = now - self.last_update_time - read_diff = amount_read - self.last_amount_read - self.last_update_time = now - self.last_amount_read = amount_read - self.ave_rate = self._temporal_rolling_ave(\ - time_diff, read_diff, self.ave_rate, self.timescale) - #print 'results', time_diff, read_diff, self.ave_rate - - ##################################################################### - # result methods - def average_rate(self): - "get the average transfer rate (in bytes/second)" - return self.ave_rate - - def elapsed_time(self): - "the time between the start of the transfer and the most recent update" - return self.last_update_time - self.start_time - - def remaining_time(self): - "estimated time remaining" - if not self.ave_rate or not self.total: return None - return (self.total - self.last_amount_read) / self.ave_rate - - def fraction_read(self): - """the fraction of the data that has been read - (can be None for unknown transfer size)""" - if self.total is None: return None - elif self.total == 0: return 1.0 - else: return float(self.last_amount_read)/self.total - - ######################################################################### - # support methods - def _temporal_rolling_ave(self, time_diff, read_diff, last_ave, timescale): - """a temporal rolling average performs smooth averaging even when - updates come at irregular intervals. This is performed by scaling - the "epsilon" according to the time since the last update. - Specifically, epsilon = time_diff / timescale - - As a general rule, the average will take on a completely new value - after 'timescale' seconds.""" - epsilon = time_diff / timescale - if epsilon > 1: epsilon = 1.0 - return self._rolling_ave(time_diff, read_diff, last_ave, epsilon) - - def _rolling_ave(self, time_diff, read_diff, last_ave, epsilon): - """perform a "rolling average" iteration - a rolling average "folds" new data into an existing average with - some weight, epsilon. epsilon must be between 0.0 and 1.0 (inclusive) - a value of 0.0 means only the old value (initial value) counts, - and a value of 1.0 means only the newest value is considered.""" - - try: - recent_rate = read_diff / time_diff - except ZeroDivisionError: - recent_rate = None - if last_ave is None: return recent_rate - elif recent_rate is None: return last_ave - - # at this point, both last_ave and recent_rate are numbers - return epsilon * recent_rate + (1 - epsilon) * last_ave - - def _round_remaining_time(self, rt, start_time=15.0): - """round the remaining time, depending on its size - If rt is between n*start_time and (n+1)*start_time round downward - to the nearest multiple of n (for any counting number n). - If rt < start_time, round down to the nearest 1. - For example (for start_time = 15.0): - 2.7 -> 2.0 - 25.2 -> 25.0 - 26.4 -> 26.0 - 35.3 -> 34.0 - 63.6 -> 60.0 - """ - - if rt < 0: return 0.0 - shift = int(math.log(rt/start_time)/math.log(2)) - rt = int(rt) - if shift <= 0: return rt - return float(int(rt) >> shift << shift) - - -def format_time(seconds, use_hours=0): - if seconds is None or seconds < 0: - if use_hours: return '--:--:--' - else: return '--:--' - else: - seconds = int(seconds) - minutes = seconds / 60 - seconds = seconds % 60 - if use_hours: - hours = minutes / 60 - minutes = minutes % 60 - return '%02i:%02i:%02i' % (hours, minutes, seconds) - else: - return '%02i:%02i' % (minutes, seconds) - -def format_number(number, SI=0, space=' '): - """Turn numbers into human-readable metric-like numbers""" - symbols = ['', # (none) - 'k', # kilo - 'M', # mega - 'G', # giga - 'T', # tera - 'P', # peta - 'E', # exa - 'Z', # zetta - 'Y'] # yotta - - if SI: step = 1000.0 - else: step = 1024.0 - - thresh = 999 - depth = 0 - max_depth = len(symbols) - 1 - - # we want numbers between 0 and thresh, but don't exceed the length - # of our list. In that event, the formatting will be screwed up, - # but it'll still show the right number. - while number > thresh and depth < max_depth: - depth = depth + 1 - number = number / step - - if type(number) == type(1) or type(number) == type(1L): - # it's an int or a long, which means it didn't get divided, - # which means it's already short enough - format = '%i%s%s' - elif number < 9.95: - # must use 9.95 for proper sizing. For example, 9.99 will be - # rounded to 10.0 with the .1f format string (which is too long) - format = '%.1f%s%s' - else: - format = '%.0f%s%s' - - return(format % (float(number or 0), space, symbols[depth])) diff --git a/naoki/urlgrabber/sslfactory.py b/naoki/urlgrabber/sslfactory.py deleted file mode 100644 index f7e6d3d7e..000000000 --- a/naoki/urlgrabber/sslfactory.py +++ /dev/null @@ -1,89 +0,0 @@ -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, -# Boston, MA 02111-1307 USA - -# This file is part of urlgrabber, a high-level cross-protocol url-grabber - -import httplib -import urllib2 - -try: - from M2Crypto import SSL - from M2Crypto import httpslib - from M2Crypto import m2urllib2 - - have_m2crypto = True -except ImportError: - have_m2crypto = False - -DEBUG = None - -if have_m2crypto: - - class M2SSLFactory: - - def __init__(self, ssl_ca_cert, ssl_context): - self.ssl_context = self._get_ssl_context(ssl_ca_cert, ssl_context) - - def _get_ssl_context(self, ssl_ca_cert, ssl_context): - """ - Create an ssl context using the CA cert file or ssl context. - - The CA cert is used first if it was passed as an option. If not, - then the supplied ssl context is used. If no ssl context was supplied, - None is returned. - """ - if ssl_ca_cert: - context = SSL.Context() - context.load_verify_locations(ssl_ca_cert) - context.set_verify(SSL.verify_peer, -1) - return context - else: - return ssl_context - - def create_https_connection(self, host, response_class = None): - connection = httplib.HTTPSConnection(host, self.ssl_context) - if response_class: - connection.response_class = response_class - return connection - - def create_opener(self, *handlers): - return m2urllib2.build_opener(self.ssl_context, *handlers) - - -class SSLFactory: - - def create_https_connection(self, host, response_class = None): - connection = httplib.HTTPSConnection(host) - if response_class: - connection.response_class = response_class - return connection - - def create_opener(self, *handlers): - return urllib2.build_opener(*handlers) - - - -def get_factory(ssl_ca_cert = None, ssl_context = None): - """ Return an SSLFactory, based on if M2Crypto is available. """ - if have_m2crypto: - return M2SSLFactory(ssl_ca_cert, ssl_context) - else: - # Log here if someone provides the args but we don't use them. - if ssl_ca_cert or ssl_context: - if DEBUG: - DEBUG.warning("SSL arguments supplied, but M2Crypto is not available. " - "Using Python SSL.") - return SSLFactory() diff --git a/naoki/util.py b/naoki/util.py deleted file mode 100644 index 22623adc4..000000000 --- a/naoki/util.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/python - -import ctypes -import fcntl -import os -import random -import select -import shutil -import string -import subprocess -import sys -import time - -from constants import * -from exception import * -from logger import getLog - -_libc = ctypes.cdll.LoadLibrary(None) -_errno = ctypes.c_int.in_dll(_libc, "errno") -_libc.personality.argtypes = [ctypes.c_ulong] -_libc.personality.restype = ctypes.c_int -_libc.unshare.argtypes = [ctypes.c_int,] -_libc.unshare.restype = ctypes.c_int -CLONE_NEWNS = 0x00020000 - -# taken from sys/personality.h -PER_LINUX32=0x0008 -PER_LINUX=0x0000 -personality_defs = { - 'linux64': PER_LINUX, - 'linux32': PER_LINUX32, -} - -def touch(filename): - getLog().debug("touching file: %s" % filename) - f = open(filename, "w") - f.close() - -def mkdir(*args): - for dirName in args: - getLog().debug("ensuring that dir exists: %s" % dirName) - if not os.path.exists(dirName): - try: - getLog().debug("creating dir: %s" % dirName) - os.makedirs(dirName) - except OSError, e: - getLog().exception("Could not create dir %s. Error: %s" % (dirName, e)) - raise Error, "Could not create dir %s. Error: %s" % (dirName, e) - -def rm(path, *args, **kargs): - """version os shutil.rmtree that ignores no-such-file-or-directory errors, - and tries harder if it finds immutable files""" - tryAgain = 1 - failedFilename = None - getLog().debug("remove tree: %s" % path) - while tryAgain: - tryAgain = 0 - try: - shutil.rmtree(path, *args, **kargs) - except OSError, e: - if e.errno == 2: # no such file or directory - pass - elif e.errno==1 or e.errno==13: - tryAgain = 1 - if failedFilename == e.filename: - raise - failedFilename = e.filename - os.system("chattr -R -i %s" % path) - else: - raise - -def logOutput(fds, logger, returnOutput=1, start=0, timeout=0): - output="" - done = 0 - - # set all fds to nonblocking - for fd in fds: - flags = fcntl.fcntl(fd, fcntl.F_GETFL) - if not fd.closed: - fcntl.fcntl(fd, fcntl.F_SETFL, flags| os.O_NONBLOCK) - - tail = "" - while not done: - if (time.time() - start)>timeout and timeout!=0: - done = 1 - break - - i_rdy,o_rdy,e_rdy = select.select(fds,[],[],1) - for s in i_rdy: - # slurp as much input as is ready - input = s.read() - if input == "": - done = 1 - break - if logger is not None: - lines = input.split("\n") - if tail: - lines[0] = tail + lines[0] - # we may not have all of the last line - tail = lines.pop() - for line in lines: - if line == '': continue - logger.info(line) - for h in logger.handlers: - h.flush() - if returnOutput: - output += input - if tail and logger is not None: - logger.debug(tail) - return output - -# these are called in child process, so no logging -def condChroot(chrootPath): - if chrootPath is not None: - os.chdir(chrootPath) - os.chroot(chrootPath) - -def condChdir(cwd): - if cwd is not None: - os.chdir(cwd) - -def condPersonality(per=None): - if not per: - return - if personality_defs.get(per, None) is None: - return - res = _libc.personality(personality_defs[per]) - if res == -1: - raise OSError(_errno.value, os.strerror(_errno.value)) - -def do(command, shell=False, chrootPath=None, cwd=None, timeout=0, raiseExc=True, returnOutput=0, personality=None, logger=None, *args, **kargs): - output = "" - start = time.time() - env = kargs.get("env", None) - preexec = ChildPreExec(personality, chrootPath, cwd) - - try: - child = None - if logger: - logger.debug("Executing command: %s" % command) - child = subprocess.Popen( - command, - shell=shell, - bufsize=0, close_fds=True, - stdin=open("/dev/null", "r"), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - preexec_fn = preexec, - env=env - ) - - # use select() to poll for output so we dont block - output = logOutput([child.stdout, child.stderr], - logger, returnOutput, start, timeout) - - except: - # kill children if they arent done - if child is not None and child.returncode is None: - os.killpg(child.pid, 9) - try: - if child is not None: - os.waitpid(child.pid, 0) - except: - pass - raise - - # wait until child is done, kill it if it passes timeout - niceExit=1 - while child.poll() is None: - if (time.time() - start)>timeout and timeout!=0: - niceExit=0 - os.killpg(child.pid, 15) - if (time.time() - start)>(timeout+1) and timeout!=0: - niceExit=0 - os.killpg(child.pid, 9) - - if not niceExit: - raise commandTimeoutExpired, ("Timeout(%s) expired for command:\n # %s\n%s" % (timeout, command, output)) - - if logger: - logger.debug("Child returncode was: %s" % str(child.returncode)) - if raiseExc and child.returncode: - if returnOutput: - raise Error, ("Command failed: \n # %s\n%s" % (command, output), child.returncode) - else: - raise Error, ("Command failed. See logs for output.\n # %s" % (command,), child.returncode) - - return output - -class ChildPreExec(object): - def __init__(self, personality, chrootPath, cwd): - self.personality = personality - self.chrootPath = chrootPath - self.cwd = cwd - - def __call__(self, *args, **kargs): - os.setpgrp() - condPersonality(self.personality) - condChroot(self.chrootPath) - condChdir(self.cwd) - -def random_string(length=5): - ret = "" - while length: - ret += random.choice(string.hexdigits) - length -= 1 - - return ret diff --git a/pkgs/Constants b/pkgs/Constants deleted file mode 100644 index d8bedf28f..000000000 --- a/pkgs/Constants +++ /dev/null @@ -1,153 +0,0 @@ - -############################################################################### -# -# Constant definitions of the naoki build system -# -############################################################################### - -# If we are running in a chroot, we map all directories to $(BASEDIR), otherwise -# BASEDIR will be the absolute path (for gathering information from the source -# packages) -ifeq "$(CHROOT)" "1" - BASEDIR = /usr/src -endif - -# Set default directories -DIR_APP = $(DIR_SRC)/$(THISAPP) -DIR_DL = $(BASEDIR)/cache -DIR_PACKAGES = /usr/src/packages/$(PKG_ARCH) -DIR_PATCHES = $(DIR_SOURCE)/patches -DIR_SRC = /usr/src -DIR_TMP = /tmp -DIR_SOURCE = $(CURDIR) -DIR_TOOLS = $(BASEDIR)/tools - -# Directory where to search for object files -VPATH = $(DIR_DL) - -# Paths to scripts -DO_EXTRACT = tar xaf -DO_QUALITY_AGENT = $(DIR_TOOLS)/quality-agent - -############################################################################### -# -# Build environment -# -############################################################################### - -# Export CFLAGS + CXXFLAGS -export CFLAGS -export CXXFLAGS - -# Options that get passed to configure by default -CONFIGURE_OPTIONS = \ - --prefix=/usr \ - --build=$(TARGET) \ - --host=$(TARGET) - -# For compatibility with pakfire3 -DISTRO_MACHINE=$(TARGET) - -############################################################################### -# -# Packager variables -# -############################################################################### -# Variables that get exported (and expanded for the sub-packages) to the -# packager process -PKG_VARIABLES = \ - CONTROL_PREIN \ - CONTROL_PREUN \ - CONTROL_POSTIN \ - CONTROL_POSTUN \ - \ - PKG_ARCH \ - PKG_BUILD_DEPS \ - PKG_DEPS \ - PKG_DESCRIPTION \ - PKG_EPOCH \ - PKG_FILES \ - PKG_GROUP \ - PKG_MAINTAINER \ - PKG_LICENSE \ - PKG_PROVIDES \ - PKG_REL \ - PKG_SUMMARY \ - PKG_URL \ - PKG_VER - -# Variables that exported to the packager process -# These reflect settings from the build system -export BUILD_DATE ?= $(shell date -u) -export BUILD_HOST ?= $(shell cat /proc/sys/kernel/hostname) -export BUILD_ID - -############################################################################### -# -# Package variables -# -############################################################################### -# -# Variables -# - -# The actual package name (the name of the directory) -PKG_NAME_REAL = $(notdir $(CURDIR)) -PKG_NAME = $(PKG_NAME_REAL) - -# Set default epoch -PKG_EPOCH = 0 - -# Shortcut to package name + version -THISAPP = $(PKG_NAME)-$(PKG_VER) -THISVER = $(PKG_EPOCH):$(PKG_VER)-$(PKG_REL) - -# All packages depend on gcc and headers by default. -PKG_BUILD_DEPS+= gcc glibc-devel kernel-headers -PKG_DEPS += - -# All PKG_OBJECTS are downloaded. This is in most cases the tarball. -PKG_OBJECTS = $(PKG_TARBALL) -OBJECTS = $(PKG_OBJECTS) - -# List of packages to build -PKG_PACKAGES = $(PKG_NAME_REAL) - -# -# Macros -# - -# Abstract variable that translates the package names to a list of filenames -PKG_PACKAGES_FILES = $(foreach package,$(PKG_PACKAGES),$(call DO_PACKAGE_FILENAME,$(package))) - -# Automatically detect all patches in "patches" -PKG_PATCHES = \ - $(foreach patch,$(wildcard $(DIR_PATCHES)/*.patch),$(notdir $(patch))) -PKG_PATCHES += \ - $(foreach patch,$(wildcard $(DIR_PATCHES)/*.patch0),$(notdir $(patch))) -PKG_PATCHES += \ - $(foreach patch,$(wildcard $(DIR_PATCHES)/*.diff),$(notdir $(patch))) - -# Dynamic command that applies all patches -DO_PATCHES = cd $(DIR_APP) && $(DIR_TOOLS)/patch $(foreach patch,$(PKG_PATCHES),$(DIR_PATCHES)/$(patch)) - -# Get a list of files that are installed automatically -PKG_DEFAULT_FILES = $(wildcard *.default) -PKG_DEFAULT_FILES += $(wildcard default/*) -PKG_INIT_FILES = $(wildcard *.init) -PKG_INIT_FILES += $(wildcard init/*.conf) -PKG_PAM_FILES = $(wildcard *.pam) -PKG_PAM_FILES += $(wildcard pam.d/*) - -############################################################################### -# -# Quality agent -# -############################################################################### - -export QUALITY_AGENT_PERMIT_NOT_FULL_RELRO -export QUALITY_AGENT_RPATH_ALLOW_ORIGIN -export QUALITY_AGENT_WHITELIST_EXECSTACK -export QUALITY_AGENT_WHITELIST_NX -export QUALITY_AGENT_WHITELIST_RPATH -export QUALITY_AGENT_WHITELIST_SONAME diff --git a/pkgs/Functions b/pkgs/Functions deleted file mode 100644 index 42cfa80fe..000000000 --- a/pkgs/Functions +++ /dev/null @@ -1,177 +0,0 @@ - -############################################################################### -# -# Function definitions of the naoki build system -# -############################################################################### - -include $(PKGROOT)/gmsl - -DO_PACKAGE_FILENAME = $(1)$(call DO_PKG_SUFFIX,$(1)) - -DO_PKG_SUFFIX = \ - -$(if $(PKG_VER-$(1)),$(PKG_VER-$(1)),$(PKG_VER))-$(DISTRO_SNAME)$(DISTRO_EPOCH)-$(PKG_ARCH).$(if $(PKG_REL-$(1)),$(PKG_REL-$(1)),$(PKG_REL)).ipk - -define DO_INIT - # Run ldconfig - ldconfig -endef - -define DO_PACKAGE - @echo "#####################################################################" - @echo "# $(1) - Package build started" - @echo "#####################################################################" - - @$(foreach var,$(PKG_VARIABLES),$(if $($(var)-$(1)),$(var)="$(strip $($(var)-$(1)))",$(var)="$(strip $($(var)))")) \ - $(DIR_TOOLS)/packager $(1) $(DIR_PACKAGES)/$(call DO_PACKAGE_FILENAME,$(1)) - - @echo "#####################################################################" - @echo "# $(1) - Package build finished" - @echo "#####################################################################" - -endef - -define DO_FILELIST - @echo "# Filelist dump" - @cd $(BUILDROOT) && find -ls -endef - -define __INSTALL_DEFAULT - -mkdir -pv $(BUILDROOT)/etc/default - cd $(DIR_APP) && cp -vf $(DIR_SOURCE)/$(1) $(BUILDROOT)/etc/default/$(subst .default,,$(notdir $(1))) - -endef - -define DO_INSTALL_DEFAULT - $(foreach file,$(PKG_DEFAULT_FILES),$(call __INSTALL_DEFAULT,$(file))) -endef - -define __INSTALL_INIT - -mkdir -pv $(BUILDROOT)/etc/init - cd $(DIR_APP) && cp -vf $(DIR_SOURCE)/$(1) $(BUILDROOT)/etc/init/$(subst .init,.conf,$(notdir $(1))) - -endef - -define DO_INSTALL_INIT - $(foreach file,$(PKG_INIT_FILES),$(call __INSTALL_INIT,$(file))) -endef - -define __INSTALL_PAM - -mkdir -pv $(BUILDROOT)/etc/pam.d - cd $(DIR_APP) && cp -vf $(DIR_SOURCE)/$(1) $(BUILDROOT)/etc/pam.d/$(subst .pam,,$(notdir $(1))) - -endef - -define DO_INSTALL_PAM - $(foreach file,$(PKG_PAM_FILES),$(call __INSTALL_PAM,$(file))) -endef - -define DO_PYTHON_COMPILE - @find $(BUILDROOT) -name "*.py" | xargs $(DIR_TOOLS)/py-compile -endef - -define DO_FIX_LIBTOOL - # remove rpath from libtool - @if [ -e "libtool" ]; then \ - sed -i libtool \ - -e 's|^hardcode_libdir_flag_spec=.*|hardcode_libdir_flag_spec=""|g' \ - -e 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g'; \ - fi -endef - -define DO_PREPARE - # Initialize the environment at the beginning - $(DO_INIT) - - @echo "#####################################################################" - @echo "# $(PKG_NAME) - Preparation started" - @echo "#####################################################################" - - $(STAGE_PREPARE) - - @echo "#####################################################################" - @echo "# $(PKG_NAME) - Preparation finished" - @echo "#####################################################################" -endef - -define DO_BUILD - @echo "#####################################################################" - @echo "# $(PKG_NAME) - Build started" - @echo "#####################################################################" - - $(STAGE_BUILD) - - @echo "#####################################################################" - @echo "# $(PKG_NAME) - Build finished" - @echo "#####################################################################" -endef - -define DO_TEST - @echo "#####################################################################" - @echo "# $(PKG_NAME) - Test started" - @echo "#####################################################################" - - $(STAGE_TEST) - - @echo "#####################################################################" - @echo "# $(PKG_NAME) - Test finished" - @echo "#####################################################################" -endef - -define DO_INSTALL - @echo "#####################################################################" - @echo "# $(PKG_NAME) - Install started" - @echo "#####################################################################" - - -mkdir -pv $(BUILDROOT) - - $(STAGE_INSTALL) - - $(DO_INSTALL_DEFAULT) - $(DO_INSTALL_INIT) - $(DO_INSTALL_PAM) - - $(DO_PYTHON_COMPILE) - - @echo "#####################################################################" - @echo "# $(PKG_NAME) - Install finished" - @echo "#####################################################################" - - $(DO_QUALITY_AGENT) - $(DO_FILELIST) -endef - -STAGE_PACKAGE_TARGETS = $(call reverse,$(PKG_PACKAGES_FILES)) -STAGE_DONE = $(ROOT)/.done - -define STAGE_PREPARE - $(if $(PKG_TARBALL),cd $(DIR_SRC) && $(DO_EXTRACT) $(DIR_DL)/$(PKG_TARBALL)) - - $(if $(PKG_PATCHES),$(DO_PATCHES)) - - $(STAGE_PREPARE_CMDS) - $(STAGE_PREPARE_CMDS2) -endef - -STAGE_BUILD_TARGETS = - -define STAGE_BUILD - cd $(DIR_APP) && \ - $(CONFIGURE_ENVIRONMENT) \ - ./configure \ - $(CONFIGURE_OPTIONS) - - $(DO_FIX_LIBTOOL) - $(STAGE_CONFIGURE_CMDS) - - cd $(DIR_APP) && make $(STAGE_BUILD_TARGETS) $(PARALLELISMFLAGS) - $(STAGE_BUILD_CMDS) -endef - -STAGE_INSTALL_TARGETS = install - -define STAGE_INSTALL - cd $(DIR_APP) && make $(STAGE_INSTALL_TARGETS) DESTDIR=$(BUILDROOT) - - $(STAGE_INSTALL_CMDS) -endef diff --git a/pkgs/Include b/pkgs/Include deleted file mode 100644 index a3d62be3d..000000000 --- a/pkgs/Include +++ /dev/null @@ -1,26 +0,0 @@ -############################################################################### -# # -# IPFire.org - A linux based firewall # -# Copyright (C) 2007, 2008, 2009 Michael Tremer & Christian Schmidt # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -############################################################################### - -.SECONDEXPANSION: - -include $(PKGROOT)/Constants -include $(PKGROOT)/Functions -include $(PKGROOT)/Targets -include $(PKGROOT)/Templates diff --git a/pkgs/Targets b/pkgs/Targets deleted file mode 100644 index a8d9caf2f..000000000 --- a/pkgs/Targets +++ /dev/null @@ -1,49 +0,0 @@ - -############################################################################### -# -# Target definitions of the naoki build system -# -############################################################################### - -.PHONY: info -info: - @echo "PKG_ARCH=\"$(PKG_ARCH)\"" - @echo "PKG_BUILD_DEPENDENCIES=\"$(PKG_BUILD_DEPS)\"" - @echo "PKG_DEPENDENCIES=\"$(PKG_DEPS)\"" - @echo "PKG_DESCRIPTION=\"$(strip $(PKG_DESCRIPTION))\"" - @echo "PKG_EPOCH=\"$(PKG_EPOCH)\"" - @echo "PKG_GROUP=\"$(PKG_GROUP)\"" - @echo "PKG_LICENSE=\"$(PKG_LICENSE)\"" - @echo "PKG_MAINTAINER=\"$(PKG_MAINTAINER)\"" - @echo "PKG_NAME=\"$(PKG_NAME_REAL)\"" - @echo "PKG_OBJECTS=\"$(strip $(OBJECTS))\"" - @echo "PKG_PACKAGES=\"$(PKG_PACKAGES)\"" - @echo "PKG_PACKAGES_FILES=\"$(PKG_PACKAGES_FILES)\"" - @echo "PKG_PATCHES=\"$(PKG_PATCHES)\"" - @echo "PKG_VER=\"$(PKG_VER)\"" - @echo "PKG_REL=\"$(PKG_REL)\"" - @echo "PKG_SUMMARY=\"$(strip $(PKG_SUMMARY))\"" - @echo "PKG_URL=\"$(PKG_URL)\"" - @echo "SOURCE_DIR=\"$(DIR_APP)\"" - -$(OBJECTS): - @echo "Object file \"$@\" is required." >&2 - @exit 1 - -.PHONY: package -package: $(STAGE_DONE) - $(foreach package,$(call reverse,$(PKG_PACKAGES)),$(call DO_PACKAGE,$(package))) - -.PHONY: shell -shell: $(OBJECTS) - $(if $(STAGE_PREPARE),$(DO_PREPARE)) - -prepare: - $(if $(STAGE_PREPARE),$(DO_PREPARE)) - -$(STAGE_DONE): $(OBJECTS) - $(if $(STAGE_PREPARE),$(DO_PREPARE)) - $(if $(STAGE_BUILD),$(DO_BUILD)) - $(if $(STAGE_TEST),$(DO_TEST)) - $(if $(STAGE_INSTALL),$(DO_INSTALL)) - @touch $@ diff --git a/pkgs/Templates b/pkgs/Templates deleted file mode 100644 index e1196ad3c..000000000 --- a/pkgs/Templates +++ /dev/null @@ -1,62 +0,0 @@ - -############################################################################### -# -# Template definitions of the naoki build system -# -############################################################################### - -############################################################################### -# Default template -############################################################################### - -PKG_PROVIDES-$(PKG_NAME_REAL) = $(PKG_NAME_REAL)=$(THISVER) - -# Package all files by default -PKG_FILES = / - -############################################################################### -# Devel template -############################################################################### - -PKG_DESCRIPTION-$(PKG_NAME_REAL)-devel = Development files of $(THISAPP). -PKG_SUMMARY-$(PKG_NAME_REAL)-devel = $(PKG_DESCRIPTION-$(PKG_NAME_REAL)-devel) - -PKG_DEPS-$(PKG_NAME_REAL)-devel = $(PKG_NAME_REAL)=$(THISVER) - -define PKG_FILES-$(PKG_NAME_REAL)-devel - /usr/bin/*-config - /usr/include - /usr/lib/*.a - /usr/lib/pkgconfig - /usr/share/aclocal - */lib/*.so - /usr/share/*/cmake - /usr/share/man/man2 - /usr/share/man/man3 - /usr/share/vala -endef - - -############################################################################### -# Library template -############################################################################### - -PKG_DESCRIPTION-$(PKG_NAME_REAL)-libs = Library files of $(THISAPP). -PKG_SUMMARY-$(PKG_NAME_REAL)-libs = $(PKG_DESCRIPTION-$(PKG_NAME_REAL)-libs) - -PKG_DEPS-$(PKG_NAME_REAL)-libs = $(PKG_NAME_REAL)=$(THISVER) - -define PKG_FILES-$(PKG_NAME_REAL)-libs - /lib/lib*.so.* - /usr/lib/lib*.so.* -endef - -# Another naming scheme template... -PKG_DESCRIPTION-lib$(PKG_NAME_REAL) = $(PKG_DESCRIPTION-$(PKG_NAME_REAL)-libs) -PKG_SUMMARY-lib$(PKG_NAME_REAL) = $(PKG_SUMMARY-$(PKG_NAME_REAL)-libs) -PKG_FILES-lib$(PKG_NAME_REAL) = $(PKG_FILES-$(PKG_NAME_REAL)-libs) - -PKG_DESCRIPTION-lib$(PKG_NAME_REAL)-devel = $(PKG_DESCRIPTION-$(PKG_NAME_REAL)-devel) -PKG_SUMMARY-lib$(PKG_NAME_REAL)-devel = $(PKG_SUMMARY-$(PKG_NAME_REAL)-devel) -PKG_FILES-lib$(PKG_NAME_REAL)-devel = $(PKG_FILES-$(PKG_NAME_REAL)-devel) blah -PKG_BUILD_DEPS-lib$(PKG_NAME_REAL)-devel = $(PKG_BUILD_DEPS-$(PKG_NAME_REAL)-devel) diff --git a/pkgs/__gmsl b/pkgs/__gmsl deleted file mode 100644 index 596ff1911..000000000 --- a/pkgs/__gmsl +++ /dev/null @@ -1,854 +0,0 @@ -# ---------------------------------------------------------------------------- -# -# GNU Make Standard Library (GMSL) -# -# A library of functions to be used with GNU Make's $(call) that -# provides functionality not available in standard GNU Make. -# -# Copyright (c) 2005-2007 John Graham-Cumming -# -# This file is part of GMSL -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# Neither the name of the John Graham-Cumming nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -# ---------------------------------------------------------------------------- - -# This is the GNU Make Standard Library version number as a list with -# three items: major, minor, revision - -gmsl_version := 1 0 11 - -# Used to output warnings and error from the library, it's possible to -# disable any warnings or errors by overriding these definitions -# manually or by setting GMSL_NO_WARNINGS or GMSL_NO_ERRORS - -__gmsl_name := GNU Make Standard Library -__gmsl_warning = $(warning $(__gmsl_name): $1) -__gmsl_error = $(error $(__gmsl_name): $1) - -ifdef GMSL_NO_WARNINGS -__gmsl_warning := -endif -ifdef GMSL_NO_ERRORS -__gmsl_error := -endif - -# If GMSL_TRACE is enabled then calls to the library functions are -# traced to stdout using warning messages with their arguments - -ifdef GMSL_TRACE -__gmsl_tr1 = $(warning $0('$1')) -__gmsl_tr2 = $(warning $0('$1','$2')) -__gmsl_tr3 = $(warning $0('$1','$2','$3')) -else -__gmsl_tr1 := -__gmsl_tr2 := -__gmsl_tr3 := -endif - -# Figure out whether we have $(eval) or not (GNU Make 3.80 and above) -# if we do not then output a warning message, if we do then some -# functions will be enabled. - -__gmsl_have_eval := $(false) -__gmsl_ignore := $(eval __gmsl_have_eval := $(true)) - -# If this is being run with Electric Cloud's emake then warn that -# their $(eval) support is incomplete. - -ifdef ECLOUD_BUILD_ID -$(warning You are using Electric Cloud's emake which has incomplete $$(eval) support) -__gmsl_have_eval := $(false) -endif - -# See if we have $(lastword) (GNU Make 3.81 and above) - -__gmsl_have_lastword := $(lastword $(false) $(true)) - -# See if we have native or and and (GNU Make 3.81 and above) - -__gmsl_have_or := $(if $(filter-out undefined, \ - $(origin or)),$(call or,$(true),$(false))) -__gmsl_have_and := $(if $(filter-out undefined, \ - $(origin and)),$(call and,$(true),$(true))) - -ifneq ($(__gmsl_have_eval),$(true)) -$(call __gmsl_warning,GNU Make $(MAKE_VERSION) does not support $$$$(eval): some functions disabled) -endif - -# ---------------------------------------------------------------------------- -# Function: gmsl_compatible -# Arguments: List containing the desired library version number (maj min rev) -# Returns: $(true) if this version of the library is compatible -# with the requested version number, otherwise $(false) -# ---------------------------------------------------------------------------- -gmsl_compatible = $(strip \ - $(if $(call gt,$(word 1,$1),$(word 1,$(gmsl_version))), \ - $(false), \ - $(if $(call lt,$(word 1,$1),$(word 1,$(gmsl_version))), \ - $(true), \ - $(if $(call gt,$(word 2,$1),$(word 2,$(gmsl_version))), \ - $(false), \ - $(if $(call lt,$(word 2,$1),$(word 2,$(gmsl_version))), \ - $(true), \ - $(call lte,$(word 3,$1),$(word 3,$(gmsl_version)))))))) - -# ########################################################################### -# LOGICAL OPERATORS -# ########################################################################### - -# not is defined in gmsl - -# ---------------------------------------------------------------------------- -# Function: and -# Arguments: Two boolean values -# Returns: Returns $(true) if both of the booleans are true -# ---------------------------------------------------------------------------- -ifneq ($(__gmsl_have_and),$(true)) -and = $(__gmsl_tr2)$(if $1,$(if $2,$(true),$(false)),$(false)) -endif - -# ---------------------------------------------------------------------------- -# Function: or -# Arguments: Two boolean values -# Returns: Returns $(true) if either of the booleans is true -# ---------------------------------------------------------------------------- -ifneq ($(__gmsl_have_or),$(true)) -or = $(__gmsl_tr2)$(if $1$2,$(true),$(false)) -endif - -# ---------------------------------------------------------------------------- -# Function: xor -# Arguments: Two boolean values -# Returns: Returns $(true) if exactly one of the booleans is true -# ---------------------------------------------------------------------------- -xor = $(__gmsl_tr2)$(if $1,$(if $2,$(false),$(true)),$(if $2,$(true),$(false))) - -# ---------------------------------------------------------------------------- -# Function: nand -# Arguments: Two boolean values -# Returns: Returns value of 'not and' -# ---------------------------------------------------------------------------- -nand = $(__gmsl_tr2)$(if $1,$(if $2,$(false),$(true)),$(true)) - -# ---------------------------------------------------------------------------- -# Function: nor -# Arguments: Two boolean values -# Returns: Returns value of 'not or' -# ---------------------------------------------------------------------------- -nor = $(__gmsl_tr2)$(if $1$2,$(false),$(true)) - -# ---------------------------------------------------------------------------- -# Function: xnor -# Arguments: Two boolean values -# Returns: Returns value of 'not xor' -# ---------------------------------------------------------------------------- -xnor =$(__gmsl_tr2)$(if $1,$(if $2,$(true),$(false)),$(if $2,$(false),$(true))) - -# ########################################################################### -# LIST MANIPULATION FUNCTIONS -# ########################################################################### - -# ---------------------------------------------------------------------------- -# Function: first (same as LISP's car, or head) -# Arguments: 1: A list -# Returns: Returns the first element of a list -# ---------------------------------------------------------------------------- -first = $(__gmsl_tr1)$(firstword $1) - -# ---------------------------------------------------------------------------- -# Function: last -# Arguments: 1: A list -# Returns: Returns the last element of a list -# ---------------------------------------------------------------------------- -ifeq ($(__gmsl_have_lastword),$(true)) -last = $(__gmsl_tr1)$(lastword $1) -else -last = $(__gmsl_tr1)$(if $1,$(word $(words $1),$1)) -endif - -# ---------------------------------------------------------------------------- -# Function: rest (same as LISP's cdr, or tail) -# Arguments: 1: A list -# Returns: Returns the list with the first element removed -# ---------------------------------------------------------------------------- -rest = $(__gmsl_tr1)$(wordlist 2,$(words $1),$1) - -# ---------------------------------------------------------------------------- -# Function: chop -# Arguments: 1: A list -# Returns: Returns the list with the last element removed -# ---------------------------------------------------------------------------- -chop = $(__gmsl_tr1)$(wordlist 2,$(words $1),x $1) - -# ---------------------------------------------------------------------------- -# Function: map -# Arguments: 1: Name of function to $(call) for each element of list -# 2: List to iterate over calling the function in 1 -# Returns: The list after calling the function on each element -# ---------------------------------------------------------------------------- -map = $(__gmsl_tr2)$(strip $(foreach a,$2,$(call $1,$a))) - -# ---------------------------------------------------------------------------- -# Function: pairmap -# Arguments: 1: Name of function to $(call) for each pair of elements -# 2: List to iterate over calling the function in 1 -# 3: Second list to iterate over calling the function in 1 -# Returns: The list after calling the function on each pair of elements -# ---------------------------------------------------------------------------- -pairmap = $(strip $(__gmsl_tr3)\ - $(if $2$3,$(call $1,$(call first,$2),$(call first,$3)) \ - $(call pairmap,$1,$(call rest,$2),$(call rest,$3)))) - -# ---------------------------------------------------------------------------- -# Function: leq -# Arguments: 1: A list to compare against... -# 2: ...this list -# Returns: Returns $(true) if the two lists are identical -# ---------------------------------------------------------------------------- -leq = $(__gmsl_tr2)$(strip $(if $(call seq,$(words $1),$(words $2)), \ - $(call __gmsl_list_equal,$1,$2),$(false))) - -__gmsl_list_equal = $(if $(strip $1), \ - $(if $(call seq,$(call first,$1),$(call first,$2)), \ - $(call __gmsl_list_equal, \ - $(call rest,$1), \ - $(call rest,$2)), \ - $(false)), \ - $(true)) - -# ---------------------------------------------------------------------------- -# Function: lne -# Arguments: 1: A list to compare against... -# 2: ...this list -# Returns: Returns $(true) if the two lists are different -# ---------------------------------------------------------------------------- -lne = $(__gmsl_tr2)$(call not,$(call leq,$1,$2)) - -# ---------------------------------------------------------------------------- -# Function: reverse -# Arguments: 1: A list to reverse -# Returns: The list with its elements in reverse order -# ---------------------------------------------------------------------------- -reverse =$(__gmsl_tr1)$(strip $(if $1,$(call reverse,$(call rest,$1)) \ - $(call first,$1))) - -# ---------------------------------------------------------------------------- -# Function: uniq -# Arguments: 1: A list from which to remove repeated elements -# Returns: The list with duplicate elements removed without reordering -# ---------------------------------------------------------------------------- -uniq = $(strip $(__gmsl_tr1)$(if $1,$(call uniq,$(call chop,$1)) \ - $(if $(filter $(call last,$1),$(call chop,$1)),,$(call last,$1)))) - -# ---------------------------------------------------------------------------- -# Function: length -# Arguments: 1: A list -# Returns: The number of elements in the list -# ---------------------------------------------------------------------------- -length = $(__gmsl_tr1)$(words $1) - -# ########################################################################### -# STRING MANIPULATION FUNCTIONS -# ########################################################################### - -# Helper function that translates any GNU Make 'true' value (i.e. a -# non-empty string) to our $(true) - -__gmsl_make_bool = $(if $(strip $1),$(true),$(false)) - -# ---------------------------------------------------------------------------- -# Function: seq -# Arguments: 1: A string to compare against... -# 2: ...this string -# Returns: Returns $(true) if the two strings are identical -# ---------------------------------------------------------------------------- -seq = $(__gmsl_tr2)$(if $(filter-out xx,x$(subst $1,,$2)$(subst $2,,$1)x),$(false),$(true)) - -# ---------------------------------------------------------------------------- -# Function: sne -# Arguments: 1: A string to compare against... -# 2: ...this string -# Returns: Returns $(true) if the two strings are not the same -# ---------------------------------------------------------------------------- -sne = $(__gmsl_tr2)$(call not,$(call seq,$1,$2)) - -# ---------------------------------------------------------------------------- -# Function: split -# Arguments: 1: The character to split on -# 2: A string to split -# Returns: Splits a string into a list separated by spaces at the split -# character in the first argument -# ---------------------------------------------------------------------------- -split = $(__gmsl_tr2)$(strip $(subst $1, ,$2)) - -# ---------------------------------------------------------------------------- -# Function: merge -# Arguments: 1: The character to put between fields -# 2: A list to merge into a string -# Returns: Merges a list into a single string, list elements are separated -# by the character in the first argument -# ---------------------------------------------------------------------------- -merge = $(__gmsl_tr2)$(strip $(if $2, \ - $(if $(call seq,1,$(words $2)), \ - $2,$(call first,$2)$1$(call merge,$1,$(call rest,$2))))) - -ifdef __gmsl_have_eval -# ---------------------------------------------------------------------------- -# Function: tr -# Arguments: 1: The list of characters to translate from -# 2: The list of characters to translate to -# 3: The text to translate -# Returns: Returns the text after translating characters -# ---------------------------------------------------------------------------- -tr = $(strip $(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3) \ - $(eval __gmsl_t := $3) \ - $(foreach c, \ - $(join $(addsuffix :,$1),$2), \ - $(eval __gmsl_t := \ - $(subst $(word 1,$(subst :, ,$c)),$(word 2,$(subst :, ,$c)), \ - $(__gmsl_t))))$(__gmsl_t)) - -# Common character classes for use with the tr function. Each of -# these is actually a variable declaration and must be wrapped with -# $() or ${} to be used. - -[A-Z] := A B C D E F G H I J K L M N O P Q R S T U V W X Y Z # -[a-z] := a b c d e f g h i j k l m n o p q r s t u v w x y z # -[0-9] := 0 1 2 3 4 5 6 7 8 9 # -[A-F] := A B C D E F # - -# ---------------------------------------------------------------------------- -# Function: uc -# Arguments: 1: Text to upper case -# Returns: Returns the text in upper case -# ---------------------------------------------------------------------------- -uc = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call tr,$([a-z]),$([A-Z]),$1) - -# ---------------------------------------------------------------------------- -# Function: lc -# Arguments: 1: Text to lower case -# Returns: Returns the text in lower case -# ---------------------------------------------------------------------------- -lc = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call tr,$([A-Z]),$([a-z]),$1) - -# ---------------------------------------------------------------------------- -# Function: strlen -# Arguments: 1: A string -# Returns: Returns the length of the string -# ---------------------------------------------------------------------------- -__gmsl_characters := A B C D E F G H I J K L M N O P Q R S T U V W X Y Z -__gmsl_characters += a b c d e f g h i j k l m n o p q r s t u v w x y z -__gmsl_characters += 0 1 2 3 4 5 6 7 8 9 -__gmsl_characters += ` ~ ! @ \# $$ % ^ & * ( ) - _ = + -__gmsl_characters += { } [ ] \ : ; ' " < > , . / ? | - -# Aside: if you read the above you might think that the lower-case -# letter x is missing, and that that's an error. It is missing, but -# it's not an error. __gmsl_characters is used by the strlen -# function. strlen works by transforming every character and space -# into the letter x and then counting the x's. Since there's no need -# to transform x into x I omitted it. - -# This results in __gmsl_space containing just a space - -__gmsl_space := -__gmsl_space += - -strlen = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(strip $(eval __temp := $(subst $(__gmsl_space),x,$1))$(foreach a,$(__gmsl_characters),$(eval __temp := $$(subst $$a,x,$(__temp))))$(eval __temp := $(subst x,x ,$(__temp)))$(words $(__temp))) - -# This results in __gmsl_newline containing just a newline - -define __gmsl_newline - - -endef - -# This results in __gmsl_tab containing a tab - -__gmsl_tab := # - -# ---------------------------------------------------------------------------- -# Function: substr -# Arguments: 1: A string -# 2: Start position (first character is 1) -# 3: End position (inclusive) -# Returns: A substring. -# Note: The string in $1 must not contain a § -# ---------------------------------------------------------------------------- - -substr = $(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3)$(strip $(eval __temp := $$(subst $$(__gmsl_space),§ ,$$1))$(foreach a,$(__gmsl_characters),$(eval __temp := $$(subst $$a,$$a$$(__gmsl_space),$(__temp))))$(eval __temp := $(wordlist $2,$3,$(__temp))))$(subst §,$(__gmsl_space),$(subst $(__gmsl_space),,$(__temp))) - -endif # __gmsl_have_eval - -# ########################################################################### -# SET MANIPULATION FUNCTIONS -# ########################################################################### - -# Sets are represented by sorted, deduplicated lists. To create a set -# from a list use set_create, or start with the empty_set and -# set_insert individual elements - -# This is the empty set -empty_set := - -# ---------------------------------------------------------------------------- -# Function: set_create -# Arguments: 1: A list of set elements -# Returns: Returns the newly created set -# ---------------------------------------------------------------------------- -set_create = $(__gmsl_tr1)$(sort $1) - -# ---------------------------------------------------------------------------- -# Function: set_insert -# Arguments: 1: A single element to add to a set -# 2: A set -# Returns: Returns the set with the element added -# ---------------------------------------------------------------------------- -set_insert = $(__gmsl_tr2)$(sort $1 $2) - -# ---------------------------------------------------------------------------- -# Function: set_remove -# Arguments: 1: A single element to remove from a set -# 2: A set -# Returns: Returns the set with the element removed -# ---------------------------------------------------------------------------- -set_remove = $(__gmsl_tr2)$(filter-out $1,$2) - -# ---------------------------------------------------------------------------- -# Function: set_is_member -# Arguments: 1: A single element -# 2: A set -# Returns: Returns $(true) if the element is in the set -# ---------------------------------------------------------------------------- -set_is_member = $(__gmsl_tr2)$(if $(filter $1,$2),$(true),$(false)) - -# ---------------------------------------------------------------------------- -# Function: set_union -# Arguments: 1: A set -# 2: Another set -# Returns: Returns the union of the two sets -# ---------------------------------------------------------------------------- -set_union = $(__gmsl_tr2)$(sort $1 $2) - -# ---------------------------------------------------------------------------- -# Function: set_intersection -# Arguments: 1: A set -# 2: Another set -# Returns: Returns the intersection of the two sets -# ---------------------------------------------------------------------------- -set_intersection = $(__gmsl_tr2)$(filter $1,$2) - -# ---------------------------------------------------------------------------- -# Function: set_is_subset -# Arguments: 1: A set -# 2: Another set -# Returns: Returns $(true) if the first set is a subset of the second -# ---------------------------------------------------------------------------- -set_is_subset = $(__gmsl_tr2)$(call set_equal,$(call set_intersection,$1,$2),$1) - -# ---------------------------------------------------------------------------- -# Function: set_equal -# Arguments: 1: A set -# 2: Another set -# Returns: Returns $(true) if the two sets are identical -# ---------------------------------------------------------------------------- -set_equal = $(__gmsl_tr2)$(call seq,$1,$2) - -# ########################################################################### -# ARITHMETIC LIBRARY -# ########################################################################### - -# Integers a represented by lists with the equivalent number of x's. -# For example the number 4 is x x x x. The maximum integer that the -# library can handle as _input_ is __gmsl_input_int which is defined -# here as 65536 - -__gmsl_sixteen := x x x x x x x x x x x x x x x x -__gmsl_input_int := $(foreach a,$(__gmsl_sixteen), \ - $(foreach b,$(__gmsl_sixteen), \ - $(foreach c,$(__gmsl_sixteen), \ - $(__gmsl_sixteen))))) - -# ---------------------------------------------------------------------------- -# Function: int_decode -# Arguments: 1: A number of x's representation -# Returns: Returns the integer for human consumption that is represented -# by the string of x's -# ---------------------------------------------------------------------------- -int_decode = $(__gmsl_tr1)$(words $1) - -# ---------------------------------------------------------------------------- -# Function: int_encode -# Arguments: 1: A number in human-readable integer form -# Returns: Returns the integer encoded as a string of x's -# ---------------------------------------------------------------------------- -int_encode = $(__gmsl_tr1)$(wordlist 1,$1,$(__gmsl_input_int)) - -# The arithmetic library functions come in two forms: one form of each -# function takes integers as arguments and the other form takes the -# encoded form (x's created by a call to int_encode). For example, -# there are two plus functions: -# -# plus Called with integer arguments and returns an integer -# int_plus Called with encoded arguments and returns an encoded result -# -# plus will be slower than int_plus because its arguments and result -# have to be translated between the x's format and integers. If doing -# a complex calculation use the int_* forms with a single encoding of -# inputs and single decoding of the output. For simple calculations -# the direct forms can be used. - -# Helper function used to wrap an int_* function into a function that -# takes a pair of integers, perhaps a function and returns an integer -# result -__gmsl_int_wrap = $(call int_decode,$(call $1,$(call int_encode,$2),$(call int_encode,$3))) -__gmsl_int_wrap1 = $(call int_decode,$(call $1,$(call int_encode,$2))) -__gmsl_int_wrap2 = $(call $1,$(call int_encode,$2),$(call int_encode,$3)) - -# ---------------------------------------------------------------------------- -# Function: int_plus -# Arguments: 1: A number in x's representation -# 2: Another number in x's represntation -# Returns: Returns the sum of the two numbers in x's representation -# ---------------------------------------------------------------------------- -int_plus = $(strip $(__gmsl_tr2)$1 $2) - -# ---------------------------------------------------------------------------- -# Function: plus (wrapped version of int_plus) -# Arguments: 1: An integer -# 2: Another integer -# Returns: Returns the sum of the two integers -# ---------------------------------------------------------------------------- -plus = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_plus,$1,$2) - -# ---------------------------------------------------------------------------- -# Function: int_subtract -# Arguments: 1: A number in x's representation -# 2: Another number in x's represntation -# Returns: Returns the difference of the two numbers in x's representation, -# or outputs an error on a numeric underflow -# ---------------------------------------------------------------------------- -int_subtract = $(strip $(__gmsl_tr2)$(if $(call int_gte,$1,$2), \ - $(filter-out xx,$(join $1,$2)), \ - $(call __gmsl_warning,Subtraction underflow))) - -# ---------------------------------------------------------------------------- -# Function: subtract (wrapped version of int_subtract) -# Arguments: 1: An integer -# 2: Another integer -# Returns: Returns the difference of the two integers, -# or outputs an error on a numeric underflow -# ---------------------------------------------------------------------------- -subtract = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_subtract,$1,$2) - -# ---------------------------------------------------------------------------- -# Function: int_multiply -# Arguments: 1: A number in x's representation -# 2: Another number in x's represntation -# Returns: Returns the product of the two numbers in x's representation -# ---------------------------------------------------------------------------- -int_multiply = $(strip $(__gmsl_tr2)$(foreach a,$1,$2)) - -# ---------------------------------------------------------------------------- -# Function: multiply (wrapped version of int_multiply) -# Arguments: 1: An integer -# 2: Another integer -# Returns: Returns the product of the two integers -# ---------------------------------------------------------------------------- -multiply = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_multiply,$1,$2) - -# ---------------------------------------------------------------------------- -# Function: int_divide -# Arguments: 1: A number in x's representation -# 2: Another number in x's represntation -# Returns: Returns the result of integer division of argument 1 divided -# by argument 2 in x's representation -# ---------------------------------------------------------------------------- -int_divide = $(__gmsl_tr2)$(strip $(if $2, \ - $(if $(call int_gte,$1,$2), \ - x $(call int_divide,$(call int_subtract,$1,$2),$2),), \ - $(call __gmsl_error,Division by zero))) - -# ---------------------------------------------------------------------------- -# Function: divide (wrapped version of int_divide) -# Arguments: 1: An integer -# 2: Another integer -# Returns: Returns the integer division of the first argument by the second -# ---------------------------------------------------------------------------- -divide = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_divide,$1,$2) - -# ---------------------------------------------------------------------------- -# Function: int_max, int_min -# Arguments: 1: A number in x's representation -# 2: Another number in x's represntation -# Returns: Returns the maximum or minimum of its arguments in x's -# representation -# ---------------------------------------------------------------------------- -int_max = $(__gmsl_tr2)$(subst xx,x,$(join $1,$2)) -int_min = $(__gmsl_tr2)$(subst xx,x,$(filter xx,$(join $1,$2))) - -# ---------------------------------------------------------------------------- -# Function: max, min -# Arguments: 1: An integer -# 2: Another integer -# Returns: Returns the maximum or minimum of its integer arguments -# ---------------------------------------------------------------------------- -max = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_max,$1,$2) -min = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_min,$1,$2) - -# ---------------------------------------------------------------------------- -# Function: int_gt, int_gte, int_lt, int_lte, int_eq, int_ne -# Arguments: Two x's representation numbers to be compared -# Returns: $(true) or $(false) -# -# int_gt First argument greater than second argument -# int_gte First argument greater than or equal to second argument -# int_lt First argument less than second argument -# int_lte First argument less than or equal to second argument -# int_eq First argument is numerically equal to the second argument -# int_ne First argument is not numerically equal to the second argument -# ---------------------------------------------------------------------------- -int_gt = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(filter-out $(words $2), \ - $(words $(call int_max,$1,$2)))) -int_gte = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(call int_gt,$1,$2)$(call int_eq,$1,$2)) -int_lt = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(filter-out $(words $1), \ - $(words $(call int_max,$1,$2)))) -int_lte = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(call int_lt,$1,$2)$(call int_eq,$1,$2)) -int_eq = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(filter $(words $1),$(words $2))) -int_ne = $(__gmsl_tr2)$(call __gmsl_make_bool, \ - $(filter-out $(words $1),$(words $2))) - -# ---------------------------------------------------------------------------- -# Function: gt, gte, lt, lte, eq, ne -# Arguments: Two integers to be compared -# Returns: $(true) or $(false) -# -# gt First argument greater than second argument -# gte First argument greater than or equal to second argument -# lt First argument less than second argument -# lte First argument less than or equal to second argument -# eq First argument is numerically equal to the second argument -# ne First argument is not numerically equal to the second argument -# ---------------------------------------------------------------------------- -gt = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_gt,$1,$2) -gte = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_gte,$1,$2) -lt = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_lt,$1,$2) -lte = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_lte,$1,$2) -eq = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_eq,$1,$2) -ne = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_ne,$1,$2) - -# increment adds 1 to its argument, decrement subtracts 1. Note that -# decrement does not range check and hence will not underflow, but -# will incorrectly say that 0 - 1 = 0 - -# ---------------------------------------------------------------------------- -# Function: int_inc -# Arguments: 1: A number in x's representation -# Returns: The number incremented by 1 in x's representation -# ---------------------------------------------------------------------------- -int_inc = $(strip $(__gmsl_tr1)$1 x) - -# ---------------------------------------------------------------------------- -# Function: inc -# Arguments: 1: An integer -# Returns: The argument incremented by 1 -# ---------------------------------------------------------------------------- -inc = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_inc,$1) - -# ---------------------------------------------------------------------------- -# Function: int_dec -# Arguments: 1: A number in x's representation -# Returns: The number decremented by 1 in x's representation -# ---------------------------------------------------------------------------- -int_dec = $(__gmsl_tr1)$(strip $(if $(call sne,0,$(words $1)), \ - $(wordlist 2,$(words $1),$1), \ - $(call __gmsl_warning,Decrement underflow))) - -# ---------------------------------------------------------------------------- -# Function: dec -# Arguments: 1: An integer -# Returns: The argument decremented by 1 -# ---------------------------------------------------------------------------- -dec = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_dec,$1) - -# double doubles its argument, and halve halves it - -# ---------------------------------------------------------------------------- -# Function: int_double -# Arguments: 1: A number in x's representation -# Returns: The number doubled (i.e. * 2) and returned in x's representation -# ---------------------------------------------------------------------------- -int_double = $(strip $(__gmsl_tr1)$1 $1) - -# ---------------------------------------------------------------------------- -# Function: double -# Arguments: 1: An integer -# Returns: The integer times 2 -# ---------------------------------------------------------------------------- -double = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_double,$1) - -# ---------------------------------------------------------------------------- -# Function: int_halve -# Arguments: 1: A number in x's representation -# Returns: The number halved (i.e. / 2) and returned in x's representation -# ---------------------------------------------------------------------------- -int_halve = $(__gmsl_tr1)$(strip $(subst xx,x,$(filter-out xy x y, \ - $(join $1,$(foreach a,$1,y x))))) - -# ---------------------------------------------------------------------------- -# Function: halve -# Arguments: 1: An integer -# Returns: The integer divided by 2 -# ---------------------------------------------------------------------------- -halve = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_halve,$1) - -ifdef __gmsl_have_eval -# ########################################################################### -# ASSOCIATIVE ARRAYS -# ########################################################################### - -# ---------------------------------------------------------------------------- -# Function: set -# Arguments: 1: Name of associative array -# 2: The key value to associate -# 3: The value associated with the key -# Returns: None -# ---------------------------------------------------------------------------- -set = $(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3)$(eval __gmsl_aa_$1_$2 = $3) - -# ---------------------------------------------------------------------------- -# Function: get -# Arguments: 1: Name of associative array -# 2: The key to retrieve -# Returns: The value stored in the array for that key -# ---------------------------------------------------------------------------- -get = $(strip $(__gmsl_tr2)$(call assert_no_dollar,$0,$1$2)$(if $(filter-out undefined,$(origin __gmsl_aa_$1_$2)), \ - $(__gmsl_aa_$1_$2))) - -# ---------------------------------------------------------------------------- -# Function: keys -# Arguments: 1: Name of associative array -# Returns: Returns a list of all defined keys in the array -# ---------------------------------------------------------------------------- -keys = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(sort $(patsubst __gmsl_aa_$1_%,%, \ - $(filter __gmsl_aa_$1_%,$(.VARIABLES)))) - -# ---------------------------------------------------------------------------- -# Function: defined -# Arguments: 1: Name of associative array -# 2: The key to test -# Returns: Returns true if the key is defined (i.e. not empty) -# ---------------------------------------------------------------------------- -defined = $(__gmsl_tr2)$(call assert_no_dollar,$0,$1$2)$(call sne,$(call get,$1,$2),) - -endif # __gmsl_have_eval - -ifdef __gmsl_have_eval -# ########################################################################### -# NAMED STACKS -# ########################################################################### - -# ---------------------------------------------------------------------------- -# Function: push -# Arguments: 1: Name of stack -# 2: Value to push onto the top of the stack (must not contain -# a space) -# Returns: None -# ---------------------------------------------------------------------------- -push = $(__gmsl_tr2)$(call assert_no_dollar,$0,$1$2)$(eval __gmsl_stack_$1 := $2 $(if $(filter-out undefined,\ - $(origin __gmsl_stack_$1)),$(__gmsl_stack_$1))) - -# ---------------------------------------------------------------------------- -# Function: pop -# Arguments: 1: Name of stack -# Returns: Top element from the stack after removing it -# ---------------------------------------------------------------------------- -pop = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(strip $(if $(filter-out undefined,$(origin __gmsl_stack_$1)), \ - $(call first,$(__gmsl_stack_$1)) \ - $(eval __gmsl_stack_$1 := $(call rest,$(__gmsl_stack_$1))))) - -# ---------------------------------------------------------------------------- -# Function: peek -# Arguments: 1: Name of stack -# Returns: Top element from the stack without removing it -# ---------------------------------------------------------------------------- -peek = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call first,$(__gmsl_stack_$1)) - -# ---------------------------------------------------------------------------- -# Function: depth -# Arguments: 1: Name of stack -# Returns: Number of items on the stack -# ---------------------------------------------------------------------------- -depth = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(words $(__gmsl_stack_$1)) - -endif # __gmsl_have_eval - -# ########################################################################### -# DEBUGGING FACILITIES -# ########################################################################### - -# ---------------------------------------------------------------------------- -# Target: gmsl-print-% -# Arguments: The % should be replaced by the name of a variable that you -# wish to print out. -# Action: Echos the name of the variable that matches the % and its value. -# For example, 'make gmsl-print-SHELL' will output the value of -# the SHELL variable -# ---------------------------------------------------------------------------- -gmsl-print-%: ; @echo $* = $($*) - -# ---------------------------------------------------------------------------- -# Function: assert -# Arguments: 1: A boolean that must be true or the assertion will fail -# 2: The message to print with the assertion -# Returns: None -# ---------------------------------------------------------------------------- -assert = $(if $1,,$(call __gmsl_error,Assertion failure: $2)) - -# ---------------------------------------------------------------------------- -# Function: assert_exists -# Arguments: 1: Name of file that must exist, if it is missing an assertion -# will be generated -# Returns: None -# ---------------------------------------------------------------------------- -assert_exists = $(call assert,$(wildcard $1),file '$1' missing) - -# ---------------------------------------------------------------------------- -# Function: assert_no_dollar -# Arguments: 1: Name of a function being executd -# 2: Arguments to check -# Returns: None -# ---------------------------------------------------------------------------- -assert_no_dollar = $(call assert,$(call not,$(findstring $$,$2)),$1 called with a dollar sign in argument) diff --git a/pkgs/gmsl b/pkgs/gmsl deleted file mode 100644 index 2ff28972d..000000000 --- a/pkgs/gmsl +++ /dev/null @@ -1,89 +0,0 @@ -# ---------------------------------------------------------------------------- -# -# GNU Make Standard Library (GMSL) -# -# A library of functions to be used with GNU Make's $(call) that -# provides functionality not available in standard GNU Make. -# -# Copyright (c) 2005-2008 John Graham-Cumming -# -# This file is part of GMSL -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# Neither the name of the John Graham-Cumming nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -# ---------------------------------------------------------------------------- - -# Determine if the library has already been included and if so don't -# bother including it again - -ifndef __gmsl_included - -# Standard definitions for true and false. true is any non-empty -# string, false is an empty string. These are intended for use with -# $(if). - -true := T -false := - -# ---------------------------------------------------------------------------- -# Function: not -# Arguments: 1: A boolean value -# Returns: Returns the opposite of the arg. (true -> false, false -> true) -# ---------------------------------------------------------------------------- -not = $(if $1,$(false),$(true)) - -# Prevent reinclusion of the library - -__gmsl_included := $(true) - -# Try to determine where this file is located. If the caller did -# include /foo/gmsl then extract the /foo/ so that __gmsl gets -# included transparently - -ifneq ($(MAKEFILE_LIST),) -__gmsl_root := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)) - -# If there are any spaces in the path in __gmsl_root then give up - -ifeq (1,$(words $(__gmsl_root))) -__gmsl_root := $(patsubst %gmsl,%,$(__gmsl_root)) -else -__gmsl_root := -endif - -include $(__gmsl_root)__gmsl - -else - -include __gmsl - -endif - -endif # __gmsl_included - diff --git a/src/bootloader/boot.msg b/src/bootloader/boot.msg deleted file mode 100644 index de246387c..000000000 --- a/src/bootloader/boot.msg +++ /dev/null @@ -1,10 +0,0 @@ - -splash.lss - - - - - To install IPFire on your system, type 04install07 and - press the 0407 key. - - Use the function keys listed below for more information. - -0f[F1-Main] [F2-Options]07 diff --git a/src/bootloader/installer.conf b/src/bootloader/installer.conf deleted file mode 100644 index bfb6735bf..000000000 --- a/src/bootloader/installer.conf +++ /dev/null @@ -1,37 +0,0 @@ - -DEFAULT vesamenu.c32 -#prompt 1 -TIMEOUT 100 - -DISPLAY boot.msg -F1 boot.msg -F2 options.msg - -MENU BACKGROUND splash.jpg -MENU TITLE Welcome to @NAME@ @VERSION@! -MENU COLOR border 0 #ffffffff #00000000 -MENU COLOR sel 7 #ffffffff #ff000000 -MENU COLOR title 0 #ffffffff #00000000 -MENU COLOR tabmsg 0 #ffffffff #00000000 -MENU COLOR unsel 0 #ffffffff #00000000 -MENU COLOR hotsel 0 #ff000000 #ffffffff -MENU COLOR hotkey 7 #ffffffff #ff000000 -MENU hidden -MENU hiddenrow 5 - -LABEL install - MENU label ^Install a new @NAME@ system - MENU default - KERNEL vmlinuz0 - APPEND initrd=initrd0 root=live:CDLABEL=@NAME@_@VERSION@ rootfstype=auto liveimg mode=install quiet -label rescue - MENU label ^Rescue installed @NAME@ system - KERNEL vmlinuz0 - APPEND initrd=initrd0 root=live:CDLABEL=@NAME@_@VERSION@ rootfstype=auto liveimg mode=rescue quiet -label local - MENU label Boot from ^local drive - LOCALBOOT 0xffff -label memtest - MENU label ^Memory test - KERNEL memtest - APPEND - diff --git a/src/bootloader/options.msg b/src/bootloader/options.msg deleted file mode 100644 index c92169caa..000000000 --- a/src/bootloader/options.msg +++ /dev/null @@ -1,30 +0,0 @@ -00 - - - 0fInstaller Boot Options07 - - - To run a rescue system, type: 04rescue 07. - - - To test the memory in your system type: 04memtest 07. - - - - - - - - - - - - - - - - - - - - - -0f[F1-Main] [F2-Options]07 diff --git a/src/bootloader/pxe.conf b/src/bootloader/pxe.conf deleted file mode 100644 index 97d32df3a..000000000 --- a/src/bootloader/pxe.conf +++ /dev/null @@ -1,30 +0,0 @@ - -DEFAULT @SNAME@/vesamenu.c32 -#prompt 1 -TIMEOUT 100 - -DISPLAY @SNAME@/boot.msg -F1 @SNAME@/boot.msg -F2 @SNAME@/options.msg - -MENU BACKGROUND @SNAME@/splash.jpg -MENU TITLE Welcome to @NAME@ @VERSION@! -MENU COLOR border 0 #ffffffff #00000000 -MENU COLOR sel 7 #ffffffff #ff000000 -MENU COLOR title 0 #ffffffff #00000000 -MENU COLOR tabmsg 0 #ffffffff #00000000 -MENU COLOR unsel 0 #ffffffff #00000000 -MENU COLOR hotsel 0 #ff000000 #ffffffff -MENU COLOR hotkey 7 #ffffffff #ff000000 -MENU hidden -MENU hiddenrow 5 - -LABEL install - MENU label ^Install a new @NAME@ system - MENU default - KERNEL @SNAME@/@SNAME@0 - APPEND initrd=@SNAME@/initrd0 root=http://pxe.ipfire.org/@IMAGENAME@.iso rootfstype=iso9660 vga=791 splash mode=install quiet ro net=dhcp -label rescue - MENU label ^Rescue installed @NAME@ system - KERNEL @SNAME@/@SNAME@0 - APPEND initrd=@SNAME@/initrd0 root=http://pxe.ipfire.org/@IMAGENAME@.iso rootfstype=iso9660 vga=791 splash mode=rescue quiet ro net=dhcp diff --git a/src/bootloader/splash.jpg b/src/bootloader/splash.jpg deleted file mode 100644 index d1d17701d..000000000 Binary files a/src/bootloader/splash.jpg and /dev/null differ diff --git a/src/bootloader/splash.lss b/src/bootloader/splash.lss deleted file mode 100644 index 0ab1f9bac..000000000 Binary files a/src/bootloader/splash.lss and /dev/null differ diff --git a/src/grsecurity/sysctl.conf b/src/grsecurity/sysctl.conf deleted file mode 100644 index a9780214c..000000000 --- a/src/grsecurity/sysctl.conf +++ /dev/null @@ -1,6 +0,0 @@ -# Begin /etc/grsec/sysctl.conf - -# Locking all settings - must be the last line -kernel.grsecurity.grsec_lock = 1 - -# End /etc/grsec/sysctl.conf diff --git a/src/install/etc/fstab b/src/install/etc/fstab deleted file mode 100644 index f8eebefb0..000000000 --- a/src/install/etc/fstab +++ /dev/null @@ -1,8 +0,0 @@ -# -# file system mount-point type options dump fsck -# order -none / auto defaults 0 0 -proc /proc proc defaults 0 0 -sysfs /sys sysfs defaults 0 0 -devpts /dev/pts devpts gid=4,mode=620 0 0 -shm /dev/shm tmpfs defaults 0 0 diff --git a/src/install/etc/group b/src/install/etc/group deleted file mode 100644 index ef7dd4589..000000000 --- a/src/install/etc/group +++ /dev/null @@ -1,33 +0,0 @@ -root:x:0:root -bin:x:1:bin,daemon,root -daemon:x:2:bin,daemon,root -sys:x:3:bin,adm,root -adm:x:4:adm,daemon,root -tty:x:5: -disk:x:6:root -lp:x:7:daemon,lp -mem:x:8: -kmem:x:9: -wheel:x:10:root -mail:x:12:mail -news:x:13:news -uucp:x:14:uucp -man:x:15: -games:x:20: -gopher:x:30: -dip:x:40: -ftp:x:50: -lock:x:54: -nobody:x:99: -users:x:100: -dbus:x:81: -utmp:x:22: -smmsp:x:51: -nscd:x:28: -floppy:x:19: -rpc:x:32: -sshd:x:74: -pcap:x:77: -utempter:x:35: -ntp:x:38: -audio:x:63: diff --git a/src/install/etc/grsec/sysctl.conf b/src/install/etc/grsec/sysctl.conf deleted file mode 100644 index f3fbde3e3..000000000 --- a/src/install/etc/grsec/sysctl.conf +++ /dev/null @@ -1,12 +0,0 @@ -# Begin /etc/grsec/sysctl.conf - -# Disable chroot caps -kernel.grsecurity.chroot_caps = 0 - -# Disable chroot exec logging -kernel.grsecurity.chroot_execlog = 0 - -# Locking all settings - must be the last line -kernel.grsecurity.grsec_lock = 1 - -# End /etc/grsec/sysctl.conf diff --git a/src/install/etc/passwd b/src/install/etc/passwd deleted file mode 100644 index eb141ce84..000000000 --- a/src/install/etc/passwd +++ /dev/null @@ -1,22 +0,0 @@ -root:x:0:0:root:/root:/bin/bash -bin:x:1:1:bin:/bin:/sbin/nologin -daemon:x:2:2:daemon:/sbin:/sbin/nologin -adm:x:3:4:adm:/var/adm:/sbin/nologin -lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin -sync:x:5:0:sync:/sbin:/bin/sync -shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown -halt:x:7:0:halt:/sbin:/sbin/halt -mail:x:8:12:mail:/var/spool/mail:/sbin/nologin -news:x:9:13:news:/etc/news: -uucp:x:10:14:uucp:/var/spool/uucp:/sbin/nologin -operator:x:11:0:operator:/root:/sbin/nologin -games:x:12:100:games:/usr/games:/sbin/nologin -gopher:x:13:30:gopher:/var/gopher:/sbin/nologin -ftp:x:14:50:FTP User:/var/ftp:/sbin/nologin -nobody:x:99:99:Nobody:/:/sbin/nologin -dbus:x:81:81:System message bus:/:/sbin/nologin -smmsp:x:51:51::/var/spool/mqueue:/sbin/nologin -nscd:x:28:28:NSCD Daemon:/:/sbin/nologin -rpc:x:32:32:Portmapper RPC user:/:/sbin/nologin -sshd:x:74:74:Privilege-separated SSH:/var/empty/sshd:/sbin/nologin -ntp:x:38:38::/etc/ntp:/sbin/nologin diff --git a/src/install/fastboot b/src/install/fastboot deleted file mode 100644 index 265bd2ccc..000000000 --- a/src/install/fastboot +++ /dev/null @@ -1 +0,0 @@ -# Just a file to skip the checkfs procdure on the live system diff --git a/src/logrotate/freeradius-server b/src/logrotate/freeradius-server deleted file mode 100644 index 8c5c6fbba..000000000 --- a/src/logrotate/freeradius-server +++ /dev/null @@ -1,56 +0,0 @@ -# You can use this to rotate the /var/log/radius/* files, simply copy -# it to /etc/logrotate.d/radiusd - -# There are different detail-rotating strategies you can use. One is -# to write to a single detail file per IP and use the rotate config -# below. Another is to write to a daily detail file per IP with: -# detailfile = ${radacctdir}/%{Client-IP-Address}/%Y%m%d-detail -# (or similar) in radiusd.conf, without rotation. If you go with the -# second technique, you will need another cron job that removes old -# detail files. You do not need to comment out the below for method #2. -/var/log/radius/radacct/*/detail { - monthly - rotate 4 - nocreate - missingok - compress -} - -/var/log/radius/checkrad.log { - monthly - rotate 4 - create - missingok - compress -} - -/var/log/radius/radius.log { - monthly - rotate 4 - create - missingok - compress -} - -/var/log/radius/radutmp { - monthly - rotate 4 - create - compress - missingok -} - -/var/log/radius/radwtmp { - monthly - rotate 4 - create - compress - missingok -} -/var/log/radius/sqltrace.sql { - monthly - rotate 4 - create - compress - missingok -} diff --git a/src/logrotate/ppp.logrotate b/src/logrotate/ppp.logrotate deleted file mode 100644 index 7a72979ca..000000000 --- a/src/logrotate/ppp.logrotate +++ /dev/null @@ -1,10 +0,0 @@ -# Logrotate file for ppp RPM - -/var/log/ppp/connect-errors { - missingok - compress - notifempty - daily - rotate 5 - create 0600 root root -} diff --git a/tools/chroot-shell b/tools/chroot-shell deleted file mode 100755 index 48833ef32..000000000 --- a/tools/chroot-shell +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -cat <. # -# # -############################################################################### - -FILES= -ROOT=/ - -while [ $# -gt 0 ]; do - case "$1" in - --root=*) - ROOT=${1#--root=} - ;; - *.ipk) - file=$1 - [ ${file:0:1} != / ] && file="$(pwd)/$file" - if [ -e "$file" ]; then - FILES="$FILES $file" - else - echo "File does not exist: $file" >&2 - #exit 1 - fi - ;; - esac - shift -done - -if [ "$ROOT" != "/" ]; then - [ -d "$ROOT" ] || mkdir -p $ROOT - cd $ROOT - if [ "$(pwd)" != "$ROOT" ]; then - echo "Could not change to root: $ROOT" >&2 - exit 1 - fi -fi - -# Sort all packages -FILES=$(for i in $FILES; do echo $i; done | sort -u) - -for file in $FILES; do - #echo "Extracting $(basename ${file})..." - cpio --extract --quiet -H newc --to-stdout data.img < $file | \ - tar --extract --use-compress-program=xz -C ${ROOT} -done diff --git a/tools/functions-common b/tools/functions-common deleted file mode 100644 index 4c44a58aa..000000000 --- a/tools/functions-common +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# Simply import all files from this directory that -# begin with functions-*. - -BASEDIR=$(dirname ${BASH_SOURCE[0]}) - -for file in ${BASEDIR}/functions-*; do - # Avoid infinite loop when importing this file again - [ "$(basename ${file})" = "functions-common" ] && continue - - . ${file} -done - diff --git a/tools/functions-constants b/tools/functions-constants deleted file mode 100644 index 396e045ed..000000000 --- a/tools/functions-constants +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Debugging mode for these scripts -DEBUG=0 - -# Interpreters that should not be found by find_interpreters() -INTERPRETERS_TO_BE_SKIPPED="/usr/bin/env" - -# Some path constants... -LIBRARY_PATHS="/lib /usr/lib /libexec /usr/libexec" -BINARY_PATHS="${LIBRARY_PATHS} /bin /sbin /usr/bin /usr/sbin" - -# List of directories that could probably empty and are removed automatically -# so they won't appear in any package. -ORPHAN_CANDIDATES="${BINARY_PATHS} /usr /usr/include /usr/share" -for i in $(seq 0 9); do - ORPHAN_CANDIDATES="${ORPHAN_CANDIDATES} /usr/share/man/man${i}" -done -ORPHAN_CANDIDATES="${ORPHAN_CANDIDATES} /usr/lib/pkgconfig" - diff --git a/tools/functions-directories b/tools/functions-directories deleted file mode 100644 index 5b0867b7a..000000000 --- a/tools/functions-directories +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -function dir_is_empty() { - [ "$(ls -A $@ 2>/dev/null | wc -l)" = "0" ] -} - -function directory_remove_orphans() { - local basedir=${1} - - log DEBUG "Removing orphans in ${basedir}" - - local dir - for dir in ${ORPHAN_CANDIDATES}; do - dir="${basedir}/${dir}" - - [ -d "${dir}" ] || continue - - if dir_is_empty ${dir}; then - log DEBUG " Found orphaned directory: ${dir}" - rm -rf ${dir} - fi - done -} - diff --git a/tools/functions-files b/tools/functions-files deleted file mode 100644 index 41ce3be0c..000000000 --- a/tools/functions-files +++ /dev/null @@ -1,195 +0,0 @@ -#!/bin/bash - -# Check if a file is an ELF binary -# -function file_is_elf() { - local file=${1} - - file "${file}" | grep -q "ELF" -} - -# Check if a file is a script. -# If the first line starts with #! this is sufficient. -# -function file_is_script() { - local file=${1} - - local first_line=$(head -n1 ${file}) - - [ "${first_line:0:2}" = "#!" ] -} - -# Get the interpreter of a file. -# -function file_get_interpreter() { - local file=${1} - - if file_is_elf ${file}; then - _file_get_elf_interpreter ${file} - elif file_is_script ${file}; then - _file_get_script_interpreter ${file} - fi -} - -# Hidden function that gets the interpreter from an ELF file. -# -function _file_get_elf_interpreter() { - local file=${1} - - readelf -l ${file} | grep "program interpreter" | \ - tr -d "]" | awk '{ print $NF }' -} - -# Hidden fucntion that gets the interpreter from a script file. -# -function _file_get_script_interpreter() { - local file=${1} - - # If the file is not executeable, no interpreter will be needed - [ -x "${file}" ] || return - - local first_line=$(head -n1 ${file}) - - first_line="${first_line:2:${#first_line}}" - - # Choose the first argument and strip any parameters if available - local interpreter - for interpreter in ${first_line}; do - echo "${interpreter}" - return - done -} - -# Check if a file is statically linked. -# -function file_is_static() { - local file=${1} - - file ${file} | grep -q "statically linked" -} - -# Get NEEDED from a file. -# -function file_get_needed() { - local file=${1} - - readelf -d ${file} | grep NEEDED | \ - tr -d "[]" | awk '{ print $NF }' -} - -# Get RPATH from a file. -# -function file_get_rpath() { - local file=${1} - - readelf -d ${file} | grep RPATH | \ - tr -d "[]" | awk '{ print $NF }' -} - -# Get SONAME from a file. -# -function file_get_soname() { - local file=${1} - - local file_basename=$(basename ${file}) - if [ "${file_basename:0:3}" = "ld-" ]; then - log DEBUG "Don't return a SONAME for linkers: ${file}" - return - fi - - readelf -d ${file} | grep SONAME | \ - tr -d "[]" | awk '{ print $NF }' -} - -# Check if a file is a shared object. -# -function file_is_shared_object() { - local file=${1} - - file ${file} | grep -q "shared object" -} - -# Check if a file has the canary. -# -function file_has_canary() { - local file=${1} - - readelf -s ${file} | grep -q "__stack_chk_fail" -} - -# Check if a file has an executeable stack. -# -function file_has_execstack() { - local file=${1} - - readelf -h ${file} | grep -qE "Type:[[:space:]]*EXEC" -} - -# Check if a file has NX. -# -function file_has_nx() { - local file=${1} - - readelf -l ${file} | grep "GNU_STACK" | grep -q "RWE" - [ $? != 0 ] -} - -# Check if a file is partly RELRO. -# -function file_is_relro_partly() { - local file=${1} - - readelf -l ${file} | grep -q "GNU_RELRO" -} - -# Check if a file is fully RELRO. -# -function file_is_relro_full() { - local file=${1} - - if file_is_relro_partly ${file}; then - readelf -d ${file} | grep -q "BIND_NOW" - return $? - fi - return 1 -} - -# Find all ELF files. -# -function find_elf_files() { - local dir - local dirs - local prefix - - while [ $# -gt 0 ]; do - case "${1}" in - --prefix=*) - prefix="${1#--prefix=}/" - ;; - *) - dirs="${dirs} ${1}" - ;; - esac - shift - done - - local file - local files - - for dir in ${dirs}; do - dir="${prefix}${dir}" - for file in $(find ${dir} -type f 2>/dev/null); do - if file_is_elf ${file} && file_is_shared_object ${file} && ! file_is_static ${file}; then - files="${files} ${file}" - fi - done - done - - echo ${files} -} - -function filter_startfiles() { - local file=${1} - - grep -qE "crt[1in]\.o$" <<<${file} -} diff --git a/tools/functions-lists b/tools/functions-lists deleted file mode 100644 index d8152d487..000000000 --- a/tools/functions-lists +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -function listsort() { - local item - for item in $@; do - echo "${item}" - done | sort -u | tr "\n" " " -} - -function listmatch() { - local arg=${1} - shift - - local item - for item in $@; do - if [ "${arg}" = "${item}" ]; then - return 0 - fi - done - return 1 -} - -function sort_by_length() { - local c - local i - for i in $@; do - echo "$(wc -c <<<${i}) ${i}" - done | sort -n -r | while read c i; do - echo "${i}" - done -} diff --git a/tools/functions-logging b/tools/functions-logging deleted file mode 100644 index 4fd43edce..000000000 --- a/tools/functions-logging +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -function log() { - local level=${1} - shift - - if [ "${level}" = "DEBUG" ] && [ "${DEBUG}" != "1" ]; then - return - fi - - printf " %1s | %s\n" "${level:0:1}" "$@" >&2 -} diff --git a/tools/functions-packager-find b/tools/functions-packager-find deleted file mode 100644 index bc97483cf..000000000 --- a/tools/functions-packager-find +++ /dev/null @@ -1,214 +0,0 @@ -#!/bin/bash - -# A function that finds needed libraries and interpreters. -# -function find_requires() { - local dir - local dirs=$@ - - # Find interpreters of all files in the dirs and skip those we provide - # ourself. - local interpreter - local interpreters - for interpreter in $(find_interpreters ${dirs}); do - local found=0 - for dir in ${dirs}; do - if [ -e "${dir}/${interpreter}" ]; then - found=1 - break - fi - done - - [ "${found}" = "0" ] && interpreters="${interpreters} ${interpreter}" - done - - # Find NEEDED libs and add them to a list if they are not provided by any - # other file in dirs. - local neededs - for file in $(find_elf_files ${dirs}); do - for needed in $(file_get_needed ${file}); do - neededs="${neededs} ${needed}" - done - done - - # Find all symlink destinations - local links=$(find_symlink_destinations ${dirs}) - - # Others - local others=$(find_python_requires ${dirs}) - others="${others} $(find_weak_symbols_requires ${dirs})" - others="${others} $(find_perl_requires ${dirs})" - - # Get provides, because packages should not depend on features they provide - # by themselves. - local provides=$(find_provides ${dirs}) - - # Return a sorted and unique(!) list - local require - local requires - for require in $(listsort ${PKG_DEPS} ${interpreters} ${neededs} ${links} ${others}); do - [ "${require:0:3}" = "ld-" ] && continue - listmatch ${require} ${provides} || requires="${requires} ${require}" - done - - echo ${requires} -} - -function find_provides() { - local dirs=$@ - - local file - local sonames - for file in $(find_elf_files ${dirs}); do - sonames="${sonames} $(file_get_soname ${file})" - done - sonames=$(listsort ${sonames}) - - # Others - local others=$(find_python_provides ${dirs}) - others="${others} $(find_weak_symbols_provides ${dirs})" - others="${others} $(find_perl_provides ${dirs})" - - listsort ${PKG_PROVIDES} ${sonames} ${others} -} - -function find_interpreters() { - local dirs=$@ - - log DEBUG "Searching for interpreters in ${dirs}" - - local file - local interpreter - local interpreters - for file in $(find ${dirs} -type f 2>/dev/null); do - # Get interpreter information from file. - interpreter=$(file_get_interpreter ${file}) - - # Skip the file silently if the result was empty. - [ -z "${interpreter}" ] && continue - - # Skip invalid interpreters that don't start with a slash. - if [ "${interpreter:0:1}" != "/" ]; then - log WARNING "Skipping invalid interpreter \"${interpreter}\" from \"${file}\"." - continue - fi - - if ! listmatch ${interpreter} ${INTERPRETERS_TO_BE_SKIPPED}; then - interpreters="${interpreters} ${interpreter}" - fi - done - - interpreters=$(listsort ${interpreters}) - - log DEBUG "find_interpreters ${dirs}: ${interpreters}" - - echo "${interpreters}" -} - -# Find the destinations of all symlinks and adds a dependency for that. -# -function find_symlink_destinations() { - local dir=$@ - - local link - local links - for link in $(find ${dir} -type l 2>/dev/null); do - link="$(readlink -m ${link})" - [ -e "${link}" ] && continue - - link="${link#${dir}}" - links="${links} ${link}" - done - - echo ${links} -} - -function find_python_provides() { - local dir=${1} - - local file - for file in $(find ${dir}/usr/bin/python* 2>/dev/null); do - file=$(basename ${file}) - file=${file#python} - - if [ -n "${file}" ]; then - echo "python-api=${file}" - fi - done -} - -function find_python_requires() { - local dir=${1} - - local file - for file in $(find ${dir}/usr/lib -maxdepth 1 2>/dev/null); do - file=$(basename ${file}) - - if [ "${file:0:6}" = "python" ]; then - file=${file#python} - - if [ -n "${file}" ]; then - echo "python-api=${file}" - fi - fi - done -} - -function find_perl_files() { - local extension - for extension in pm pl; do - find $@ -name "*.${extension}" 2>/dev/null - done -} - -function find_perl_provides() { - [ -x "/usr/bin/perl" ] || return 0 - ${BASEDIR}/perl.prov $(find_perl_files $@) | sort -u -} - -function find_perl_requires() { - [ -x "/usr/bin/perl" ] || return 0 - ${BASEDIR}/perl.req $(find_perl_files $@) | sort -u -} - -function find_weak_symbols_provides() { - local dirs=$@ - - local file - local soname - local symbol - for file in $(find_elf_files ${dirs}); do - soname=$(file_get_soname ${file}) - [ -z "${soname}" ] && continue - - for symbol in $(objdump -p ${file} | grep -E "^[0-9]+" | awk '{ print $4 }'); do - [ "${symbol}" = "${soname}" ] && continue - [ "${symbol}" = "GLIBC_PRIVATE" ] && continue - - echo "${soname}(${symbol})" - done - done | sort -u -} - -function find_weak_symbols_requires() { - local dirs=$@ - - local file - for file in $(find_elf_files ${dirs}); do - objdump -p ${file} | awk 'BEGIN { START=0; LIBNAME=""; } - /^$/ { START=0; } - /^Dynamic Section:$/ { START=1; } - (START==1) && /NEEDED/ { - print $2; - } - (START==2) && /^[A-Za-z]/ { START=3; } - /^Version References:$/ { START=2; } - (START==2) && /required from/ { - sub(/:/, "", $3); - LIBNAME=$3; - } - (START==2) && (LIBNAME!="") && ($4!="") && (($4~/^GLIBC_*/) || ($4~/^GCC_*/)) { - print LIBNAME "(" $4 ")"; - }' - done | grep -v "GLIBC_PRIVATE" | sort -u -} diff --git a/tools/generator b/tools/generator deleted file mode 100755 index a04b8bf97..000000000 --- a/tools/generator +++ /dev/null @@ -1,255 +0,0 @@ -#!/bin/bash -x - -BOOTLOADER_DIR=/usr/src/src/bootloader -IMAGES_DIR=/usr/src/images - -# Size of the ext filesystem in gigabytes -FSSIZE=2 -FSTYPE=ext4 - -ISO_FILENAME=${DISTRO_SNAME}-${DISTRO_VERSION}.${PKG_ARCH}.iso - -. $(dirname ${0})/common-functions - -function find_latest_kernel_release() { - local i - for i in /lib/modules/*; do - i=$(basename ${i}) - done - - echo "${i}" -} - -KERNEL_RELEASE=$(find_latest_kernel_release) - -function installer_image() { - local target=${1} - local dir=${2} - - # Create target directory if not existant - mkdir -p ${target} 2>/dev/null - - # Create installer images: - # (1) osmin.img (minimal cow image) - # (2) squashfs.img (installer) - - local i - for i in dev proc sys lib/modules; do - mkdir -p ${dir}/${i} 2>/dev/null - done - - # Copy kernel modules - cp -vfr /lib/modules/${KERNEL_RELEASE} ${dir}/lib/modules/ - - # Copy overlay - cp -vfr /usr/src/src/install/* ${dir} - - _installer_image_ext3fs \ - ${target}/$(basename ${target})/ext3fs.img /installer - _installer_image_osmin \ - ${target}/osmin.img \ - ${target}/$(basename ${target})/ext3fs.img - - cd ${target} - mksquashfs * ${target}/squashfs.img -no-progress - rm -fr $(basename ${target}) -} - -function _installer_image_osmin() { - local file=${1} - local image=${2} - - local tmp=$(mktemp -d) - - local image_loop=$(losetup -f) - - # Setting up loop for image - losetup ${image_loop} ${image} - - # Setting up loop for an empty 64byte sparse file - sparse=${tmp}/osmin - sparse_loop=$(losetup -f) - - _ext3fs_sparse ${sparse} 128M - losetup ${sparse_loop} ${sparse} - - # Create a DM snapshot device... - local name="imgcreate-$$" - local size=$(stat --format="%s" ${image}) - size=$(( ${size} / 512 )) - - dmsetup create ${name} \ - --table "0 ${size} snapshot ${image_loop} ${sparse_loop} p 8" - - # ...and resize it to its minimal size - _ext3fs_resize /dev/mapper/${name} - - # Get size of the bytes used by the cow image - local cow_size=( $(dmsetup status | grep "^${name}") ) - cow_size=${cow_size[4]} - cow_size=$(awk -F"/" '{ print $1 }' <<<${cow_size}) - cow_size=$(( ${cow_size} * 512 )) - - sleep 2 - dmsetup remove ${name} - - # ... and truncate it to its minimal size - truncate -s ${cow_size} ${sparse} - - losetup -d ${sparse_loop} - losetup -d ${image_loop} - - rm -f ${file} - ( cd ${tmp} && \ - mksquashfs * ${file} -no-progress ) -} - -function _ext3fs_blocks() { - local device=${1} - - dumpe2fs -h ${device} 2>/dev/null | \ - grep "Block count" | awk '{ print $NF }' -} - -function _ext3fs_bytes() { - echo $(( $(_ext3fs_blocks $@) * 512 )) -} - -function _ext3fs_resize() { - local device=${1} - local size=${2} - - [ -z "${size}" ] && size="-M" - - e2fsck -f -y ${device} - resize2fs ${device} ${size} - - # Resparse after every resize operation - _ext3fs_resparse ${device} -} - -function _ext3fs_sparse() { - local file=${1} - local size=${2} - - # Create a sparse disk with given size - dd if=/dev/zero of=${file} bs=1 count=0 seek=${size} -} - -function _ext3fs_resparse() { - local file=${1} - - zerofree ${file} -} - -function _installer_image_ext3fs() { - local file=${1} - local dir=${2} - - local zero=$(mktemp) - - mkdir -p $(dirname ${file}) 2>/dev/null - - # Create a zeroed file - _ext3fs_sparse ${file} ${FSSIZE}G - - # Create a temporary directory - # and get a free loop device - local tmp_dir=$(mktemp -d) - local loop=$(losetup -f) - - # Set up the loop device - losetup ${loop} ${file} - - # Create filesystem - mkfs.${FSTYPE} \ - -L "$(basename ${file})" \ - -m 1 \ - ${loop} - - # Tune the FS - tune2fs -c0 -i0 -Odir_index -ouser_xattr,acl ${loop} - - # Mount and copy all files to the FS - mount ${loop} ${tmp_dir} - cp -frp ${dir}/* ${tmp_dir} - umount ${tmp_dir} - - losetup -d ${loop} - - _ext3fs_resparse ${file} - - rm -rf ${tmp_dir} -} - -function install_config() { - local src=${1} - local dest=${2} - - sed \ - -e "s/@NAME@/${DISTRO_NAME}/g" \ - -e "s/@SNAME@/${DISTRO_SNAME}/g" \ - -e "s/@VERSION@/${DISTRO_VERSION}/g" \ - -e "s/@SLOGAN@/${DISTRO_SLOGAN}/g" \ - -e "s/@KERNEL@/${KERNEL_RELEASE}/g" \ - < ${src} > ${dest} -} - -function install_isolinux() { - local dest=${1} - - mkdir -p ${dest} 2>/dev/null - - local i - for i in isolinux.bin vesamenu.c32; do - cp -f /usr/share/syslinux/${i} ${dest}/ - done - - install_config ${BOOTLOADER_DIR}/installer.conf ${dest}/isolinux.cfg - cp -f ${BOOTLOADER_DIR}/{*.msg,splash.*} ${dest}/ - - install_kernel ${dest} -} - -function install_kernel() { - local dest=${1} - - mkdir -p ${dest} 2>/dev/null - - local file - for file in vmlinuz; do - cp -f /boot/${file}-${KERNEL_RELEASE}* ${dest}/${file}0 - done - - dracut -f ${dest}/initrd0 ${KERNEL_RELEASE} -} - -case "${1}" in - iso) - log DEBUG "Creating ISO image..." - - ISO_DIR=$(mktemp -d) - ISO_FILE=$(mktemp) - - # Copy installer image to ISO - installer_image ${ISO_DIR}/LiveOS /installer - - # Install bootloader - install_isolinux ${ISO_DIR}/isolinux - - # "Copy" packages - mv /packages ${ISO_DIR}/Packages - - cd ${ISO_DIR} && \ - mkisofs -J -r -V "${DISTRO_NAME}_${DISTRO_VERSION}" \ - -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot \ - -boot-load-size 4 -boot-info-table . > ${ISO_FILE} - - cat ${ISO_FILE} > ${IMAGES_DIR}/${ISO_FILENAME} - ;; - - *) - log ERROR "Unknown type of image: ${1}" - exit 1 - ;; -esac diff --git a/tools/lspkg b/tools/lspkg deleted file mode 100755 index f56683646..000000000 --- a/tools/lspkg +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -export PATH=${PATH}:/tools_i686/bin - -if [ -z "${1}" ]; then - exit 1 -fi - -cpio --extract --to-stdout data.img <${1} | xzcat | \ - tar tvv - -cpio --extract --to-stdout info < ${1} diff --git a/tools/packager b/tools/packager deleted file mode 100755 index c998780d8..000000000 --- a/tools/packager +++ /dev/null @@ -1,279 +0,0 @@ -#!/bin/bash -############################################################################### -# # -# IPFire.org - A linux based firewall # -# Copyright (C) 2010 Michael Tremer & Christian Schmidt # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -############################################################################### - -BASEDIR=$(dirname ${0}) - -PACKAGER_HASH_ALGORITHMS="md5 sha1 sha256 sha512" -PACKAGER_VERSION="1" - -TAR_OPTIONS="--posix --acls --no-recursion --sparse --xz" - -. ${BASEDIR}/common-functions - -# Create an alias for PKG_ORIGIN to track which source package has built this -# binary package. -PKG_ORIGIN=${PKG_NAME} - -### MAIN ### - -if [ $# -ne 2 ]; then - echo "${0}: Wrong number of arguments: $# - $@" >&2 - exit 2 -fi - -PKG_NAME=${1} -PKG_TARGET=${2} - - -### Function definitions ### - -function create_metafile() { - local files=${1} - - cat </dev/null | sort); do - # Remove ${BUILDROOT} - file="${file#${BUILDROOT}}" - - # Remove all leading slashes - while [ "${file:0:1}" = "/" ]; do - file="${file:1:${#file}}" - done - - echo "${file}" - done -} - -function __filelist() { - local paths - local exclude_paths - - # Disable globbing - set -f - - local path - local exclude - for path in ${PKG_FILES}; do - if [ "${path:0:1}" = "!" ]; then - exclude="1" - path="${path:1:${#path}}" - else - exclude="0" - fi - - if [ "${path:0:1}" != "/" ]; then - path="/${path}" - fi - - path="${BUILDROOT}${path}" - if [ "${exclude}" = "0" ]; then - paths="${paths} ${path}" - else - exclude_paths="${exclude_paths} ${path}" - fi - done - - # Enable globbing again - set +f - - # If not paths were found -> break - [ -z "${paths}" ] && return - - local excludes - if [ -n "${exclude_paths}" ]; then - excludes=$(find_files ${exclude_paths}) - fi - - for file in $(find_files ${paths}); do - if [ -n "${excludes}" ]; then - if ! listmatch ${file} ${excludes}; then - echo "${file}" - fi - else - echo "${file}" - fi - done -} - -function create_dataimg() { - local dir=${1} - local img=${2} - - # First, remove all crappy directories - directory_remove_orphans ${BUILDROOT} - - # Generate the filelist. - local filelist=$(mktemp) - __filelist > ${filelist} - - if [ "${DEBUG}" = "1" ]; then - cat ${filelist} >&2 - fi - - # Create the tarball - # Prints the filelist on fd 2. - cd ${BUILDROOT} && \ - tar ${TAR_OPTIONS} \ - --create -vv \ - --file=${img} \ - --files-from=${filelist} >&2 - - # Untar the tarball for further investigation - cd ${dir} && \ - tar ${TAR_OPTIONS} \ - --extract \ - --file=${img} - - local file - - # Remove the just copied files - sort_by_length $(<${filelist}) | \ - while read file; do - [ -z "${file}" ] && continue - - if [ -d "${BUILDROOT}/${file}" ]; then - if dir_is_empty ${BUILDROOT}/${file}; then - rm -rf ${BUILDROOT}/${file} - fi - continue - fi - rm -rf ${BUILDROOT}/${file} - done - - # Return the filelist - cd ${dir} && \ - for file in $(find . -not -type d); do - echo "${file:1:${#file}}" - done -} - -function create_package() { - local target_file=${1} - - # Create temporary directory where to put in all files that will go into - # the package. - local tmp=$(mktemp -d) - - # A place where to copy the package files for further processing. - local files=$(mktemp -d) - - # Create the package content - create_dataimg ${files} ${tmp}/data.img > ${tmp}/filelist - create_metafile ${files} > ${tmp}/info - create_controlfile > ${tmp}/control - - # Add an empty file to hold place for the signature - touch ${tmp}/signature - - # Calculate hash sums of the package files - log DEBUG "Calculating hash sums." - local algorithm - local file - local files=$(cd ${tmp} && ls) - for algorithm in ${PACKAGER_HASH_ALGORITHMS}; do - log DEBUG " Algorithm: ${algorithm}" - for file in ${files}; do - log DEBUG " ${file}" - echo "${algorithm} $(cd ${tmp} && ${algorithm}sum ${file})" \ - >> ${tmp}/sums - done - done - - # Create the package - cd ${tmp} && \ - find . | cpio -o -H newc --quiet > ${target_file} - - # Cleanup - rm -rf ${tmp} ${files} -} - -create_package ${PKG_TARGET} - -exit $? - diff --git a/tools/patch b/tools/patch deleted file mode 100755 index 68fef901a..000000000 --- a/tools/patch +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -for patch in $@; do - echo "Applying file ${patch}..." - - if [ "${patch##*/*.}" = "patch0" ]; then - cmd="patch -Np0" - else - cmd="patch -Np1" - fi - - ${cmd} -i ${patch} - ret=$? - - if [ ${ret} -ne 0 ]; then - exit ${ret} - fi -done diff --git a/tools/perl.prov b/tools/perl.prov deleted file mode 100755 index 73bec513b..000000000 --- a/tools/perl.prov +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/perl - -# RPM (and it's source code) is covered under two separate licenses. - -# The entire code base may be distributed under the terms of the GNU -# General Public License (GPL), which appears immediately below. -# Alternatively, all of the source code in the lib subdirectory of the -# RPM source code distribution as well as any code derived from that -# code may instead be distributed under the GNU Library General Public -# License (LGPL), at the choice of the distributor. The complete text -# of the LGPL appears at the bottom of this file. - -# This alternative is allowed to enable applications to be linked -# against the RPM library (commonly called librpm) without forcing -# such applications to be distributed under the GPL. - -# Any questions regarding the licensing of RPM should be addressed to -# Erik Troan . - -# a simple script to print the proper name for perl libraries. - -# To save development time I do not parse the perl grammmar but -# instead just lex it looking for what I want. I take special care to -# ignore comments and pod's. - -# it would be much better if perl could tell us the proper name of a -# given script. - -# The filenames to scan are either passed on the command line or if -# that is empty they are passed via stdin. - -# If there are lines in the file which match the pattern -# (m/^\s*\$VERSION\s*=\s+/) -# then these are taken to be the version numbers of the modules. -# Special care is taken with a few known idioms for specifying version -# numbers of files under rcs/cvs control. - -# If there are strings in the file which match the pattern -# m/^\s*\$RPM_Provides\s*=\s*["'](.*)['"]/i -# then these are treated as additional names which are provided by the -# file and are printed as well. - -# I plan to rewrite this in C so that perl is not required by RPM at -# build time. - -# by Ken Estes Mail.com kestes@staff.mail.com - -if ("@ARGV") { - foreach (@ARGV) { - process_file($_); - } -} else { - - # notice we are passed a list of filenames NOT as common in unix the - # contents of the file. - - foreach (<>) { - process_file($_); - } -} - - -foreach $module (sort keys %require) { - if (length($require{$module}) == 0) { - print "perl($module)\n"; - } else { - - # I am not using rpm3.0 so I do not want spaces arround my - # operators. Also I will need to change the processing of the - # $RPM_* variable when I upgrade. - - print "perl($module)=$require{$module}\n"; - } -} - -exit 0; - - - -sub process_file { - - my ($file) = @_; - chomp $file; - - open(FILE, "<$file") || return; - - my ($package, $version, $incomment, $inover) = (); - - while () { - - # skip the documentation - - # we should not need to have item in this if statement (it - # properly belongs in the over/back section) but people do not - # read the perldoc. - - if (m/^=(head[1-4]|pod|item)/) { - $incomment = 1; - } - - if (m/^=(cut)/) { - $incomment = 0; - $inover = 0; - } - - if (m/^=(over)/) { - $inover = 1; - } - - if (m/^=(back)/) { - $inover = 0; - } - - if ($incomment || $inover) { - next; - } - - # skip the data section - if (m/^__(DATA|END)__$/) { - last; - } - - # not everyone puts the package name of the file as the first - # package name so we report all namespaces except some common - # false positives as if they were provided packages (really ugly). - - if (m/^\s*package\s+([_:a-zA-Z0-9]+)\s*;/) { - $package=$1; - undef $version; - if ($package eq 'main') { - undef $package; - } else { - # If $package already exists in the $require hash, it means - # the package definition is broken up over multiple blocks. - # In that case, don't stomp a previous $VERSION we might have - # found. (See BZ#214496.) - $require{$package}=undef unless (exists $require{$package}); - } - } - - # after we found the package name take the first assignment to - # $VERSION as the version number. Exporter requires that the - # variable be called VERSION so we are safe. - - # here are examples of VERSION lines from the perl distribution - - #FindBin.pm:$VERSION = $VERSION = sprintf("%d.%02d", q$Revision: 1.9 $ =~ /(\d+)\.(\d+)/); - #ExtUtils/Install.pm:$VERSION = substr q$Revision: 1.9 $, 10; - #CGI/Apache.pm:$VERSION = (qw$Revision: 1.9 $)[1]; - #DynaLoader.pm:$VERSION = $VERSION = "1.03"; # avoid typo warning - #General.pm:$Config::General::VERSION = 2.33; - # - # or with the new "our" pragma you could (read will) see: - # - # our $VERSION = '1.00' - if (($package) && (m/^\s*(our\s+)?\$(\Q$package\E::)?VERSION\s*=\s+/)) { - - # first see if the version string contains the string - # '$Revision' this often causes bizzare strings and is the most - # common method of non static numbering. - - if (m/(\$Revision: (\d+[.0-9]+))/) { - $version= $2; - } elsif (m/[\'\"]?(\d+[.0-9]+)[\'\"]?/) { - - # look for a static number hard coded in the script - - $version= $1; - } - $require{$package}=$version; - } - - # Allow someone to have a variable that defines virtual packages - # The variable is called $RPM_Provides. It must be scoped with - # "our", but not "local" or "my" (just would not make sense). - # - # For instance: - # - # $RPM_Provides = "blah bleah" - # - # Will generate provides for "blah" and "bleah". - # - # Each keyword can appear multiple times. Don't - # bother with datastructures to store these strings, - # if we need to print it print it now. - - if ( m/^\s*(our\s+)?\$RPM_Provides\s*=\s*["'](.*)['"]/i) { - foreach $_ (split(/\s+/, $2)) { - print "$_\n"; - } - } - - } - - close(FILE) || - die("$0: Could not close file: '$file' : $!\n"); - - return ; -} diff --git a/tools/perl.req b/tools/perl.req deleted file mode 100755 index d0a1cd729..000000000 --- a/tools/perl.req +++ /dev/null @@ -1,249 +0,0 @@ -#!/usr/bin/perl - -# RPM (and its source code) is covered under two separate licenses. - -# The entire code base may be distributed under the terms of the GNU -# General Public License (GPL), which appears immediately below. -# Alternatively, all of the source code in the lib subdirectory of the -# RPM source code distribution as well as any code derived from that -# code may instead be distributed under the GNU Library General Public -# License (LGPL), at the choice of the distributor. The complete text -# of the LGPL appears at the bottom of this file. - -# This alternatively is allowed to enable applications to be linked -# against the RPM library (commonly called librpm) without forcing -# such applications to be distributed under the GPL. - -# Any questions regarding the licensing of RPM should be addressed to -# Erik Troan . - -# a simple makedepend like script for perl. - -# To save development time I do not parse the perl grammmar but -# instead just lex it looking for what I want. I take special care to -# ignore comments and pod's. - -# It would be much better if perl could tell us the dependencies of a -# given script. - -# The filenames to scan are either passed on the command line or if -# that is empty they are passed via stdin. - -# If there are strings in the file which match the pattern -# m/^\s*\$RPM_Requires\s*=\s*["'](.*)['"]/i -# then these are treated as additional names which are required by the -# file and are printed as well. - -# I plan to rewrite this in C so that perl is not required by RPM at -# build time. - -# by Ken Estes Mail.com kestes@staff.mail.com - -if ("@ARGV") { - foreach (@ARGV) { - process_file($_); - } -} else { - - # notice we are passed a list of filenames NOT as common in unix the - # contents of the file. - - foreach (<>) { - process_file($_); - } -} - - -foreach $module (sort keys %require) { - if (length($require{$module}) == 0) { - print "perl($module)\n"; - } else { - - # I am not using rpm3.0 so I do not want spaces around my - # operators. Also I will need to change the processing of the - # $RPM_* variable when I upgrade. - - print "perl($module)>=$require{$module}\n"; - } -} - -exit 0; - - - -sub process_file { - - my ($file) = @_; - chomp $file; - - open(FILE, "<$file") || return; - - while () { - - # skip the "= <<" block - - if ( ( m/^\s*\$(?:.*)\s*=\s*<<\s*(["'`])(.*)\1/) || - ( m/^\s*\$(.*)\s*=\s*<<(\w*)\s*;/) ) { - $tag = $2; - while () { - chomp; - ( $_ eq $tag ) && last; - } - $_ = ; - } - - # skip q{} quoted sections - just hope we don't have curly brackets - # within the quote, nor an escaped hash mark that isn't a comment - # marker, such as occurs right here. Draw the line somewhere. - if ( m/^.*\Wq[qxwr]?\s*([\{\(\[#|\/])[^})\]#|\/]*$/ && ! m/^\s*(require|use)\s/ ) { - $tag = $1; - $tag =~ tr/{\(\[\#|\//})]#|\//; - while () { - ( $_ =~ m/\}/ ) && last; - } - } - - # skip the documentation - - # we should not need to have item in this if statement (it - # properly belongs in the over/back section) but people do not - # read the perldoc. - - if ( (m/^=(head[1-4]|pod|item)/) .. (m/^=(cut)/) ) { - next; - } - - if ( (m/^=(over)/) .. (m/^=(back)/) ) { - next; - } - - # skip the data section - if (m/^__(DATA|END)__$/) { - last; - } - - # Each keyword can appear multiple times. Don't - # bother with datastructures to store these strings, - # if we need to print it print it now. - # - # Again allow for "our". - if ( m/^\s*(our\s+)?\$RPM_Requires\s*=\s*["'](.*)['"]/i) { - foreach $_ (split(/\s+/, $2)) { - print "$_\n"; - } - } - - if ( - -# ouch could be in a eval, perhaps we do not want these since we catch -# an exception they must not be required - -# eval { require Term::ReadLine } or die $@; -# eval "require Term::Rendezvous;" or die $@; -# eval { require Carp } if defined $^S; # If error/warning during compilation, - - - (m/^(\s*) # we hope the inclusion starts the line - (require|use)\s+(?!\{) # do not want 'do {' loops - # quotes around name are always legal - [\'\"]?([^\;\ \'\"\t]*)[\'\"]?[\t\;\ ] - # the syntax for 'use' allows version requirements - \s*([.0-9]*) - /x) - ) { - my ($whitespace, $statement, $module, $version) = ($1, $2, $3,$4); - - # we only consider require statements that are flush against - # the left edge. any other require statements give too many - # false positives, as they are usually inside of an if statement - # as a fallback module or a rarely used option - - ($whitespace ne "" && $statement eq "require") && next; - - # if there is some interpolation of variables just skip this - # dependency, we do not want - # do "$ENV{LOGDIR}/$rcfile"; - - ($module =~ m/\$/) && next; - - # skip if the phrase was "use of" -- shows up in gimp-perl, et al. - next if $module eq 'of'; - - # if the module ends in a comma we probaly caught some - # documentation of the form 'check stuff,\n do stuff, clean - # stuff.' there are several of these in the perl distribution - - ($module =~ m/[,>]$/) && next; - - # if the module name starts in a dot it is not a module name. - # Is this necessary? Please give me an example if you turn this - # back on. - - # ($module =~ m/^\./) && next; - - # if the module ends with .pm strip it to leave only basename. - # starts with /, which means its an absolute path to a file - if ($module =~ m(^/)) { - print "$module\n"; - next; - } - - # sometimes people do use POSIX qw(foo), or use POSIX(qw(foo)) etc. - # we can strip qw.*$, as well as (.*$: - $module =~ s/qw.*$//; - $module =~ s/\(.*$//; - - $module =~ s/\.pm$//; - - # some perl programmers write 'require URI/URL;' when - # they mean 'require URI::URL;' - - $module =~ s/\//::/; - - # trim off trailing parentheses if any. Sometimes people pass - # the module an empty list. - - $module =~ s/\(\s*\)$//; - - if ( $module =~ m/^v?([0-9._]+)$/ ) { - # if module is a number then both require and use interpret that - # to mean that a particular version of perl is specified - - my $ver=$1; - if ($ver =~ /5.00/) { - print "perl>=0:$ver\n"; - next; - } - else { - print "perl>=1:$ver\n"; - next; - } - - }; - - # ph files do not use the package name inside the file. - # perlmodlib documentation says: - - # the .ph files made by h2ph will probably end up as - # extension modules made by h2xs. - - # so do not expend much effort on these. - - - # there is no easy way to find out if a file named systeminfo.ph - # will be included with the name sys/systeminfo.ph so only use the - # basename of *.ph files - - ($module =~ m/\.ph$/) && next; - - $require{$module}=$version; - $line{$module}=$_; - } - - } - - close(FILE) || - die("$0: Could not close file: '$file' : $!\n"); - - return ; -} diff --git a/tools/py-compile b/tools/py-compile deleted file mode 100755 index 66c33edfe..000000000 --- a/tools/py-compile +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/sh - -PYTHON=$(which python 2>/dev/null) - -if [ -z "${PYTHON}" ]; then - # Python is not present. Fail silently. - exit 0 -fi - -files="" -for i in $*; do - if [ -e ${i}c ] && [ -e ${i}o ]; then - continue # all files we want are already there - fi - files="$files $i" -done - -if [ -z "${files}" ]; then - # No files need to be proceeded. - exit 0 -fi - -$PYTHON -c " -import sys, os, string, py_compile - -files = '''$files''' -print 'Byte-compiling python modules...' -for file in string.split(files): - if not os.path.exists(file) or not (len(file) >= 3 and file[-3:] == '.py'): - continue - print file, - sys.stdout.flush() - py_compile.compile(file) -print" || exit $? - -# this will fail for python < 1.5, but that doesn't matter ... -$PYTHON -O -c " -import sys, os, string, py_compile - -files = '''$files''' -print 'Byte-compiling python modules (optimised versions) ...' -for file in string.split(files): - if not os.path.exists(file) or not (len(file) >= 3 and file[-3:] == '.py'): - continue - print file, - sys.stdout.flush() - py_compile.compile(file) -print" 2>/dev/null || : diff --git a/tools/quality-agent b/tools/quality-agent deleted file mode 100755 index 57d06680f..000000000 --- a/tools/quality-agent +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -DIR_QA=${0}.d - -failed=0 -for file in ${DIR_QA}/*; do - [ -x "${file}" ] || continue - - ${file} || failed=1 -done - -exit ${failed} diff --git a/tools/quality-agent.d/001-include-files b/tools/quality-agent.d/001-include-files deleted file mode 100755 index 34257e690..000000000 --- a/tools/quality-agent.d/001-include-files +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="Include files have to belong to the root user. \ - This script will fix this automatically." - -check() { - if [ ! -d "${BUILDROOT}/usr/include" ]; then - return 0 - fi - - chown -R root:root ${BUILDROOT}/usr/include -} - -run diff --git a/tools/quality-agent.d/001-remove-info-files b/tools/quality-agent.d/001-remove-info-files deleted file mode 100755 index e742dc977..000000000 --- a/tools/quality-agent.d/001-remove-info-files +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="Remove documentation files." - -function check() { - for dir in ${BUILDROOT}/usr/{,share}/{doc,gtk-doc,info}; do - if [ -d "${dir}" ]; then - log DEBUG " Removing: ${dir}" - rm -rf ${dir} || exit $? - fi - done -} - -run - diff --git a/tools/quality-agent.d/001-remove-static-libs b/tools/quality-agent.d/001-remove-static-libs deleted file mode 100755 index e5c6e541f..000000000 --- a/tools/quality-agent.d/001-remove-static-libs +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="Removing unwanted files: *.a *.la" - -function check() { - for file in $(find ${BUILDROOT} -name "*.a" -or -name "*.la"); do - - # Don't remove libc_nonshared.a. It is used by gcc/ld. - [ "${file##*/}" = "libc_nonshared.a" ] && continue - [ "${file##*/}" = "libpthread_nonshared.a" ] && continue - [ "${file##*/}" = "libgcc.a" ] && continue - [ "${file##*/}" = "libgcc_eh.a" ] && continue - [ "${file##*/}" = "libfl_pic.a" ] && continue - [ "${file##*/}" = "libpython2.6.a" ] && continue - - log DEBUG " Removing: ${file}" - rm -f ${file} || exit $? - done -} - -run - diff --git a/tools/quality-agent.d/001-unsafe-files b/tools/quality-agent.d/001-unsafe-files deleted file mode 100755 index 93a5dc8d3..000000000 --- a/tools/quality-agent.d/001-unsafe-files +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="Searching for world-writeable files..." - -function check() { - local ret=0 - - local files=$(find ${BUILDROOT} -type f -perm -2 2>/dev/null) - if [ -n "${files}" ]; then - log ERROR " QA Security Notice:" - log ERROR " - The folloing files will be world writable." - log ERROR " - This may or may not be a security problem, most of the time it is one." - log ERROR " - Please double check that these files really need a world writeable bit and file bugs accordingly." - log ERROR - log ERROR "${files}" - ret=1 - fi - - files=$(find ${BUILDROOT} -type f '(' -perm -2002 -o -perm -4002 ')') - if [ -n "${files}" ]; then - log ERROR " QA Notice: Unsafe files detected (set*id and world writable)" - log ERROR - log ERROR "${files}" - ret=1 - fi - - return ${ret} -} - -run - diff --git a/tools/quality-agent.d/002-bad-symlinks b/tools/quality-agent.d/002-bad-symlinks deleted file mode 100755 index 595a7c5b6..000000000 --- a/tools/quality-agent.d/002-bad-symlinks +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -# Check for absolute symlinks. -# We do not allow them because they may point to any bad location. - -log_debug "Search for absolute symlinks" - -function check() { - local failed=0 - - for link in $(find ${BUILDROOT} -type l); do - if fgrep -q "/lib/udev/devices" <<<${link}; then - continue - fi - - destination=$(readlink ${link}) - if [ "${destination:0:1}" = "/" ]; then - log ERROR " Absolute symlink: ${link}" - failed=1 - fi - if [ ! -e "${link%/*}/${destination}" ]; then - log ERROR " Not existant destination: ${link} -> ${destination}" - failed=1 - fi - done - - return ${failed} -} - -run - diff --git a/tools/quality-agent.d/003-libs-location b/tools/quality-agent.d/003-libs-location deleted file mode 100755 index 185f44f06..000000000 --- a/tools/quality-agent.d/003-libs-location +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="Checking correct installation of libraries" - -function check() { - local failed=0 - for lib in $(find ${BUILDROOT}/lib -type f -name "lib*.so.*" 2>/dev/null); do - lib=${lib##*/} - lib=${lib%%.so*} - - if [ ! -e "${BUILDROOT}/usr/lib/${lib}.so" ]; then - log ERROR " /usr/lib/${lib}.so is missing" - failed=1 - fi - done - - return ${failed} -} - -run - diff --git a/tools/quality-agent.d/050-canary b/tools/quality-agent.d/050-canary deleted file mode 100755 index 67b25e531..000000000 --- a/tools/quality-agent.d/050-canary +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="Every binary file has to provide a canary." - -function check() { - local failed=0 - - local file - for file in $(find_elf_files --prefix=${BUILDROOT} ${BINARY_PATHS}); do - if filter_startfiles ${file}; then - continue - fi - - if ! file_has_canary ${file}; then - log_warning " Has no canary: ${file}" - failed=1 - fi - done - - # This is currently disabled and will only return a warning ! - failed=0 - - return ${failed} -} - -run diff --git a/tools/quality-agent.d/050-execstacks b/tools/quality-agent.d/050-execstacks deleted file mode 100755 index 9540eee55..000000000 --- a/tools/quality-agent.d/050-execstacks +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="Files with executable stacks will not work properly (or at all!) \ - on some architectures/operating systems." - -check() { - local failed=0 - - local file - for file in $(find_elf_files --prefix=${BUILDROOT} ${BINARY_PATHS}); do - if file_has_execstack ${file}; then - log_error " File has execstack: ${file}" - failed=1 - fi - done - - return ${failed} -} - -run diff --git a/tools/quality-agent.d/050-invalid-interpreters b/tools/quality-agent.d/050-invalid-interpreters deleted file mode 100755 index 09d0ba55e..000000000 --- a/tools/quality-agent.d/050-invalid-interpreters +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="Detect invalid interpreters." - -check() { - local failed=0 - - local file - local interpreter - for file in $(find ${BUILDROOT} -type f 2>/dev/null); do - # If a file is not executeable we don't need to check it - [ -x "${file}" ] || continue - - if file_is_script ${file}; then - interpreter=$(file_get_interpreter ${file}) - - if grep -q /usr/local <<<${interpreter}; then - failed=1 - log_error " Interpreter in /usr/local: ${file}" - fi - - # Search for bad /usr/bin/env - if [ "$(basename ${interpreter})" = "env" ]; then - # Autofix that crap - sed -i ${file} \ - -e "s,/usr/bin/env python.\..,/usr/bin/python," \ - -e "s,/usr/bin/env python,/usr/bin/python," \ - -e "s,/usr/bin/env perl,/usr/bin/perl," - - # If we could not fix it, raise an error - if [ "${interpreter}" = "$(file_get_interpreter ${file})" ]; then - failed=1 - log_error " Script uses forbidden \"env\" interpreter: ${file}" - fi - fi - fi - done - - return ${failed} -} - -run diff --git a/tools/quality-agent.d/050-libs-needed b/tools/quality-agent.d/050-libs-needed deleted file mode 100755 index 93f600dc3..000000000 --- a/tools/quality-agent.d/050-libs-needed +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="Every shared object has to provide the NEEDED entry." - -check() { - local failed=0 - - local file - local needed - for file in $(find_elf_files --prefix=${BUILDROOT} ${LIBARY_PATHS}); do - if ! file_is_shared_object ${file}; then - continue - fi - - if ! file_has_interpreter ${file}; then - continue - fi - - needed=$(file_get_needed ${file}) - if [ -z "${needed}" ]; then - log_error " File lacks needed attribute: ${file}" - failed=1 - fi - done - - return ${failed} -} - -run diff --git a/tools/quality-agent.d/050-libs-soname b/tools/quality-agent.d/050-libs-soname deleted file mode 100755 index b6232b644..000000000 --- a/tools/quality-agent.d/050-libs-soname +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="Every shared object has to provide the SONAME entry." - -check() { - local failed=0 - - local file - local soname - for file in $(find_elf_files --prefix=${BUILDROOT} ${LIBARY_PATHS}); do - if ! grep -q "\.so" <<<${file}; then - continue - fi - - if ! file_is_shared_object ${file}; then - continue - fi - - if ! file_has_interpreter ${file}; then - continue - fi - - soname=$(file_get_soname ${file}) - if [ -z "${soname}" ]; then - log_error " File lacks soname attribute: ${file}" - failed=1 - fi - done - - return ${failed} -} - -run diff --git a/tools/quality-agent.d/050-nx b/tools/quality-agent.d/050-nx deleted file mode 100755 index c9c27338d..000000000 --- a/tools/quality-agent.d/050-nx +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -function check() { - local failed=0 - - FILTER="${QUALITY_AGENT_WHITELIST_NX}" - - local file - for file in $(find_elf_files --prefix=${BUILDROOT} ${BINARY_PATHS}); do - if filtered ${file}; then - continue - fi - - if ! file_has_nx ${file}; then - log_error " No NX: ${file}" - failed=1 - fi - done - - return ${failed} -} - -run diff --git a/tools/quality-agent.d/050-relro b/tools/quality-agent.d/050-relro deleted file mode 100755 index f01105657..000000000 --- a/tools/quality-agent.d/050-relro +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="Text relocations force the dynamic linker to perform extra \ - work at startup, waste system resources, and may pose a security \ - risk. On some architectures, the code may not even function \ - properly, if at all." - -function check() { - local failed=0 - - local file - for file in $(find_elf_files --prefix=${BUILDROOT} ${BINARY_PATHS}); do - if filter_startfiles ${file}; then - continue - fi - - if ! file_is_relro_full ${file}; then - if [ "${QUALITY_AGENT_PERMIT_NOT_FULL_RELRO}" = "yes" ]; then - log_warning " Is not full relro: ${file}" - else - log_error " Is not relro: ${file}" - failed=1 - fi - fi - done - - return ${failed} -} - -run diff --git a/tools/quality-agent.d/050-root-links-to-usr b/tools/quality-agent.d/050-root-links-to-usr deleted file mode 100755 index 98385c5b8..000000000 --- a/tools/quality-agent.d/050-root-links-to-usr +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="Check for binaries in /bin or /sbin that link to /usr/..." - -function check() { - local ret=0 - - for file in $(find ${BUILDROOT}/{bin,lib,sbin}/* 2>/dev/null); do - [ -f "${file}" ] || continue - log DEBUG " ${file}" - - interpreter=$(file_get_interpreter ${file}) - if [ ! -e "${interpreter}" ]; then - log WARN " SKIPPED because interpreter is not available" - continue - fi - - libs=$(ldd ${file}) - if grep -q /usr/lib <<<${libs}; then - log ERROR "${file} links to libs in /usr/lib..." - log ERROR " ${libs}" - ret=1 - fi - done - - return ${ret} -} - -run diff --git a/tools/quality-agent.d/050-rpaths b/tools/quality-agent.d/050-rpaths deleted file mode 100755 index 8250612bf..000000000 --- a/tools/quality-agent.d/050-rpaths +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="Searching for RPATHs. We don't want paths that point to the tree where \ - the package was built (older, broken libtools would do this). \ - Also check for null paths because the loader will search \$PWD when it \ - finds null paths." - -check() { - local failed=0 - - local file - local rpath - for file in $(find_elf_files --prefix=${BUILDROOT} ${BINARY_PATHS}); do - if filtered ${file}; then - continue - fi - - rpath=$(file_get_rpath ${file}) - if [ -n "${rpath}" ]; then - if [ "${QUALITY_AGENT_RPATH_ALLOW_ORIGIN}" = "yes" ]; then - [ "${rpath}" = '$ORIGIN' ] && continue - fi - if listmatch ${rpath} ${QUALITY_AGENT_WHITELIST_RPATH}; then - continue - fi - log_error " File has unallowed rpath: ${file} - ${rpath}" - failed=1 - fi - done - - return ${failed} -} - -run diff --git a/tools/quality-agent.d/090-man-pages b/tools/quality-agent.d/090-man-pages deleted file mode 100755 index 0720d100e..000000000 --- a/tools/quality-agent.d/090-man-pages +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="Decompressing man-pages..." - -function check() { - for file in $(find ${BUILDROOT}/usr/share/man -type f 2>/dev/null); do - log DEBUG " Processing: ${file}" - if [[ ${file} =~ \.gz$ ]]; then - gzip -d ${file} - fi - done -} - -run - diff --git a/tools/quality-agent.d/090-python-hardlinks b/tools/quality-agent.d/090-python-hardlinks deleted file mode 100755 index e2abf76a9..000000000 --- a/tools/quality-agent.d/090-python-hardlinks +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="Python byte-code files could be hardlinked if the optimized one is equal" -DESC="${DESC} to the other one." - -function check() { - for py in $(find ${BUILDROOT} -type f -name "*.py"); do - if [ -e "${py}c" ] && [ -e "${py}o" ]; then - if cmp -s "${py}c" "${py}o"; then - log DEBUG " ${py}c -> ${py}o" - ln -f "${py}c" "${py}o" - fi - fi - done -} - -run - diff --git a/tools/quality-agent.d/090-remove-empty-dirs b/tools/quality-agent.d/090-remove-empty-dirs deleted file mode 100755 index d1d61a195..000000000 --- a/tools/quality-agent.d/090-remove-empty-dirs +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="Remove unwanted files." - -function check() { - for dir in $(sort_by_length {,/usr}/{{,s}bin,lib{,exec}} /usr/share/man{,/man{0,1,2,3,4,5,6,7,8,9}}); do - dir="${BUILDROOT}/${dir}" - if [ -d "${dir}" ] && [ "$(ls -1A ${dir} | wc -l)" = "0" ]; then - log DEBUG " Removing ${dir}" - rm -rf ${dir} - fi - done -} - -run - diff --git a/tools/quality-agent.d/095-directory-layout b/tools/quality-agent.d/095-directory-layout deleted file mode 100755 index 14f600d7b..000000000 --- a/tools/quality-agent.d/095-directory-layout +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="The filelayout should comply to the FHS." - -DIRS="/etc/init.d /etc/rc.d /lib/pkgconfig /usr/etc /usr/libexec /usr/local /usr/man /usr/usr /usr/var" - -function check() { - local failed=0 - - local dir - for dir in ${DIRS}; do - if [ -d "${BUILDROOT}${dir}" ]; then - log_error "Bad directory: ${dir}" - failed=1 - fi - done - - return ${failed} -} - -run diff --git a/tools/quality-agent.d/099-strip b/tools/quality-agent.d/099-strip deleted file mode 100755 index bd7a16a45..000000000 --- a/tools/quality-agent.d/099-strip +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -. $(dirname ${0})/qa-include - -DESC="Stripping reduces size of binary files." - -function check() { - # Strip debugging symbols - for f in $(find ${BUILDROOT} -type f \( -perm -0100 -or -perm -0010 -or -perm -0001 \)); do - if (file $f | grep -q ' shared object,'); then - strip --strip-debug "$f" || : - fi - done - - for f in $(find ${BUILDROOT} -type f); do - if (file $f | grep -q ' shared object,'); then - strip --strip-unneeded "$f" || : - fi - done -} - -run - diff --git a/tools/quality-agent.d/qa-include b/tools/quality-agent.d/qa-include deleted file mode 100644 index 4fe1c472b..000000000 --- a/tools/quality-agent.d/qa-include +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash - -# Include additional functions -. $(dirname ${0})/../common-functions - -function debug() { - [ "${NAOKI_DEBUG}" = "1" ] || [ "${DEBUG}" = "1" ] -} - -#function log() { -# local facility=${1} -# shift -# -# printf " %-7s %s\n" "${facility}" "$@" -#} - -function log_debug() { - debug && log DEBUG "$@" -} - -function log_error() { - log "ERROR" "$@" -} - -function log_info() { - log "INFO" "$@" -} - -function log_warning() { - log "WARNING" "$@" -} - -if [ -z "${BUILDROOT}" ]; then - echo "${0##*/}: ERROR: BUILDROOT is not set." >&2 - exit 1 -fi - -function filtered() { - [ -z "${FILTER}" ] && return 1 - grep -qE ${FILTER} <<<$@ -} - -function print_description() { - # Remove all whitespaces - local desc=$(echo ${DESC}) - - log_info "Check: $(basename ${0})" - IFS=' -' - for line in $(fold -s -w 60 <<<${desc}); do - log_info " ${line}" - done - log_info # Empty line - - unset IFS -} - -function qa_find() { - local filetype=${1} - local command=${2} - - log_debug "Running qa_find with command ${command} in ${filetype}" - - local file - for file in $(find_elf_files --prefix=${BUILDROOT} ${!filetype}); do - ${command} ${file} - done -} - -function check() { - log_error "REPLACE THIS FUNCTION BY A CUSTOM CHECK" - return 1 -} - -function run() { - local error_message - local ret - - error_message=$(check) - ret=$? - - [ -z "${error_message}" ] && \ - [ "${ret}" = "0" ] && return 0 - - print_description - - echo "${error_message}" - return ${ret} -} - diff --git a/tools/runpychecker.sh b/tools/runpychecker.sh deleted file mode 100755 index 449dd9d9b..000000000 --- a/tools/runpychecker.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -pychecker --only --limit 1000 \ - --maxlines 500 --maxargs 20 --maxbranches 80 --maxlocals 60 --maxreturns 20 \ - --no-callinit --no-local --no-shadow --no-shadowbuiltin \ - --no-import --no-miximport --no-pkgimport --no-reimport \ - --no-argsused --no-varargsused --no-override \ - naoki/*.py diff --git a/tools/ssh-keyput b/tools/ssh-keyput deleted file mode 100644 index da4aa2b9b..000000000 --- a/tools/ssh-keyput +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash -# -# ssh-keyput -- set up passwordless openssh login. -# -# Copyright (C) 2001, 2002, 2006 by SWsoft. -# Author: Kir Kolyshkin -# -# This script is used to put your public ssh keys to another host's -# authorized_keys[2], so you will be able to ssh login without entering -# a password. Key pairs are generated if needed, and connectivity -# is checked after putting the keys. - -PROGNAME=`basename $0` - -function usage() -{ - echo "Usage: $PROGNAME [user@]IP [[user@]IP ...]" 1>&2 - exit 0 -} - -# Check for correct number of parameters -test $# -gt 0 || usage; - -SSH_KEYGEN=`which ssh-keygen` -if test $? -ne 0; then - # Error message is printed by 'which' - exit 1 -fi - -SSH_DIR=~/.ssh -if ! test -d $SSH_DIR; then - mkdir $SSH_DIR -fi -chmod 700 $SSH_DIR - - -if [ ! -f $SSH_DIR/identity ] || [ ! -f $SSH_DIR/identity.pub ]; then - echo "Generating ssh1 RSA keys - please wait..." - rm -f $SSH_DIR/identity $SSH_DIR/identity.pub - $SSH_KEYGEN -t rsa1 -f $SSH_DIR/identity -P '' - if [ $? -ne 0 ]; then - echo "Command \"$SSH_KEYGEN -t rsa1 -f $SSH_DIR/identity" \ - "-P ''\" failed" 1>&2 - exit 1 - fi -else - echo "ssh1 RSA key is present" -fi - -if [ ! -f $SSH_DIR/id_dsa ] || [ ! -f $SSH_DIR/id_dsa.pub ]; then - echo "Generating ssh2 DSA keys - please wait..." - rm -f $SSH_DIR/id_dsa $SSH_DIR/id_dsa.pub - $SSH_KEYGEN -t dsa -f $SSH_DIR/id_dsa -P '' - if test $? -ne 0; then - echo "Command \"$SSH_KEYGEN -t dsa -f $SSH_DIR/id_dsa" \ - "-P ''\" failed" 1>&2 - exit 1 - fi -else - echo "ssh2 DSA key is present" -fi - -SSH1_RSA_KEY=`cat $SSH_DIR/identity.pub` -SSH2_DSA_KEY=`cat $SSH_DIR/id_dsa.pub` - -for IP in $*; do - echo "You will now be asked for password for $IP" -# set -x - ssh -oStrictHostKeyChecking=no $IP "mkdir -p ~/.ssh; chmod 700 ~/.ssh; \ - echo \"$SSH1_RSA_KEY\" >> ~/.ssh/authorized_keys; \ - echo \"$SSH2_DSA_KEY\" >> ~/.ssh/authorized_keys2; \ - chmod 600 ~/.ssh/authorized_keys ~/.ssh/authorized_keys2" -# set +x - if test $? -eq 0; then - echo "Keys were put successfully" - else - echo "Error putting keys to $IP" 1>&2 - fi -done - -for IP in $*; do - for ver in 1 2; do - echo -n "Checking $IP connectivity by ssh$ver... " - ssh -q -oProtocol=${ver} -oBatchMode=yes \ - -oStrictHostKeyChecking=no $IP /bin/true - if [ $? -eq 0 ]; then - echo "OK" - else - echo "failed" 1>&2 - fi - done -done