+++ /dev/null
-[default]
-
-; The default architecture
-default = i686
-
-[i686]
-
-machine = i686
-personality = linux32
-
-cflags = -O2 -fomit-frame-pointer -fasynchronous-unwind-tables -pipe
-cxxflags = %(cflags)s
-
-;[i586]
-;
-;machine = i586
-;personality = linux32
-;
-;cflags = -O2 -fomit-frame-pointer -pipe
-;cxxflags = %(cflags)s
-;
-;[i486]
-;
-;machine = i486
-;personality = linux32
-;
-;cflags = -O2 -fomit-frame-pointer -pipe
-;cxxflags = %(cflags)s
-
-;[alix]
-;
-; While i586 works fine on a Geode LX, i486 should be a more performant choice
-; right now due to the way Geode LX CPU pipeline ans scheduling works. glibc
-; i586 assembler optimized routines are measureably slower than the i486 ones on
-; a Geode LX.
-;machine = i486
-;personality = linux32
-;
-;cflags = -Os -march=geode -fno-align-jumps -fno-align-functions -fno-align-labels -fno-align-loops -fomit-frame-pointer -pipe
-;cxxflags = %(cflags)s
-
-
-[x86_64]
-
-machine = x86_64
-personality = linux64
-
-cflags = -O2 -fomit-frame-pointer -pipe
-cxxflags = %(cflags)s
+++ /dev/null
-.
-.
-.
-.
-.
-HOSTNAME
-.
-
-
+++ /dev/null
-root:x:0:
-bin:x:1:
-sys:x:2:
-kmem:x:3:
-tty:x:4:
-tape:x:5:
-daemon:x:6:
-floppy:x:7:
-disk:x:8:
-lp:x:9:
-dialout:x:10:
-audio:x:11:
-video:x:12:
-utmp:x:13:
-usb:x:14:
-cdrom:x:15:
-messagebus:x:18:
-haldaemon:x:19:
-uucp:x:20:
-fcron:x:22:
-squid:x:23:
-www:x:33:
-mail:x:34:
-ntp:x:38:
-ftp:x:45:
-vsftpd:x:47:
-rsyncd:x:48:
-sshd:x:50:
-nobody:x:99:
-users:x:100:
-snort:x:101:
-logwatch:x:102:
-dnsmasq:x:103:
-cron:x:104:
-postfix:x:107:
-postdrop:x:108:
-clamav:x:109:
-amavis:x:110:
-mldonkey:x:111:
-samba:x:1000:
+++ /dev/null
-order hosts,bind
+++ /dev/null
-# rotate log files weekly
-weekly
-
-# keep 52 weeks worth of backlogs
-rotate 52
-
-# create new (empty) log files after rotating old ones
-create
-
-# uncomment this if you want your log files compressed
-compress
-
-# wtmp
-/var/log/wtmp {
- weekly
- create 0664 root utmp
- rotate 1
-}
-
-/var/log/httpd/access_log /var/log/httpd/error_log /var/log/httpd/ssl_request_log /var/log/httpd/ssl_engine_log {
- missingok
- sharedscripts
- postrotate
- /bin/kill -HUP `cat /var/run/httpd.pid 2>/dev/null` 2> /dev/null || true
- endscript
-}
-
-/var/log/snort/alert {
- weekly
- copytruncate
- compress
- ifempty
- missingok
- postrotate
- /usr/bin/find /var/log/snort -path '/var/log/snort/[0-9]*' -prune -exec /bin/rm -rf {} \;
- /usr/bin/find /var/log/snort -name 'snort.log.*' -mtime +28 -exec /bin/rm -rf {} \;
- /usr/local/bin/restartsnort
- endscript
-}
-
-/var/log/squid/access.log /var/log/squid/user_agent.log /var/log/squid/referer.log {
- weekly
- copytruncate
- ifempty
- missingok
-}
-
-/var/log/squid/cache.log {
- weekly
- rotate 3
- copytruncate
- compress
- missingok
-}
-
-/var/log/squid/store.log {
- weekly
- rotate 3
- copytruncate
- compress
- missingok
- postrotate
- /bin/chmod -R ugo+rX /var/log/squid
- /usr/sbin/squid -k rotate
- endscript
-}
-
-/var/log/messages /var/log/boot.log /var/log/dhcpcd.log {
- create 664 root syslogd
- sharedscripts
- ifempty
- postrotate
- /bin/kill -HUP `cat /var/run/syslogd.pid 2> /dev/null` 2> /dev/null || true
- endscript
-}
-
-/var/log/squidGuard/*.log {
- weekly
- rotate 4
- copytruncate
- compress
- notifempty
- missingok
-}
-
-/var/log/updatexlrator/*.log {
- weekly
- rotate 4
- copytruncate
- compress
- notifempty
- missingok
-}
+++ /dev/null
-# USB HID Settings
-above hid keybdev
-
-# ISDN Settings
-alias char-major-43 hisax
-alias char-major-44 hisax
-alias char-major-45 hisax
-alias ippp0 off
-alias ippp1 off
-
-# PPP Settings
-alias char-major-108 ppp_generic
-alias /dev/ppp ppp_generic
-alias tty-ldisc-3 ppp_async
-alias tty-ldisc-13 n_hdlc
-alias tty-ldisc-14 ppp_synctty
-alias ppp-compress-21 bsd_comp
-alias ppp-compress-24 ppp_deflate
-alias ppp-compress-26 ppp_deflate
-alias net-pf-8 atm
-alias net-pf-24 pppoe
-alias char-major-144 pppox
-alias char-major-166 acm
-
+++ /dev/null
-root:x:0:0:root:/root:/bin/bash
-bin:x:1:1:bin:/bin:/bin/false
-daemon:x:2:2:daemon:/sbin:/bin/false
-mail:x:8:12:mail:/var/spool/mail:/bin/false
-messagebus:x:18:18:D-BUS Message Daemon User:/dev/null:/bin/false
-haldaemon:x:19:19:HAL Daemon User:/dev/null:/bin/false
-fcron:x:22:22:Fcron User:/dev/null:/bin/false
-squid:x:23:23:ftp:/var/spool/squid:/bin/false
-www:x:33:33:Lighttpd:/srv/web/ipfire:/bin/false
-ntp:x:38:38::/etc/ntp:/bin/false
-mysql:x:41:41:MySQL Server:/dev/null:/bin/false
-ftp:x:45:45:anonymous_user:/home/ftp:/bin/false
-vsftpd:x:47:47:vsftpd User:/home/ftp:/bin/false
-rsyncd:x:48:48:rsyncd Daemon:/home/rsync:/bin/false
-sshd:x:50:50:sshd PrivSep:/var/lib/sshd:/bin/false
-nobody:x:99:99:Nobody:/home/nobody:/bin/false
-postfix:x:100:100::/var/spool/postfix:/bin/false
-snort:x:101:101:ftp:/var/log/snort:/bin/false
-logwatch:x:102:102::/var/log/logwatch:/bin/false
-dnsmasq:x:103:103::/:/bin/false
-cron:x:104:104::/:/bin/false
-clamav:x:109:109:Clam AntiVirus:/home/clamav:/bin/false
-amavis:x:110:110:Amavisd-new user:/var/amavis:
-cyrus:x:111:12:Cyrus user:/usr/cyrus:
-filter:x:112:12:Spam user:/home/filter:/bin/false
-mldonkey:x:113:111:Mldonkey user:/opt/mldonkey:/bin/false
-samba:x:1000:1000:Samba User:/var/empty:/bin/false
+++ /dev/null
-
-[architecture]
-
-config = config/architectures.conf
-
-arch = i686
-
-[distro]
-
-; Name of the distribution we build.
-name = IPFire
-sname = ipfire
-
-; Major version number
-epoch = 3
-
-; Minor version string
-version = %(epoch)s.0-prealpha2
-
-; A descriptive slogan
-slogan = "Gluttony"
-
-; The distribution vendor
-vendor = The IPFire Team
-
-
-[sources]
-
-; Where do we get the tarballs from?
-download_url = http://source.ipfire.org/source-3.x
+++ /dev/null
-#!/usr/bin/python
-
-import logging
-import sys
-
-import naoki
-
-# silence Python 2.6 buggy warnings about Exception.message
-if sys.version_info[:2] == (2, 6):
- import warnings
- warnings.filterwarnings(
- action="ignore",
- message="BaseException.message has been deprecated as of Python 2.6",
- category=DeprecationWarning)
-
-# Initialize system
-n = naoki.Naoki()
-
-try:
- # Run...
- n.run()
- exitStatus = 0
-
-except (SystemExit,):
- raise
-
-except (KeyboardInterrupt,):
- exitStatus = 7
- logging.error("Exiting on user interrupt, <CTRL>-C")
-
-sys.exit(exitStatus)
+++ /dev/null
-#!/usr/bin/python
-
-import ConfigParser
-import fcntl
-import logging
-import os.path
-import random
-import sys
-import time
-
-import architectures
-import build
-import dependencies
-import environ
-import generators
-import logger
-import packages
-import repositories
-import terminal
-import util
-
-from constants import *
-
-# Initialize logging
-log = logger.Logging()
-
-class Naoki(object):
- def __init__(self):
- # Second, parse the command line options
- self.cli = terminal.Commandline(self)
-
- logging.debug("Successfully initialized naoki instance")
- for k, v in config.items():
- logging.debug(" %s: %s" % (k, v))
-
- def debug(self, state):
- """
- Enable or disable debugging mode.
- """
-
- # Set logging to debug mode
- log.debug(state)
-
- config.debug = state
-
- def run(self):
- args = self.cli.args
-
- # If there is no action provided, exit
- if not args.has_key("action"):
- self.cli.help()
- return 1
-
- if config["nice_level"]:
- os.nice(int(config["nice_level"]))
-
- actionmap = {
- "build" : self.call_build,
- "package" : self.call_package,
- "source" : self.call_source,
- "shell" : self.call_shell,
- "generate" : self.call_generate,
- "batch" : self.call_batch,
- }
-
- return actionmap[args.action.name](args.action)
-
- def _get_source_repos(self, arch=None):
- if not arch:
- arches = architectures.Architectures()
- arch = arches.get_default()
-
- return repositories.SourceRepositories(arch=arch)
-
- def call_build(self, args):
- # Source repository
- repo = self._get_source_repos()
-
- config["shell_on_failure"] = args.shell
-
- package = repo.find_package_by_name(args.package)
- if not package:
- raise Exception, "Could not find package: %s" % name
-
- # We found a package... build it:
- b = build.Build(package,
- ignore_dependency_errors=args.ignore_dependency_errors)
-
- b.build()
-
- def call_package(self, args):
- if not args.has_key("action"):
- self.cli.help()
- return 1
-
- actionmap = {
- "dependencies" : self.call_package_dependencies,
- "info" : self.call_package_info,
- "list" : self.call_package_list,
- "groups" : self.call_package_groups,
- "raw" : self.call_package_raw,
- "provides" : self.call_package_provides,
- }
-
- return actionmap[args.action.name](args.action)
-
- def call_package_dependencies(self, args):
- repo = self._get_source_repos()
-
- p = repo.find_package_by_name(args.package)
- if not p:
- raise Exception, "Could not find package: %s" % args.package
-
- deps = dependencies.DependencySet()
- for dep in p.get_dependencies() + p.get_dependencies("build"):
- deps.add_dependency(dep)
-
- deps.resolve()
-
- identifiers = set([d.identifier for d in deps.dependencies])
-
- for identifier in sorted(identifiers):
- print identifier
-
- def call_package_info(self, args):
- # Source repositories
- repo = self._get_source_repos()
-
- for package in repo.packages:
- if args.packages:
- if not package.name in args.packages:
- continue
-
- if args.long:
- print package.fmtstr("""\
---------------------------------------------------------------------------------
-Name : %(PKG_NAME)s
-Version : %(PKG_VER)s
-Release : %(PKG_REL)s
-
- %(PKG_SUMMARY)s
-
-%(PKG_DESCRIPTION)s
-
-Maintainer : %(PKG_MAINTAINER)s
-License : %(PKG_LICENSE)s
-
-Objects : %(PKG_OBJECTS)s
-Patches : %(PKG_PATCHES)s
---------------------------------------------------------------------------------\
-""")
- else:
- print package.fmtstr("""\
---------------------------------------------------------------------------------
-Name : %(PKG_NAME)s
-Version : %(PKG_VER)s
-Release : %(PKG_REL)s
-
- %(PKG_SUMMARY)s
-
---------------------------------------------------------------------------------\
-""")
-
- def call_package_list(self, args):
- repo = self._get_source_repos()
-
- for package in repo.packages:
- # Skip unbuilt packages if we want built packages
- if args.built and not package.is_built:
- continue
-
- # Skip built packages if we want unbuilt only
- if args.unbuilt and package.is_built:
- continue
-
- if args.long:
- print package.fmtstr("%(PKG_NAME)-32s | %(PKG_VER)-15s | %(PKG_SUMMARY)s")
- else:
- print package.name
-
- def call_package_groups(self, args):
- groups = []
-
- repo = self._get_source_repos()
-
- for package in repo.packages:
- group = package.group
- if not group in groups:
- groups.append(group)
-
- print "\n".join(sorted(groups))
-
- def call_package_raw(self, args):
- filename = args.package
-
- if os.path.exists(filename):
- p = packages.BinaryPackage(filename)
- else:
- repo = self._get_source_repos()
-
- p = repo.find_package_by_name(args.package)
- if not p:
- raise Exception, "Could not find package: %s" % args.package
-
- p.print_raw_info()
-
- def call_package_provides(self, args):
- provides = dependencies.Provides(args.provides)
-
- arch = architectures.Architectures().get_default()
- repo = repositories.BinaryRepository(arch)
-
- for package in repo.find_packages_by_provides(provides):
- print package.fmtstr("%(PKG_NAME)s %(PKG_VER)s-%(PKG_REL)s %(PKG_SUMMARY)s")
-
- def call_source(self, args):
- if not args.has_key("action"):
- self.cli.help()
- sys.exit(1)
-
- actionmap = {
- "download" : self.call_source_download,
- "upload" : self.call_source_upload,
- }
-
- return actionmap[args.action.name](args.action)
-
- def call_source_download(self, args):
- repo = self._get_source_repos()
-
- for package in repo.packages:
- if args.packages:
- if not package.name in args.packages:
- continue
-
- package.source_download()
-
- def call_source_upload(self, args):
- pass # TODO
-
- def call_shell(self, args):
- # Load architecture set
- arches = architectures.Architectures()
-
- # Choose default architecture
- arch = arches.get_default()
-
- # Load all source packages
- repo = repositories.SourceRepositories(arch=arch)
-
- # Pick the one we need
- p = repo.find_package_by_name(args.package)
- if not p:
- raise Exception, "Could not find package: %s" % args.package
-
- # Initialize and run the shell
- shell = build.PackageShell(p)
-
- return shell.shell()
-
- def call_generate(self, args):
- if not args.type in ("iso",):
- return
-
- arch = architectures.Architectures().get_default()
-
- gen = generators.Generator(args.type, arch)
- return gen.run()
-
- def call_batch(self, args):
- try:
- self._lock = open(LOCK_BATCH, "a+")
- except IOError, e:
- return 0
-
- try:
- fcntl.lockf(self._lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
- except IOError, e:
- print >>sys.stderr, "Batch operations are locked by another process"
- return 0
-
- actionmap = {
- "cron" : self.call_batch_cron,
- }
-
- return actionmap[args.action.name](args.action)
-
- def call_batch_cron(self, args):
- pkgs = []
- candidates = []
-
- # Choose architecture
- arches = architectures.Architectures()
- arch = arches.get_default()
-
- repo = repositories.SourceRepositories(arch=arch)
- for package in repo.packages:
- if not package.is_built:
- pkgs.append(package)
- else:
- candidates.append(package)
-
- # Initialize a job queue
- jobs = build.Jobs()
-
- # Add all unbuilt packages to the job queue
- for package in pkgs:
- package = build.Build(package)
- jobs.add(package)
-
- # If we have less than ten packages in the queue we add some random
- # ones
- need_counter = 10 - len(jobs)
- if need_counter >= 0:
- for candidate in random.sample(candidates, need_counter):
- candidate = build.Build(candidate)
- jobs.add(candidate)
-
- while jobs.has_jobs:
- jobs.process_next()
+++ /dev/null
-#!/usr/bin/python
-
-import logging
-import os.path
-
-from ConfigParser import ConfigParser, DEFAULTSECT
-
-from constants import *
-from decorators import *
-
-class Architecture(object):
- def __init__(self, name, **settings):
-
- self._settings = {
- "default" : False,
- "name" : name,
- }
- self._settings.update(settings)
-
- logging.debug("Set up new architecture: %s" % self._settings)
-
- def __repr__(self):
- s = "<%s %s" % (self.__class__.__name__, self._settings["name"])
- if self.default:
- s += " (default)"
- s += ">"
- return s
-
- @property
- def buildable(self):
- """ Check if this architecture is buildable on the local host"""
-
- return True # TODO
-
- def get_default(self):
- return self._settings["default"]
-
- def set_default(self, value):
- self._settings["default"] = value
-
- default = property(get_default, set_default)
-
- def __getattr__(self, attr):
- try:
- return self._settings[attr]
- except KeyError:
- raise AttributeError, attr
-
-
-@singleton
-class Architectures(object):
- def __init__(self):
- self._architectures = []
-
- # Try to read the default architectures
- self.read(ARCHES_DEFAULT)
-
- def read(self, filename):
- logging.debug("Reading architectures from %s" % filename)
-
- if not os.path.exists(filename):
- return
-
- p = ConfigParser()
- p.read(filename)
-
- default = {}
-
- for arch in p.sections():
- if arch == "default":
- default = p.items(arch)
- continue
-
- settings = {}
- for key, val in p.items(arch):
- settings[key] = val
-
- a = Architecture(arch, **settings)
- self.add_architecture(a)
-
- for key, val in default:
- if key == "default":
- self.default = val
-
- def add_architecture(self, arch):
- assert isinstance(arch, Architecture)
-
- self._architectures.append(arch)
-
- def get_default(self):
- for a in self._architectures:
- if a.default:
- return a
-
- raise Exception, "Cannot find default architecture"
-
- def set_default(self, arch):
- if arch is None:
- return
-
- if not arch in [a.name for a in self.all]:
- raise Exception, "Cannot set default architecture: Unknown architecture %s" % arch
-
- logging.debug("Setting default architecture: %s" % arch)
-
- for a in self._architectures:
- if a.name == arch:
- a.default = True
- else:
- a.default = False
-
- default = property(get_default, set_default)
-
- def get(self, name):
- for arch in self._architectures:
- if arch.name == name:
- return arch
-
- raise Exception, "Could not find arch: %s" % name
-
- @property
- def all(self):
- return self._architectures[:]
-
-
-if __name__ == "__main__":
- arches = Architectures()
-
- for arch in arches.all:
- print "Name: %s" % arch.name
- for key, val in arch._settings.items():
- print " %-20s : %s" % (key, val)
-
- print arches.get("i686")
- print arches.default
-
+++ /dev/null
-#!/usr/bin/python
-
-import logging
-import uuid
-
-import dependencies
-import environ
-
-from constants import *
-from exception import *
-
-
-class Build(object):
- def __init__(self, package, **kwargs):
- self.package = package
-
- # Generate a random, but unique id
- self.id = uuid.uuid4()
-
- # Create dependency set
- self.dependency_set = dependencies.DependencySet(arch=self.arch)
-
- # Add all mandatory packages and build dependencies
- deps = [dependencies.Dependency(p) for p in config["mandatory_packages"]]
- deps += self.package.get_dependencies("build")
- for package in deps:
- self.dependency_set.add_dependency(package)
-
- self.settings = {
- "ignore_dependency_errors" : False,
- }
-
- self.settings.update(kwargs)
-
- def __repr__(self):
- return "<%s %s-%s:%s>" % \
- (self.__class__.__name__, self.id, self.package.name, self.arch.name)
-
- @property
- def arch(self):
- return self.package.arch
-
- def build(self, **settings):
- self.settings.update(settings)
-
- logging.info("Building: %s (%s)" % (self.package.id, self.id))
- logging.info("")
- logging.info(" %s" % self.package.summary)
- logging.info("")
-
- # Download the source tarballs
- self.package.source_download()
-
- # Resolve the dependencies
- try:
- self.dependency_set.resolve()
- except DependencyResolutionError, e:
- if self.settings["ignore_dependency_errors"]:
- logging.warning("Ignoring dependency errors: %s" % e)
- else:
- raise
-
- e = environ.Build(self.package, build_id="%s" % self.id)
-
- # Extract all tools
- for package in self.dependency_set.packages:
- e.extract(package)
-
- # Build :D
- e.build()
-
-
-class PackageShell(Build):
- def __init__(self, *args, **kwargs):
- Build.__init__(self, *args, **kwargs)
-
- # Add shell packages to have a little bit more
- # comfort in here...
- for dependency in config["shell_packages"]:
- dependency = dependencies.Dependency(dependency)
- self.dependency_set.add_dependency(dependency)
-
- def shell(self, **settings):
- # Resolve the dependencies
- try:
- self.dependency_set.resolve()
- except DependencyResolutionError, e:
- if self.settings["ignore_dependency_errors"]:
- logging.warning("Ignoring dependency errors: %s" % e)
- else:
- raise
-
- e = environ.Build(self.package, build_id="%s" % self.id)
-
- # Extract all tools
- for package in self.dependency_set.packages:
- e.extract(package)
-
- # Download the source tarballs
- self.package.source_download()
-
- # Preparing source...
- e.make("prepare")
-
- # Run the shell
- e.shell()
+++ /dev/null
-#!/usr/bin/python
-
-from ConfigParser import ConfigParser, DEFAULTSECT
-import math
-import os
-import socket
-import sys
-
-BASEDIR = os.getcwd()
-
-BUILDDIR = os.path.join(BASEDIR, "build")
-CACHEDIR = os.path.join(BASEDIR, "cache")
-CCACHEDIR = os.path.join(BASEDIR, "ccache")
-CONFIGDIR = os.path.join(BASEDIR, "config")
-DOCDIR = os.path.join(BASEDIR, "doc")
-GENDIR = os.path.join(BUILDDIR, "generators")
-IMAGESDIR = os.path.join(BUILDDIR, "images")
-LOGDIR = os.path.join(BASEDIR, "logs")
-PKGSDIR = os.path.join(BASEDIR, "pkgs")
-PACKAGESDIR = os.path.join(BUILDDIR, "packages")
-REPOSDIR = os.path.join(BUILDDIR, "repositories")
-TOOLSDIR = os.path.join(BASEDIR, "tools")
-
-ARCHES_DEFAULT = os.path.join(CONFIGDIR, "architectures.conf")
-
-CONFIGFILE = os.path.join(CONFIGDIR, "naoki.conf")
-
-CHROOT_PATH = "/sbin:/bin:/usr/sbin:/usr/bin"
-
-LOCK_BATCH = os.path.join(BUILDDIR, ".batch")
-
-LOG_MARKER = "### LOG MARKER ###"
-
-DEP_INVALID, DEP_FILE, DEP_LIBRARY, DEP_PACKAGE = range(4)
-
-def calc_parallelism():
- """
- Calculate how many processes to run
- at the same time.
-
- We take the log10(number of processors) * factor
- """
- num = os.sysconf("SC_NPROCESSORS_CONF")
- if num == 1:
- return 2
- else:
- return int(round(math.log10(num) * 26))
-
-class Config(object):
- _items = {
- "toolchain" : False,
- "mandatory_packages" : [
- "build-essentials",
- ],
- "shell_packages" : [
- "/bin/bash",
- "less",
- "vim",
- ],
- "nice_level" : 0,
- "parallelism" : calc_parallelism(),
- #
- # Cleanup settings
- "cleanup_on_failure" : False,
- "cleanup_on_success" : True,
- #
- # Shell settings
- "shell_on_failure" : False,
- #
- # CLI variables
- "debug" : False,
- "quiet" : False,
- #
- # Distro items
- "distro_name" : "unknown",
- "distro_sname" : "unknown",
- "distro_epoch" : "unknown",
- "distro_version" : "unknown",
- "distro_slogan" : "unknown",
- "disto_vendor" : "unknown",
- #
- # Logging
- "log_file" : os.path.join(LOGDIR, "naoki.log"),
- #
- # Reporting
- "error_report_recipient" : None,
- "error_report_sender" : "buildsystem@%s" % socket.gethostname(),
- "error_report_subject" : "[NAOKI] %(id)s got a build failure",
- #
- # SMTP
- "smtp_server" : None,
- "smtp_user" : None,
- "smtp_password" : None,
- }
-
- def __init__(self):
- self.read([CONFIGFILE, os.path.join(BASEDIR, ".config")])
-
- def read(self, files):
- parser = ConfigParser()
- parser.read(files)
-
- config = {}
- for key, val in parser.items(DEFAULTSECT):
- config[key] = val
-
- for section in parser.sections():
- for key, val in parser.items(section):
- config["%s_%s" % (section, key)] = val
-
- self._items.update(config)
-
- def items(self):
- return self._items.items()
-
- def __getitem__(self, item):
- return self._items[item]
-
- def __setitem__(self, item, value):
- self._items[item] = value
-
- def __getattr__(self, *args):
- return self.__getitem__(*args)
-
- def __setattr__(self, *args):
- return self.__setitem__(*args)
-
- @property
- def environment(self):
- ret = {
- "HOME" : os.environ.get("HOME", "/root"),
- "TERM" : os.environ.get("TERM", ""),
- "PS1" : "\u:\w\$ ",
- #
- "DISTRO_NAME" : self["distro_name"],
- "DISTRO_SNAME" : self["distro_sname"],
- "DISTRO_EPOCH" : self["distro_epoch"],
- "DISTRO_VERSION" : self["distro_version"],
- "DISTRO_SLOGAN" : self["distro_slogan"],
- "DISTRO_VENDOR" : self["distro_vendor"],
- #
- "PARALLELISMFLAGS" : "-j%d" % self["parallelism"],
- }
-
- if self["debug"]:
- ret["NAOKI_DEBUG"] = "1"
-
- return ret
-
-
-# Create a globally useable instance of the configuration
-config = Config()
-
+++ /dev/null
-#!/usr/bin/python
-
-# A decorator to use singleton on a class
-# http://en.wikipedia.org/wiki/Singleton_pattern#Python_.28using_decorators.29
-def singleton(cls):
- instance_container = []
- def getinstance():
- if not len(instance_container):
- instance_container.append(cls())
- return instance_container[0]
- return getinstance
-
+++ /dev/null
-#!/usr/bin/python
-
-import logging
-import os
-import re
-
-import architectures
-import repositories
-import packages
-
-from constants import *
-from exception import *
-
-class Dependency(object):
- def __init__(self, identifier, origin=None):
- self.identifier = identifier
- self.origin = origin
-
- def __repr__(self):
- s = "<%s %s" % (self.__class__.__name__, self.identifier)
- if self.origin:
- s += " by %s" % self.origin.name
- s += "(%s)" % os.path.basename(self.origin.filename)
- s += ">"
- return s
-
- @property
- def type(self):
- if self.identifier.startswith("/"):
- return DEP_FILE
- elif ".so" in self.identifier:
- return DEP_LIBRARY
- elif re.match("^[A-Za-z0-9\-\+_]+$", self.identifier):
- return DEP_PACKAGE
-
- return DEP_INVALID
-
- def match(self, other):
- return self.type == other.type \
- and self.identifier == other.identifier
-
- __eq__ = match
-
-
-class Provides(Dependency):
- pass
-
-
-class DependencySet(object):
- def __init__(self, dependencies=[], arch=None):
- if not arch:
- arches = architectures.Architectures()
- arch = arches.get_default()
- self.arch = arch
-
- self.repo = repositories.BinaryRepository(self.arch)
-
- # initialize package lists
- self._dependencies = []
- self._items = []
-
- # caches
- self.__provides = {}
- self.__dependencies = {}
-
- # add all provided dependencies
- for dependency in dependencies:
- self.add_dependency(dependency)
-
- logging.debug("Successfully initialized %s" % self)
-
- @property
- def hash(self):
- hash = ["%s" % i for i in self._items]
-
- return "".join(hash)
-
- def __repr__(self):
- return "<%s>" % (self.__class__.__name__)
-
- def add_dependency(self, item):
- assert isinstance(item, Dependency)
-
- # If this dependency is already provided by something else
- # we skip the add
- for p in self.provides:
- if p.match(item):
- logging.debug("%s matches %s" % (p, item))
- return
-
- if not item in self.dependencies:
- self._dependencies.append(item)
-
- logging.debug("Added new dependency %s" % item)
-
- def add_package(self, item):
- assert isinstance(item, packages.BinaryPackage)
-
- if item in self._items:
- return
-
- # Packages conflict
- for package in self._items:
- if package.name == item.name:
- if item > package:
- logging.debug("Replacing package %s by %s" % (package, item))
- self._items.remove(package)
- break
-
- self._items.append(item)
- logging.debug("Added new package %s" % item)
-
- def resolve(self):
- logging.debug("Resolving %s" % self)
-
- # Safe for endless loop
- counter = 1000
-
- while True:
- counter -= 1
- if not counter:
- logging.debug("Maximum count of dependency loop was reached")
- break
-
- dependency = self.next_unresolved_dependency
- if not dependency:
- break
-
- logging.debug("Processing dependency: %s" % dependency.identifier)
-
- if dependency.type == DEP_PACKAGE:
- package = self.repo.find_package_by_name(dependency.identifier)
- if package:
- # Found a package and add this
- self.add_package(package)
- continue
-
- elif dependency.type == DEP_LIBRARY:
- package = self.repo.find_package_by_provides(dependency)
- if package:
- self.add_package(package)
- continue
-
- elif dependency.type == DEP_FILE:
- package = self.repo.find_package_by_file(dependency.identifier)
- if package:
- self.add_package(package)
- continue
-
- logging.warning("Cannot find a package which provides file %s" % \
- dependency.identifier)
-
- elif dependency.type == DEP_INVALID:
- logging.warning("Dropping invalid dependency %s" % dependency.identifier)
- continue
-
- # Found not solution
- logging.debug("No match found: %s" % dependency)
-
- if self.unresolved_dependencies:
- #logging.error("Unresolveable dependencies: %s" % self.dependencies)
- raise DependencyResolutionError, "%s" % self.unresolved_dependencies
-
- @property
- def unresolved_dependencies(self):
- return self.calculate_unresolved_dependencies(next=False)
-
- @property
- def next_unresolved_dependency(self):
- d = self.calculate_unresolved_dependencies(next=True)
- if d:
- return d[0]
-
- def calculate_unresolved_dependencies(self, next=False):
- dependencies = []
-
- # Cache provides
- provides = self.provides
-
- for dependency in self._dependencies + self.dependencies:
- if dependency.type == DEP_INVALID:
- continue
-
- found = False
- for item in self._items:
- if item.does_provide(dependency):
- found = True
- break
-
- if found:
- continue
-
- #for provide in provides:
- # if dependency.match(provide):
- # found = True
- # break
- #
- #if found:
- # continue
-
- dependencies.append(dependency)
-
- # If next is set return only one.
- if next:
- return dependencies
-
- return dependencies
-
- @property
- def dependencies(self):
- if not self.__dependencies.has_key(self.hash):
- self.__dependencies[self.hash] = self.calculate_dependencies()
-
- return self.__dependencies[self.hash]
-
- def calculate_dependencies(self):
- dependencies = []
- for item in self._items:
- dependencies += item.get_dependencies()
-
- return list(set(dependencies))
-
- @property
- def packages(self):
- return sorted(self._items)
-
- def calculate_provides(self):
- provides = []
- for item in self._items:
- provides += item.get_provides()
-
- return list(set(provides))
-
- @property
- def provides(self):
- if not self.__provides.has_key(self.hash):
- self.__provides[self.hash] = self.calculate_provides()
-
- return self.__provides[self.hash]
-
-
-if __name__ == "__main__":
- import architectures
- arches = architectures.Architectures()
-
- ds = DependencySet(arch=arches.get("i686"))
- ds.add_dependency(Dependency("/bin/bash"))
- ds.resolve()
- print ds.packages
+++ /dev/null
-#!/usr/bin/python
-
-import grp
-import logging
-import os
-import stat
-import time
-
-import logger
-import mail
-import util
-
-from constants import *
-
-# XXX to be moved to somewhere else
-ENVIRONMENT_ARGS = ["PATH", "PWD" ]
-
-def set(e):
- env = {}
-
- for key in ENVIRONMENT_ARGS:
- if os.environ.has_key(key):
- env[key] = os.environ[key]
-
- env.update(config.environment)
- env.update(e)
-
- return env
-
-
-class _Environment(object):
- kernel_version = os.uname()[2]
-
- def __init__(self, arch, **settings):
- self.arch = arch
-
- self._settings = {
- "enable_loop" : False,
- }
- self._settings.update(settings)
-
- self.logger.debug("Successfully initialized %s" % self)
-
- # XXX check if already locked
- self.prepare()
-
- def chrootPath(self, *args):
- raise NotImplementedError
-
- def clean(self):
- self.logger.debug("Cleaning environment %s" % self)
- if os.path.exists(self.chrootPath()):
- util.rm(self.chrootPath())
-
- def prepare(self):
- self.clean()
-
- self.logger.debug("Preparing environment %s" % self)
-
- dirs = (
- CACHEDIR,
- CCACHEDIR,
- IMAGESDIR,
- PACKAGESDIR,
- "dev",
- "dev/pts",
- "dev/shm",
- "proc",
- "root",
- "sys",
- "tmp",
- "usr/src",
- "usr/src/cache",
- "usr/src/ccache",
- "usr/src/images",
- "usr/src/packages",
- "usr/src/pkgs",
- "usr/src/src",
- "usr/src/tools",
- "var/tmp",
- )
-
- for dir in dirs:
- util.mkdir(self.chrootPath(dir))
-
- files = (
- "etc/fstab",
- "etc/mtab"
- )
-
- for file in files:
- file = self.chrootPath(file)
- dir = os.path.dirname(file)
- if not os.path.exists(dir):
- util.mkdir(dir)
- util.touch(file)
-
- # Prepare the other stuff
- self._prepare_dev()
- self._prepare_users()
- self._prepare_dns()
-
- def _prepare_dev(self):
- prevMask = os.umask(0000)
-
- devNodes = [
- (stat.S_IFCHR | 0666, os.makedev(1, 3), "dev/null"),
- (stat.S_IFCHR | 0666, os.makedev(1, 7), "dev/full"),
- (stat.S_IFCHR | 0666, os.makedev(1, 5), "dev/zero"),
- (stat.S_IFCHR | 0666, os.makedev(1, 8), "dev/random"),
- (stat.S_IFCHR | 0444, os.makedev(1, 9), "dev/urandom"),
- (stat.S_IFCHR | 0666, os.makedev(5, 0), "dev/tty"),
- (stat.S_IFCHR | 0600, os.makedev(5, 1), "dev/console")
- ]
-
- if self._settings["enable_loop"]:
- for i in range(0, 7):
- devNodes.append((stat.S_IFBLK | 0660, os.makedev(7, i), "dev/loop%d" % i))
-
- # make device node for el4 and el5
- if self.kernel_version < "2.6.19":
- devNodes.append((stat.S_IFCHR | 0666, os.makedev(5, 2), "dev/ptmx"))
-
- for i in devNodes:
- # create node
- os.mknod(self.chrootPath(i[2]), i[0], i[1])
-
- os.symlink("/proc/self/fd/0", self.chrootPath("dev", "stdin"))
- os.symlink("/proc/self/fd/1", self.chrootPath("dev", "stdout"))
- os.symlink("/proc/self/fd/2", self.chrootPath("dev", "stderr"))
- os.symlink("/proc/self/fd", self.chrootPath("dev", "fd"))
-
- if self.kernel_version >= "2.6.19":
- os.symlink("/dev/pts/ptmx", self.chrootPath("dev", "ptmx"))
-
- os.umask(prevMask)
-
- def _prepare_users(self):
- f = open(self.chrootPath("etc", "passwd"), "w")
- f.write("root:x:0:0:root:/root:/bin/bash\n")
- f.write("nobody:x:99:99:Nobody:/:/sbin/nologin\n")
- f.close()
-
- f = open(self.chrootPath("etc", "group"), "w")
- f.write("root:x:0:root\n")
- f.write("nobody:x:99:\n")
- f.close()
-
- def _prepare_dns(self):
- # XXX to be replaced
- nameservers = []
- f = open("/etc/resolv.conf")
- for line in f.readlines():
- if line.startswith("nameserver"):
- nameservers.append(line.split(" ")[-1].strip())
- f.close()
-
- self.logger.debug("Using nameservers: %s" % nameservers)
-
- f = open(self.chrootPath("etc", "resolv.conf"), "w")
- for nameserver in nameservers:
- f.write("nameserver %s" % nameserver)
- f.close()
-
- self.logger.debug("Creating record for localhost")
- f = open(self.chrootPath("etc", "hosts"), "w")
- f.write("127.0.0.1 localhost\n")
- f.close()
-
- def extract(self, package, *args):
- self.logger.info("Extracting %s" % package)
-
- package.extract(self.chrootPath(*args))
-
- def variables(self):
- env = set({
- "HOME" : "/root",
- "PATH" : "/sbin:/bin:/usr/sbin:/usr/bin",
- "BASEDIR" : "/usr/src",
- "PKGROOT" : "/usr/src/pkgs",
- "TARGET" : "%s-ipfire-linux-gnu" % self.arch.machine,
- "TARGET_MACHINE" : self.arch.machine,
- "CHROOT" : "1", # XXX to be removed
- "CFLAGS" : self.arch.cflags,
- "CXXFLAGS" : self.arch.cxxflags,
- "PKG_ARCH" : self.arch.name,
- })
-
- # Settings for ccache
- env.update({
- "PATH" : "/usr/ccache/bin:%s" % env["PATH"],
- "CCACHE_COMPILERCHECK" : "none",
- "CCACHE_COMPRESS" : "1",
- "CCACHE_DIR" : "/usr/src/ccache",
- })
-
- return env
-
- def doChroot(self, command, shell=True, *args, **kwargs):
- ret = None
- try:
- env = self.variables()
-
- if kwargs.has_key("env"):
- env.update(kwargs.pop("env"))
-
- self._mountall()
-
- if not kwargs.has_key("chrootPath"):
- kwargs["chrootPath"] = self.chrootPath()
-
- ret = util.do(command,
- personality=self.arch.personality,
- shell=shell,
- env=env,
- logger=self.logger,
- *args, **kwargs)
-
- finally:
- self._umountall()
-
- return ret
-
- def _mountall(self):
- self.logger.debug("Mounting environment")
- for cmd in self.mountCmds:
- util.do(cmd, shell=True)
-
- def _umountall(self):
- self.logger.debug("Umounting environment")
- for cmd in self.umountCmds:
- util.do(cmd, raiseExc=0, shell=True)
-
- @property
- def umountCmds(self):
- return (
- "umount -n %s" % self.chrootPath("proc"),
- "umount -n %s" % self.chrootPath("sys"),
- "umount -n %s" % self.chrootPath("usr", "src", "cache"),
- "umount -n %s" % self.chrootPath("usr", "src", "ccache"),
- "umount -n %s" % self.chrootPath("usr", "src", "images"),
- "umount -n %s" % self.chrootPath("usr", "src", "packages"),
- "umount -n %s" % self.chrootPath("usr", "src", "pkgs"),
- "umount -n %s" % self.chrootPath("usr", "src", "src"),
- "umount -n %s" % self.chrootPath("usr", "src", "tools"),
- "umount -n %s" % self.chrootPath("dev", "pts"),
- "umount -n %s" % self.chrootPath("dev", "shm")
- )
-
- @property
- def mountCmds(self):
- ret = [
- "mount -n -t proc naoki_chroot_proc %s" % self.chrootPath("proc"),
- "mount -n -t sysfs naoki_chroot_sysfs %s" % self.chrootPath("sys"),
- "mount -n --bind %s %s" % (CACHEDIR, self.chrootPath("usr", "src", "cache")),
- "mount -n --bind %s %s" % (CCACHEDIR, self.chrootPath("usr", "src", "ccache")),
- "mount -n --bind %s %s" % (IMAGESDIR, self.chrootPath("usr", "src", "images")),
- "mount -n --bind %s %s" % (PACKAGESDIR, self.chrootPath("usr", "src", "packages")),
- "mount -n --bind %s %s" % (PKGSDIR, self.chrootPath("usr", "src", "pkgs")),
- "mount -n --bind %s %s" % (os.path.join(BASEDIR, "src"), self.chrootPath("usr", "src", "src")),
- "mount -n --bind %s %s" % (TOOLSDIR, self.chrootPath("usr", "src", "tools")),
- ]
-
- mountopt = "gid=%d,mode=0620,ptmxmode=0666" % grp.getgrnam("tty").gr_gid
- if self.kernel_version >= "2.6.29":
- mountopt += ",newinstance"
-
- ret.extend([
- "mount -n -t devpts -o %s naoki_chroot_devpts %s" % (mountopt, self.chrootPath("dev", "pts")),
- "mount -n -t tmpfs naoki_chroot_shmfs %s" % self.chrootPath("dev", "shm")])
-
- return ret
-
- @property
- def logger(self):
- return logging.getLogger()
-
-
-class Build(_Environment):
- def __init__(self, package, build_id=None):
- self.package = package
- self.build_id = build_id
-
- _Environment.__init__(self, self.package.arch)
-
- @property
- def logger(self):
- if not hasattr(self, "_logger"):
- logfile = os.path.join(LOGDIR, self.package.repository.name,
- self.package.name, self.package.id + ".log")
-
- # Ensure that logging directory exists
- util.mkdir(os.path.dirname(logfile))
-
- self._logger = logging.getLogger(self.package.id)
-
- fh = logging.FileHandler(logfile)
- fh.setFormatter(logging.Formatter("%(message)s"))
- fh.setLevel(logging.NOTSET)
-
- self._logger.addHandler(fh)
-
- return self._logger
-
- def chrootPath(self, *args):
- return os.path.join(BUILDDIR, "environments", self.package.id, *args)
-
- @property
- def buildroot(self):
- if not hasattr(self, "__buildroot"):
- self.__buildroot = "buildroot.%s" % util.random_string()
-
- return self.__buildroot
-
- def variables(self):
- v = _Environment.variables(self)
- v.update({
- "BUILDROOT" : "/%s" % self.buildroot,
- "BUILD_ID" : self.build_id or "",
- })
- return v
-
- def build(self, *args, **kwargs):
- # Save starting time
- time_start = time.time()
-
- try:
- self.make("package")
- except:
- if config["shell_on_failure"]:
- self.shell()
- return
-
- if config["cleanup_on_failure"]:
- self.clean()
-
- # Send email report about an error
- mail.report_error(self.package)
- raise
-
- # Log time that was needed to build the package
- self.logger.info("Build time: %.2fs" % (time.time() - time_start))
-
- if config["cleanup_on_success"]:
- self.clean()
-
- def make(self, target):
- file = "/usr/src%s" % self.package.filename[len(BASEDIR):]
-
- return self.doChroot("make -C %s -f %s %s" % \
- (os.path.dirname(file), file, target), shell=True)
-
- def shell(self, args=[]):
- command = "%s chroot %s /usr/src/tools/chroot-shell %s" % \
- (self.arch.personality, self.chrootPath(), " ".join(args))
-
- for key, val in self.variables().items():
- command = "%s=\"%s\" " % (key, val) + command
-
- if self.package.source_dir:
- command = "SOURCE_DIR=%s %s" % (self.package.source_dir, command)
-
- logging.debug("Shell command: %s" % command)
-
- try:
- self._mountall()
-
- shell = os.system(command)
- return os.WEXITSTATUS(shell)
-
- finally:
- self._umountall()
-
- # Clean up the environment
- self.clean()
+++ /dev/null
-#!/usr/bin/python
-
-class Error(Exception):
- "base class for our errors."
- def __init__(self, msg, status=None):
- Exception.__init__(self)
- self.msg = msg
- self.resultcode = 1
- if status is not None:
- self.resultcode = status
-
- def __str__(self):
- return self.msg
-
-
-class BuildRootLocked(Error):
- "build root in use by another process."
- def __init__(self, msg):
- Error.__init__(self, msg)
- self.msg = msg
- self.resultcode = 60
-
-
-class commandTimeoutExpired(Error):
- def __init__(self, msg):
- Error.__init__(self, msg)
- self.msg = msg
- self.resultcode = 10
-
-
-class DownloadError(Error):
- pass
-
-
-class DependencyResolutionError(Error):
- pass
+++ /dev/null
-#!/usr/bin/python
-
-import logging
-import os
-
-import dependencies
-import repositories
-import util
-
-from constants import *
-from environ import _Environment
-
-
-class Generator(_Environment):
- def __init__(self, type, arch=None):
- _Environment.__init__(self, arch, enable_loop=True)
-
- self.type = type
-
- self.repos = repositories.SourceRepositories(self.arch)
-
- for r in self.repos.all:
- if not r.completely_built:
- raise Exception, "The repo is not built completely: %s" % r
-
- self.build_deps = dependencies.DependencySet(arch=self.arch)
- deps = [
- "basesystem",
- "dracut",
- "e2fsprogs",
- "kernel",
- "squashfs-tools",
- "syslinux",
- "util-linux-ng",
- "zerofree",
- "/sbin/dmsetup",
- "/usr/bin/mkisofs",
- ]
- for dep in deps:
- dep = dependencies.Dependency(dep)
- self.build_deps.add_dependency(dep)
-
- self.installer_deps = dependencies.DependencySet(arch=self.arch)
- deps = [
- "basesystem",
- "installer",
- # TODO needs to be replaced
- "xorg-x11-drv-ati",
- "xorg-x11-drv-evdev",
- "xorg-x11-drv-intel",
- "xorg-x11-drv-keyboard",
- "xorg-x11-drv-mouse",
- "xorg-x11-drv-nv",
- "xorg-x11-drv-synaptics",
- "xorg-x11-drv-vesa",
- "xorg-x11-drv-vmware",
- ]
- for dep in deps:
- dep = dependencies.Dependency(dep)
- self.installer_deps.add_dependency(dep)
-
- def chrootPath(self, *args):
- return os.path.join(GENDIR, self.arch.name, *args)
-
- @property
- def logger(self):
- return logging.getLogger()
-
- def run(self):
- # Unpacking packages we need in this environment
- logging.info("Resolving dependencies...")
- self.build_deps.resolve()
- self.installer_deps.resolve()
-
- for package in self.build_deps.packages:
- self.extract(package)
-
- util.mkdir(self.chrootPath("installer"))
- for package in self.installer_deps.packages:
- self.extract(package, "installer")
-
- util.mkdir(self.chrootPath("packages"))
- for p in self.repos.packages:
- for bin in p.binary_files:
- os.link(os.path.join(PACKAGESDIR, self.arch.name, bin),
- self.chrootPath("packages", bin))
-
- self.doChroot("/usr/src/tools/generator %s" % self.type, shell=True)
+++ /dev/null
-#!/usr/bin/python
-
-import grp
-import os
-import pwd
-import stat
-import time
-
-def ftype(mode):
- if stat.S_ISBLK(mode):
- return "b"
- elif stat.S_ISCHR(mode):
- return "c"
- elif stat.S_ISDIR(mode):
- return "d"
- elif stat.S_ISREG(mode):
- return "-"
- elif stat.S_ISFIFO(mode):
- return "p"
- elif stat.S_ISLINK(mode):
- return "l"
- elif stat.S_ISSOCK(mode):
- return "s"
- return "?"
-
-def rwx(mode):
- ret = ""
- if mode & stat.S_IRUSR:
- ret += "r"
- else:
- ret += "-"
-
- if mode & stat.S_IWUSR:
- ret += "w"
- else:
- ret += "-"
-
- if mode & stat.S_IXUSR:
- ret += "x"
- else:
- ret += "-"
-
- return ret
-
-def fmode(mode):
- ret = ftype(mode)
- ret += rwx((mode & 0700) << 0)
- ret += rwx((mode & 0070) << 3)
- ret += rwx((mode & 0007) << 6)
- return ret
-
-class CpioError(Exception):
- pass
-
-
-class CpioEntry(object):
- def __init__(self, hdr, archive, offset):
- self.archive = archive
- self.hdr = hdr
-
- self.offset = offset + 110 + self.namesize
- self.offset += (4 - (self.offset % 4)) % 4
- self.current = 0
-
- self.closed = False
-
- if len(self.hdr) < 110:
- raise CpioError("Header too short.")
-
- if not self.hdr.startswith("070701") and not self.hdr.startswith("070702"):
- raise CpioError("Invalid header: %s" % self.hdr[:6])
-
- def close(self):
- self.closed = True
-
- def flush(self):
- pass # noop
-
- def read(self, size=None):
- """Read data from the entry.
-
- Keyword arguments:
- size -- Number of bytes to read (default: whole entry)
- """
- if self.closed:
- raise ValueError("Read operation on closed file.")
-
- self.archive.file.seek(self.offset + self.current, os.SEEK_SET)
-
- if size and size < self.size - self.current:
- ret = self.archive.file.read(size)
- else:
- ret = self.archive.file.read(self.size - self.current)
- self.current += len(ret)
- return ret
-
- def seek(self, offset, whence=0):
- """Move to new position within an entry.
-
- Keyword arguments:
- offset -- Byte count
- whence -- Describes how offset is used.
- 0: From beginning of file
- 1: Forwards from current position
- 2: Backwards from current position
- Other values are ignored.
- """
- if self.closed:
- raise ValueError("Seek operation on closed file.")
-
- if whence == os.SEEK_SET:
- self.current = offset
- elif whence == os.SEEK_REL:
- self.current += offset
- elif whence == os.SEEK_END:
- self.current -= offset
-
- self.current = min(max(0, self.current), self.size)
-
- def tell(self):
- """Get current position within an entry"""
- if self.closed:
- raise ValueError("Tell operation on closed file.")
- return self.current
-
- def __repr__(self):
- return "<CpioEntry %s 0x%s>" % (self.name, self.checksum,)
-
- @property
- def checksum(self):
- return int(self.hdr[102:110], 16)
-
- @property
- def devmajor(self):
- return int(self.hdr[62:70], 16)
-
- @property
- def devminor(self):
- return int(self.hdr[70:78], 16)
-
- @property
- def gid(self):
- return int(self.hdr[30:38], 16)
-
- @property
- def inode(self):
- return int(self.hdr[6:14], 16)
-
- @property
- def mode(self):
- return int(self.hdr[14:22], 16)
-
- @property
- def mtime(self):
- return int(self.hdr[46:54], 16)
-
- @property
- def name(self):
- end = 110 + self.namesize - 1
- return self.hdr[110:end]
-
- @property
- def namesize(self):
- return int(self.hdr[94:102], 16)
-
- @property
- def nlinks(self):
- return int(self.hdr[38:46], 16)
-
- @property
- def rdevmajor(self):
- return int(self.hdr[78:86], 16)
-
- @property
- def rdevminor(self):
- return int(self.hdr[86:94], 16)
-
- @property
- def size(self):
- return int(self.hdr[54:62], 16)
-
- @property
- def uid(self):
- return int(self.hdr[22:30], 16)
-
-
-class CpioArchive(object):
- _entries = []
- file = None
-
- def __init__(self, filename):
-
- self.filename = filename
- self.file = open(self.filename, "r")
- self.__readfile()
-
- self.closed = False
-
- def close(self):
- if self.closed:
- return
- self.closed = True
-
- self.file.close()
-
- def __readfile(self):
- if not self.file:
- raise CpioError("File was not yet opened.")
-
- self._entries = []
- sposition = self.file.tell()
- hdr = self.file.read(110)
- while hdr:
- namelen = int(hdr[94:102], 16) # Length of the name
- hdr += self.file.read(namelen)
- ce = CpioEntry(hdr, self, sposition)
- if ce.name == "TRAILER!!!":
- return
- self._entries.append(ce)
-
- self.file.seek((4 - (self.file.tell()-sposition) % 4) % 4, os.SEEK_CUR)
- self.file.seek(ce.size, os.SEEK_CUR)
- self.file.seek((4 - (self.file.tell()-sposition) % 4) % 4, os.SEEK_CUR)
-
- sposition = self.file.tell()
- hdr = self.file.read(110)
- else:
- raise CpioError("Premature end of headers.")
-
- @property
- def entries(self):
- return sorted(self._entries)
-
- @property
- def size(self):
- return os.path.getsize(self.filename)
-
- def ls(self):
- for x in self.entries:
- print x.name
-
- def ll(self):
- for x in self.entries:
- print "%s %s %s %s %9d %s %s" % \
- (fmode(x.mode),
- x.nlinks,
- pwd.getpwuid(x.uid)[0],
- grp.getgrgid(x.gid)[0],
- x.size,
- time.strftime("%Y-%m-%d %H:%M", time.localtime(x.mtime)),
- x.name,)
-
- def get(self, item):
- for x in self.entries:
- if x.name == item:
- return x
- raise KeyError("No such file or directory.")
-
- def __getitem__(self, item):
- x = self.get(item)
- x.seek(0)
- return x.read()
+++ /dev/null
-#!/usr/bin/python
-
-import curses
-import logging
-import logging.handlers
-import sys
-import time
-
-# fix for python 2.4 logging module bug:
-logging.raiseExceptions = 0
-
-from constants import *
-
-class Logging(object):
- def __init__(self):
- self.log = logging.getLogger()
- self.log.setLevel(logging.NOTSET)
-
- # Initialize console
- ch = self.console
- ch.setLevel(logging.INFO)
-
- if sys.stderr.isatty():
- curses.setupterm()
- ch.setFormatter(_ColorLogFormatter())
- else:
- ch.setFormatter(logging.Formatter("[%(levelname)s] %(message)s"))
- self.log.addHandler(ch)
-
- # Initialize log file
- if not os.path.isdir(LOGDIR):
- os.makedirs(LOGDIR)
- fh = logging.handlers.RotatingFileHandler(config["log_file"],
- maxBytes=10*1024**2, backupCount=6)
- fh.setFormatter(logging.Formatter("[%(levelname)s] %(message)s"))
- fh.setLevel(logging.DEBUG)
- self.log.addHandler(fh)
-
- def debug(self, val):
- if val:
- self.console.setLevel(logging.DEBUG)
- self.log.debug("Enabled debug logging mode")
- else:
- self.log.debug("Disabled debug logging mode")
- self.console.setLevel(logging.INFO)
-
- @property
- def console(self):
- console = self.log.handlers[0]
-
- assert isinstance(console, logging.StreamHandler)
-
- return console
-
-
-# defaults to module verbose log
-# does a late binding on log. Forwards all attributes to logger.
-# works around problem where reconfiguring the logging module means loggers
-# configured before reconfig dont output.
-class getLog(object):
- def __init__(self, name=None, prefix="", *args, **kargs):
- if name is None:
- frame = sys._getframe(1)
- name = frame.f_globals["__name__"]
-
- self.name = prefix + name
-
- def __getattr__(self, name):
- logger = logging.getLogger(self.name)
- return getattr(logger, name)
-
-
-# Borrowed from tornado
-class _ColorLogFormatter(logging.Formatter):
- def __init__(self, *args, **kwargs):
- logging.Formatter.__init__(self, *args, **kwargs)
- fg_color = curses.tigetstr("setaf") or curses.tigetstr("setf") or ""
- self._colors = {
- logging.DEBUG: curses.tparm(fg_color, 4), # Blue
- logging.INFO: curses.tparm(fg_color, 2), # Green
- logging.WARNING: curses.tparm(fg_color, 3), # Yellow
- logging.ERROR: curses.tparm(fg_color, 1), # Red
- }
- self._normal = curses.tigetstr("sgr0")
-
- def format(self, record):
- try:
- record.message = record.getMessage()
- except Exception, e:
- record.message = "Bad message (%r): %r" % (e, record.__dict__)
- record.asctime = time.strftime(
- "%H:%M:%S", self.converter(record.created))
- prefix = " %(levelname)-7s" % record.__dict__
- color = self._colors.get(record.levelno, self._normal)
- formatted = color + prefix + self._normal + " " + record.message
- if record.exc_info:
- if not record.exc_text:
- record.exc_text = self.formatException(record.exc_info)
- if record.exc_text:
- formatted = formatted.rstrip() + "\n" + record.exc_text
- return formatted.replace("\n", "\n ")
+++ /dev/null
-#!/usr/bin/python
-
-import email.mime.multipart
-import email.mime.text
-import logging
-import os
-import smtplib
-
-from constants import *
-
-def report_error(package):
- # Do not send a report if no recipient is configured
- if not config["error_report_recipient"]:
- return
-
- try:
- connection = smtplib.SMTP(config["smtp_server"])
- #connection.set_debuglevel(1)
-
- if config["smtp_user"] and config["smtp_password"]:
- connection.login(config["smtp_user"], config["smtp_password"])
-
- except smtplib.SMTPConnectError, e:
- logging.error("Could not establish a connection to the smtp server: %s" % e)
- return
- except smtplib.SMTPAuthenticationError, e:
- logging.error("Could not successfully login to the smtp server: %s" % e)
- return
-
- msg = email.mime.multipart.MIMEMultipart()
- msg["From"] = config["error_report_sender"]
- msg["To"] = config["error_report_recipient"]
- msg["Subject"] = config["error_report_subject"] % package.all
- msg.preamble = 'You will not see this in a MIME-aware mail reader.\n'
-
- body = """\
-The package %(name)s had a difficulty to build itself.
-This email will give you a short report about the error.
-
-Package information:
- Name : %(name)s - %(summary)s
- Version : %(version)s
- Release : %(release)s
-
- This package in maintained by %(maintainer)s.
-
-
-A detailed logfile is attached.
-
-Sincerely,
- Naoki
- """ % {
- "name" : package.name,
- "summary" : package.summary,
- "version" : package.version,
- "release" : package.release,
- "maintainer" : package.maintainer,
- }
-
- msg.attach(email.mime.text.MIMEText(body))
-
- # Read log and append it to mail
- loglines = []
- if os.path.exists(package.logfile):
- f = open(package.logfile)
- line = f.readline()
- while line:
- line = line.rstrip()
- if line.endswith(LOG_MARKER):
- # Reset log
- loglines = []
-
- loglines.append(line)
- line = f.readline()
-
- f.close()
-
- if not loglines:
- loglines = ["Logfile wasn't found."]
-
- log = email.mime.text.MIMEText("\n".join(loglines), _subtype="plain")
- log.add_header('Content-Disposition', 'attachment',
- filename="%s.log" % package.id)
- msg.attach(log)
-
- try:
- connection.sendmail(config["error_report_sender"],
- config["error_report_recipient"], msg.as_string())
- except Exception, e:
- logging.error("Could not send error report: %s: %s" % (e.__class__.__name__, e))
- return
-
- connection.quit()
+++ /dev/null
-#!/usr/bin/python
-
-import logging
-import os
-import re
-
-import urlgrabber
-import urlgrabber.progress
-
-import architectures
-import dependencies
-import environ
-import io
-import util
-
-from constants import *
-from exception import *
-
-def version_compare_epoch(e1, e2):
- return cmp(e1, e2)
-
-def version_compare_version(v1, v2):
- return cmp(v1, v2)
-
-def version_compare_release(r1, r2):
- return cmp(r1, r2)
-
-def version_compare((e1, v1, r1), (e2, v2, r2)):
-
- ret = version_compare_epoch(e1, e2)
- if not ret == 0:
- return ret
-
- ret = version_compare_version(v1, v2)
- if not ret == 0:
- return ret
-
- return version_compare_release(r1, r2)
-
-
-class Package(object):
- def __repr__(self):
- return "<%s %s:%s>" % \
- (self.__class__.__name__, self.name, self.arch.name)
-
- #@property
- #def arch(self):
- # raise NotImplementedError
-
- @property
- def epoch(self):
- epoch = self._info.get("PKG_EPOCH", None)
- if not epoch:
- epoch = 0
- return int(epoch)
-
- @property
- def name(self):
- return self._info["PKG_NAME"]
-
- @property
- def group(self):
- return self._info["PKG_GROUP"]
-
- @property
- def id(self):
- return "%s-%s-%s.%s" % \
- (self.name, self.version, self.release, self.arch.name)
-
- @property
- def release(self):
- return int(self._info["PKG_REL"])
-
- @property
- def version(self):
- return self._info["PKG_VER"]
-
- def print_raw_info(self):
- for k in sorted(self._info.keys()):
- print "%s=\"%s\"" % (k, self._info.get(k))
-
- def fmtstr(self, s):
- return s % self._info
-
-
-class SourcePackage(Package):
- def __init__(self, filename, repo, arch):
- self.arch = arch
- self.filename = filename
- self.repository = repo
-
- self.init()
-
- logging.debug("Successfully initialized %s" % self)
-
- def init(self):
- self._info = {}
-
- env = environ.set({
- "PKG_ARCH" : self.arch.name,
- "PKGROOT" : PKGSDIR,
- })
-
- output = util.do("make -f %s" % os.path.basename(self.filename),
- shell=True,
- cwd=os.path.dirname(self.filename),
- returnOutput=1,
- env=env)
-
- for line in output.splitlines():
- if not line:
- continue
-
- m = re.match(r"^(\w+)=(.*)$", line)
- if m is None:
- continue
-
- key, val = m.groups()
- self._info[key] = val.strip("\"")
-
- # XXX should return a dependencyset
- def get_dependencies(self, type=""):
- type2var = {
- "" : "PKG_DEPENDENCIES",
- "build" : "PKG_BUILD_DEPENDENCIES",
- }
-
- type = type2var[type]
-
- return [dependencies.Dependency(d, origin=self) for d in self._info[type].split()]
-
- @property
- def source_dir(self):
- return self._info.get("SOURCE_DIR", "")
-
- @property
- def summary(self):
- return self._info["PKG_SUMMARY"]
-
- @property
- def binary_files(self):
- return self._info.get("PKG_PACKAGES_FILES").split()
-
- @property
- def is_built(self):
- for file in self.binary_files:
- file = os.path.join(PACKAGESDIR, self.arch.name, file)
- if not os.path.exists(file):
- return False
-
- return True
-
- @property
- def source_files(self):
- return self._info.get("PKG_OBJECTS").split()
-
- def source_download(self):
- g = urlgrabber.grabber.URLGrabber(
- prefix = config["sources_download_url"],
- progress_obj = urlgrabber.progress.TextMeter(),
- quote=0,
- )
-
- for file in self.source_files:
- file = os.path.join(CACHEDIR, file)
-
- if os.path.exists(file):
- continue
-
- util.mkdir(CACHEDIR)
-
- try:
- g.urlgrab(os.path.basename(file), filename=file)
- except urlgrabber.grabber.URLGrabError,e :
- raise DownloadError, "%s %s" % (os.path.basename(file), e)
-
-
-class BinaryPackage(Package):
- def __init__(self, filename):
- self.filename = filename
-
- self.init()
- logging.debug("Successfully initialized %s" % self)
-
- def __repr__(self):
- return "<%s %s:%s-%s-%s>" % \
- (self.__class__.__name__, self.epoch, self.name, self.version,
- self.release)
-
- def __cmp__(self, other):
- # Packages are equal
- if self.id == other.id:
- return 0
-
- # if packages differ names return in alphabetical order
- if not self.name == other.name:
- return cmp(self.name, other.name)
-
- ret = version_compare((self.epoch, self.version, self.release),
- (other.epoch, other.version, other.release))
-
- if ret == 0:
- logging.debug("%s is equal to %s" % (self, other))
- elif ret < 0:
- logging.debug("%s is more recent than %s" % (other, self))
- elif ret > 0:
- logging.debug("%s is more recent than %s" % (self, other))
-
- return ret
-
- def _readfile(self, name):
- try:
- f = io.CpioArchive(self.filename)
- except ValueError, e:
- raise Exception, "Could not open package '%s' correctly: %s" \
- % (self.filename, e)
-
- # If file is not available, return None
- ret = None
- if name in [e.name for e in f.entries]:
- ret = f.get(name).read()
-
- f.close()
- return ret
-
- def init(self):
- self._info = {}
- self._filelist = []
-
- info = self._readfile("info")
-
- for line in info.splitlines():
- if not line:
- continue
-
- m = re.match(r"^(\w+)=(.*)$", line)
- if m is None:
- continue
-
- key, val = m.groups()
- self._info[key] = val.strip("\"")
-
- self.__dependencies = self.get_dependencies()
- self.__provides = self.get_provides()
-
- def extract(self, path):
- logging.debug("Extracting %s to %s" % (self, path))
-
- cmd = os.path.join(TOOLSDIR, "decompressor")
- cmd += " --root=%s %s" % (path, self.filename)
-
- util.do(cmd, shell=True, logger=logging.getLogger())
-
- @property
- def arch(self):
- arches = architectures.Architectures()
-
- arch = self._info.get("PKG_ARCH", None)
- if arch:
- arch = arches.get(arch)
- else:
- arch = arches.get_default()
-
- return arch
-
- @property
- def name(self):
- return self._info["PKG_NAME"]
-
- def get_dependencies(self):
- if hasattr(self, "__dependencies"):
- return self.__dependencies
-
- objects = self._info.get("PKG_DEPS", "").split()
-
- # Compatibility to older package format
- objects += self._info.get("PKG_REQUIRES", "").split()
-
- return [dependencies.Dependency(o, origin=self) for o in objects]
-
- def get_provides(self):
- if hasattr(self, "__provides"):
- return self.__provides
-
- return [dependencies.Provides(p, origin=self) \
- for p in self._info.get("PKG_PROVIDES", "").split()]
-
- @property
- def filelist(self):
- if not self._filelist:
-
- filelist = self._readfile("filelist")
- if not filelist:
- return []
-
- for file in filelist.splitlines():
- if not file or not file.startswith("/"):
- continue
-
- self._filelist.append(file)
-
- return self._filelist
-
- def does_provide(self, what):
- if what.type == DEP_PACKAGE:
- return what.identifier == self.name
-
- elif what.type == DEP_FILE:
- return what.identifier in self.filelist
-
- elif what.type in (DEP_LIBRARY,):
- for provide in self.get_provides():
- if provide.match(what):
- return True
-
- return False
-
- # For invalid dependencies alsways return false
- return False
+++ /dev/null
-#!/usr/bin/python
-
-import logging
-import operator
-import os
-
-import architectures
-import packages
-
-from constants import *
-from decorators import *
-
-class Repository(object):
- def __init__(self, arch):
- assert isinstance(arch, architectures.Architecture)
- self.arch = arch
-
- logging.debug("Successfully initialized %s" % self)
-
- def __repr__(self):
- return "<%s %s>" % (self.__class__.__name__, self.path)
-
- @property
- def path(self):
- raise NotImplementedError
-
-
-class BinaryRepository(Repository):
- _cache = {}
-
- @property
- def path(self):
- return os.path.join(PACKAGESDIR, self.arch.name)
-
- def find_package_by_name(self, name):
- pkgs = self.find_packages_by_name(name)
-
- if pkgs:
- return pkgs[0]
-
- def find_packages_by_name(self, name):
- pkgs = []
-
- for package in self.all:
- if package.name == name:
- pkgs.append(package)
-
- return sorted(pkgs, reverse=True)
-
- @property
- def all(self):
- if not self._cache.has_key(self.arch.name):
- l = []
-
- for package in os.listdir(self.path):
- # Construct filename
- package = os.path.join(self.path, package)
-
- # Create package instance
- package = packages.BinaryPackage(package)
-
- # Append package to list
- l.append(package)
-
- self._cache[self.arch.name] = l
-
- return self._cache[self.arch.name]
-
- def find_package_by_file(self, filename):
- pkgs = self.find_packages_by_file(filename)
-
- if pkgs:
- return pkgs[0]
-
- def find_packages_by_file(self, filename):
- pkgs = []
-
- for package in self.all:
- if filename in package.filelist:
- pkgs.append(package)
-
- return sorted(pkgs, reverse=True)
-
- def find_package_by_provides(self, provides):
- pkgs = self.find_packages_by_provides(provides)
-
- if pkgs:
- return pkgs[0]
-
- def find_packages_by_provides(self, provides):
- pkgs = []
-
- for package in self.all:
- if package.does_provide(provides):
- pkgs.append(package)
-
- return sorted(pkgs, reverse=True)
-
-
-class SourceRepository(Repository):
- def __init__(self, name, arch):
- self.name = name
-
- Repository.__init__(self, arch)
-
- def __iter__(self):
- pkgs = [os.path.join(self.path, p, p + ".nm") for p in self.packages]
-
- args = {
- "arch" : self.arch,
- "repo" : self,
- }
-
- return PackageIterator(pkgs, packages.SourcePackage, args)
-
- def find_package_by_name(self, name, fast=False):
- if fast:
- filename = os.path.join(self.path, name, name + ".nm")
- if os.path.exists(filename):
- return packages.SourcePackage(filename, repo=self, arch=self.arch)
-
- else:
- for package in self:
- if package.name == name:
- return package
-
- @property
- def path(self):
- return os.path.join(PKGSDIR, self.name)
-
- @property
- def packages(self):
- return sorted(os.listdir(self.path))
-
- @property
- def completely_built(self):
- for p in self:
- if not p.is_built:
- return False
-
- return True
-
-
-class PackageIterator(object):
- def __init__(self, paks, package_cls=None, cls_args={}):
- self.packages = paks
- self.package_cls = package_cls
- self.package_cls_args = cls_args
-
- self.__i = 0
-
- def __iter__(self):
- return self
-
- def next(self):
- if self.__i >= len(self.packages):
- raise StopIteration
-
- pkgs = self.packages[self.__i]
- self.__i += 1
-
- if self.package_cls:
- package = self.package_cls(pkgs, **self.package_cls_args)
-
- return package
-
-
-class SourceRepositories(object):
- def __init__(self, arch):
- assert isinstance(arch, architectures.Architecture)
- self.arch = arch
-
- self._repositories = []
-
- self.find_all()
-
- def find_all(self):
- for repo in sorted(os.listdir(PKGSDIR)):
- # Skip all non-directories
- if not os.path.isdir(os.path.join(PKGSDIR, repo)):
- continue
-
- logging.debug("Found source repository: %s" % repo)
-
- repo = SourceRepository(repo, self.arch)
- self._repositories.append(repo)
-
- @property
- def all(self):
- return self._repositories[:]
-
- def find_package_by_name(self, name):
- for fast in (True, False):
- for repo in self._repositories:
- package = repo.find_package_by_name(name, fast=fast)
- if package:
- return package
-
- @property
- def packages(self):
- #pkgs = []
- #for repository in self._repositories:
- # pkgs += [p for p in repository]
- #
- #return pkgs
-
- for repository in self._repositories:
- for p in repository:
- yield p
-
-
-if __name__ == "__main__":
- arches = architectures.Architectures()
-
- r = SourceRepositories(arches.get("i686"))
-
- for repo in r.all:
- print "%s" % repo.name
-
- #for pack in repo:
- # print " %s" % pack.name
-
- #for package in ("gcc", "screen", "iptables", "system-release", "glibc", "nonexistant"):
- # print "Searching for %s" % package,
- # print r.find_package_by_name(package)
-
- b = BinaryRepository(arches.get("i686"))
-
- print b.find_packages_by_name("aiccu")
- print b.find_package_by_name("aiccu")
+++ /dev/null
-#!/usr/bin/python
-
-import fcntl
-import os
-import struct
-import sys
-import termios
-
-import architectures
-from constants import *
-
-arches = architectures.Architectures()
-
-class ParsingError(Exception):
- pass
-
-
-class NameSpace(dict):
- def __getattr__(self, attr):
- try:
- return self.__getitem__(attr)
- except KeyError:
- raise AttributeError
-
-
-class Parser(object):
- def __init__(self, name, arguments=[], parsers=[], **kwargs):
- self.name = name
- self.arguments = arguments
- self.parsers = parsers
-
- self._help = kwargs.get("help", "No help available")
-
- self.subparser = None
-
- def __repr__(self):
- return "<Parser %s>" % self.name
-
- def check(self, args):
- if not args:
- return False
-
- return self.name == args[0]
-
- def parse(self, args):
- if args and self.name == args[0]:
- args = args[1:]
-
- for argument in self.arguments:
- args = argument.parse(args)
-
- if args and self.parsers:
- for parser in self.parsers:
- if not parser.check(args):
- continue
-
- self.subparser = parser
- break
-
- if self.subparser:
- args = self.subparser.parse(args)
-
- return args
-
- @property
- def values(self):
- ret = NameSpace(
- name=self.name,
- )
- if self.subparser:
- ret["action"] = self.subparser.values
-
- for argument in self.arguments:
- ret[argument.name] = argument.value()
-
- return ret
-
- @property
- def help_line(self):
- ret = ""
- for argument in self.arguments:
- ret += " %s" % argument.help_line
-
- if self.parsers:
- ret += " <command ...>"
-
- return ret
-
- def help(self, indent=0):
- ret = self.name
-
- help_line = self.help_line
- if help_line:
- ret += self.help_line + "\n"
- if self._help:
- ret += "\n " + self._help + "\n"
- else:
- ret += " - " + self._help + "\n"
-
- if self.arguments:
- ret += "\n"
- for argument in sorted(self.arguments):
- ret += " %15s | %s\n" % (", ".join(argument.args), argument.help)
- ret += "\n"
-
- if self.parsers:
- ret += "\n"
- for parser in self.parsers:
- ret += parser.help(indent=indent + 4)
-
- indent_string = " " * 4
- return indent_string.join(["%s\n" % line for line in ret.split("\n")])
-
-
-class _Argument(object):
- DEFAULT_HELP = "No help available"
-
- def __init__(self, name, args, **kwargs):
- self.name = name
- self.args = sorted(args, reverse=True, key=str.__len__)
- self.help = kwargs.get("help", self.DEFAULT_HELP)
-
- self._parsed = False
- self._parsed_args = []
-
- def __cmp__(self, other):
- return cmp(self.name, other.name)
-
- def parse(self, args):
- raise NotImplementedError
-
- def value(self):
- raise NotImplementedError
-
- @property
- def help_line(self):
- raise NotImplementedError
-
-
-class Argument(_Argument):
- def __init__(self, name, **kwargs):
- _Argument.__init__(self, name, [], **kwargs)
-
- def parse(self, args):
- self._parsed = True
-
- if len(args) >= 1:
- self._parsed_args = args[:1]
-
- return args[1:]
-
- def value(self):
- if self._parsed_args:
- return self._parsed_args[0]
-
- return []
-
- @property
- def help_line(self):
- return self.name
-
-
-class Option(_Argument):
- def parse(self, args):
- self._parsed = True
-
- new_args = []
- for arg in args:
- if arg in self.args:
- self._parsed_args.append(arg)
- else:
- new_args.append(arg)
-
- return new_args
-
- def value(self):
- for arg in self._parsed_args:
- if arg in self.args:
- return True
-
- return False
-
- @property
- def help_line(self):
- return "[%s]" % ", ".join(self.args)
-
-
-class Choice(_Argument):
- def __init__(self, *args, **kwargs):
- _Argument.__init__(self, *args, **kwargs)
-
- self.choices = kwargs.get("choices", [])
- self.choices.sort()
-
- self.help += " [%s]" % ", ".join(self.choices)
-
- def parse(self, args):
- self._parsed = True
-
- new_args = []
- next_arg = False
- for arg in args:
- if next_arg:
- if self.choices and not arg in self.choices:
- raise ParsingError, "'%s' is not an allowed choice" % arg
-
- self._parsed_args.append(arg)
- next_arg = False
- continue
-
- if arg in self.args:
- self._parsed_args.append(arg)
- next_arg = True
- else:
- new_args.append(arg)
-
- return new_args
-
- def value(self):
- if self._parsed_args:
- return self._parsed_args[-1]
-
- return None
-
- @property
- def help_line(self):
- return "[%s %s]" % (", ".join(self.args), self.name.upper())
-
-
-class List(_Argument):
- def __init__(self, name, **kwargs):
- _Argument.__init__(self, name, [], **kwargs)
-
- def parse(self, args):
- self._parsed = True
- self._parsed_args = args
-
- return []
-
- def value(self):
- return self._parsed_args
-
- @property
- def help_line(self):
- name = self.name[:-1] + " "
- return "[" + name * 2 + "...]"
-
-
-class Commandline(object):
- def __init__(self, naoki):
- self.naoki = naoki
-
- # Parse the stuff
- self.args = self.__parse()
-
- # ... afterwards, process global directives
- self.__process_global(self.args)
-
- def __process_global(self, args):
- # Set debugging mode
- self.naoki.debug(args.debug)
-
- # Set architecture
- arches.set_default(args.arch)
-
- def __parse(self):
- parser = Parser(sys.argv[0],
- help="Global help",
- arguments=[
- Option("help", ["-h", "--help"], help="Show help text"),
- Option("quiet", ["-q", "--quiet"], help="Set quiet mode"),
- Option("debug", ["-d", "--debug"], help="Set debugging mode"),
- Choice("arch", ["-a", "--arch"], help="Set architecture",
- choices=[arch.name for arch in arches.all]),
- ],
- parsers=[
- # Build
- Parser("build",
- help="Primary build command",
- arguments=[
- Option("all", ["--all"], help="Build all packages"),
- Option("ignore_dependency_errors", ["-i", "--ignore-dependency-errors"],
- help="Ignore dependency errors."),
- Option("shell", ["-s", "--shell"], help="Enter a shell after unsuccessfull build."),
- Argument("package", help="Name of a package to build."),
- ]),
-
- # Package
- Parser("package",
- parsers=[
- Parser("dependencies",
- help="Show package dependencies.",
- arguments=[
- Argument("package", help="Name of the package."),
- ]),
- Parser("info",
- help="Show detailed information about given packages",
- arguments=[
- Option("long", ["-l", "--long"], help="Show long list of information"),
- Option("machine", ["--machine"], help="Output in machine parseable format"),
- List("packages"),
- ]),
- Parser("list",
- help="Show package list",
- arguments=[
- Option("long", ["-l", "--long"], help="Show list with lots of information"),
- Option("unbuilt", ["-u", "--unbuilt"], help="Do only show unbuilt packages"),
- Option("built", ["-b", "--built"], help="Do only show already built packages"),
- ]),
- Parser("groups",
- help="Show package groups",
- ),
- Parser("raw",
- help="Show package information in a machine parseable format.",
- arguments=[
- Argument("package", help="Name of the package."),
- ]),
- Parser("provides",
- help="Find packages which provide a specific thing.",
- arguments=[
- Argument("provides", help="Pattern to search for."),
- ]),
- ]),
-
- # Source
- Parser("source",
- help="Handle source tarballs",
- parsers=[
- Parser("download",
- help="Download source tarballs",
- arguments=[
- List("packages"),
- ]),
- Parser("upload",
- help="Upload source tarballs",
- arguments=[
- List("packages"),
- ]),
- ]),
-
- # Check
- Parser("check",
- help="Check commands",
- parsers=[
- Parser("host", help="Check if host fullfills prerequisites"),
- ]),
-
- # Batch
- Parser("batch",
- help="Batch command - use with caution",
- parsers=[
- Parser("cron", help="Command that gets called by cron"),
- ]),
-
- # Shell
- Parser("shell",
- help="Shell environment",
- arguments=[
- Argument("package", help="Package to process."),
- ]),
-
- # Generator
- Parser("generate",
- help="Generator command",
- arguments=[
- Argument("type", help="Type of image to generate"),
- ]),
- ])
-
- self.parser = parser
-
- args = parser.parse(sys.argv[1:])
-
- if args:
- raise ParsingError, "Unknown argument(s) passed: %s" % args
-
- return parser.values
-
- def help(self):
- print >>sys.stderr, self.parser.help(),
-
-
-DEFAULT_COLUMNS = 80
-DEFAULT_LINES = 25
-
-def get_size():
- """
- returns the size of the terminal
-
- tupel: lines, columns
- """
- def ioctl_GWINSZ(fd):
- try:
- cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))
- except:
- return None
-
- return cr
-
- cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
- if not cr:
- try:
- fd = os.open(os.ctermid(), os.O_RDONLY)
- cr = ioctl_GWINSZ(fd)
- os.close(fd)
- except:
- pass
-
- if not cr:
- try:
- cr = (os.environ['LINES'], os.environ['COLUMNS'])
- except:
- cr = (DEFAULT_LINES, DEFAULT_COLUMNS)
-
- return int(cr[1]), int(cr[0])
-
-def get_lines():
- return get_size()[0]
-
-def get_columns():
- return get_size()[1]
-
-
-if __name__ == "__main__":
- cl = Commandline(naoki=None)
+++ /dev/null
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Library General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-# Copyright 2002-2006 Michael D. Stenner, Ryan Tomayko
-
-# $Id: __init__.py,v 1.20 2006/09/22 00:58:55 mstenner Exp $
-
-"""A high-level cross-protocol url-grabber.
-
-Using urlgrabber, data can be fetched in three basic ways:
-
- urlgrab(url) copy the file to the local filesystem
- urlopen(url) open the remote file and return a file object
- (like urllib2.urlopen)
- urlread(url) return the contents of the file as a string
-
-When using these functions (or methods), urlgrabber supports the
-following features:
-
- * identical behavior for http://, ftp://, and file:// urls
- * http keepalive - faster downloads of many files by using
- only a single connection
- * byte ranges - fetch only a portion of the file
- * reget - for a urlgrab, resume a partial download
- * progress meters - the ability to report download progress
- automatically, even when using urlopen!
- * throttling - restrict bandwidth usage
- * retries - automatically retry a download if it fails. The
- number of retries and failure types are configurable.
- * authenticated server access for http and ftp
- * proxy support - support for authenticated http and ftp proxies
- * mirror groups - treat a list of mirrors as a single source,
- automatically switching mirrors if there is a failure.
-"""
-
-__version__ = '3.1.0'
-__date__ = '2006/09/21'
-__author__ = 'Michael D. Stenner <mstenner@linux.duke.edu>, ' \
- 'Ryan Tomayko <rtomayko@naeblis.cx>'
-__url__ = 'http://linux.duke.edu/projects/urlgrabber/'
-
-from grabber import urlgrab, urlopen, urlread
+++ /dev/null
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330,
-# Boston, MA 02111-1307 USA
-
-# This file is part of urlgrabber, a high-level cross-protocol url-grabber
-# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
-
-# $Id: byterange.py,v 1.12 2006/07/20 20:15:58 mstenner Exp $
-
-import os
-import stat
-import urllib
-import urllib2
-import rfc822
-
-DEBUG = None
-
-try:
- from cStringIO import StringIO
-except ImportError, msg:
- from StringIO import StringIO
-
-class RangeError(IOError):
- """Error raised when an unsatisfiable range is requested."""
- pass
-
-class HTTPRangeHandler(urllib2.BaseHandler):
- """Handler that enables HTTP Range headers.
-
- This was extremely simple. The Range header is a HTTP feature to
- begin with so all this class does is tell urllib2 that the
- "206 Partial Content" reponse from the HTTP server is what we
- expected.
-
- Example:
- import urllib2
- import byterange
-
- range_handler = range.HTTPRangeHandler()
- opener = urllib2.build_opener(range_handler)
-
- # install it
- urllib2.install_opener(opener)
-
- # create Request and set Range header
- req = urllib2.Request('http://www.python.org/')
- req.header['Range'] = 'bytes=30-50'
- f = urllib2.urlopen(req)
- """
-
- def http_error_206(self, req, fp, code, msg, hdrs):
- # 206 Partial Content Response
- r = urllib.addinfourl(fp, hdrs, req.get_full_url())
- r.code = code
- r.msg = msg
- return r
-
- def http_error_416(self, req, fp, code, msg, hdrs):
- # HTTP's Range Not Satisfiable error
- raise RangeError('Requested Range Not Satisfiable')
-
-class HTTPSRangeHandler(HTTPRangeHandler):
- """ Range Header support for HTTPS. """
-
- def https_error_206(self, req, fp, code, msg, hdrs):
- return self.http_error_206(req, fp, code, msg, hdrs)
-
- def https_error_416(self, req, fp, code, msg, hdrs):
- self.https_error_416(req, fp, code, msg, hdrs)
-
-class RangeableFileObject:
- """File object wrapper to enable raw range handling.
- This was implemented primarilary for handling range
- specifications for file:// urls. This object effectively makes
- a file object look like it consists only of a range of bytes in
- the stream.
-
- Examples:
- # expose 10 bytes, starting at byte position 20, from
- # /etc/aliases.
- >>> fo = RangeableFileObject(file('/etc/passwd', 'r'), (20,30))
- # seek seeks within the range (to position 23 in this case)
- >>> fo.seek(3)
- # tell tells where your at _within the range_ (position 3 in
- # this case)
- >>> fo.tell()
- # read EOFs if an attempt is made to read past the last
- # byte in the range. the following will return only 7 bytes.
- >>> fo.read(30)
- """
-
- def __init__(self, fo, rangetup):
- """Create a RangeableFileObject.
- fo -- a file like object. only the read() method need be
- supported but supporting an optimized seek() is
- preferable.
- rangetup -- a (firstbyte,lastbyte) tuple specifying the range
- to work over.
- The file object provided is assumed to be at byte offset 0.
- """
- self.fo = fo
- (self.firstbyte, self.lastbyte) = range_tuple_normalize(rangetup)
- self.realpos = 0
- self._do_seek(self.firstbyte)
-
- def __getattr__(self, name):
- """This effectively allows us to wrap at the instance level.
- Any attribute not found in _this_ object will be searched for
- in self.fo. This includes methods."""
- if hasattr(self.fo, name):
- return getattr(self.fo, name)
- raise AttributeError, name
-
- def tell(self):
- """Return the position within the range.
- This is different from fo.seek in that position 0 is the
- first byte position of the range tuple. For example, if
- this object was created with a range tuple of (500,899),
- tell() will return 0 when at byte position 500 of the file.
- """
- return (self.realpos - self.firstbyte)
-
- def seek(self,offset,whence=0):
- """Seek within the byte range.
- Positioning is identical to that described under tell().
- """
- assert whence in (0, 1, 2)
- if whence == 0: # absolute seek
- realoffset = self.firstbyte + offset
- elif whence == 1: # relative seek
- realoffset = self.realpos + offset
- elif whence == 2: # absolute from end of file
- # XXX: are we raising the right Error here?
- raise IOError('seek from end of file not supported.')
-
- # do not allow seek past lastbyte in range
- if self.lastbyte and (realoffset >= self.lastbyte):
- realoffset = self.lastbyte
-
- self._do_seek(realoffset - self.realpos)
-
- def read(self, size=-1):
- """Read within the range.
- This method will limit the size read based on the range.
- """
- size = self._calc_read_size(size)
- rslt = self.fo.read(size)
- self.realpos += len(rslt)
- return rslt
-
- def readline(self, size=-1):
- """Read lines within the range.
- This method will limit the size read based on the range.
- """
- size = self._calc_read_size(size)
- rslt = self.fo.readline(size)
- self.realpos += len(rslt)
- return rslt
-
- def _calc_read_size(self, size):
- """Handles calculating the amount of data to read based on
- the range.
- """
- if self.lastbyte:
- if size > -1:
- if ((self.realpos + size) >= self.lastbyte):
- size = (self.lastbyte - self.realpos)
- else:
- size = (self.lastbyte - self.realpos)
- return size
-
- def _do_seek(self,offset):
- """Seek based on whether wrapped object supports seek().
- offset is relative to the current position (self.realpos).
- """
- assert offset >= 0
- if not hasattr(self.fo, 'seek'):
- self._poor_mans_seek(offset)
- else:
- self.fo.seek(self.realpos + offset)
- self.realpos+= offset
-
- def _poor_mans_seek(self,offset):
- """Seek by calling the wrapped file objects read() method.
- This is used for file like objects that do not have native
- seek support. The wrapped objects read() method is called
- to manually seek to the desired position.
- offset -- read this number of bytes from the wrapped
- file object.
- raise RangeError if we encounter EOF before reaching the
- specified offset.
- """
- pos = 0
- bufsize = 1024
- while pos < offset:
- if (pos + bufsize) > offset:
- bufsize = offset - pos
- buf = self.fo.read(bufsize)
- if len(buf) != bufsize:
- raise RangeError('Requested Range Not Satisfiable')
- pos+= bufsize
-
-class FileRangeHandler(urllib2.FileHandler):
- """FileHandler subclass that adds Range support.
- This class handles Range headers exactly like an HTTP
- server would.
- """
- def open_local_file(self, req):
- import mimetypes
- import mimetools
- host = req.get_host()
- file = req.get_selector()
- localfile = urllib.url2pathname(file)
- stats = os.stat(localfile)
- size = stats[stat.ST_SIZE]
- modified = rfc822.formatdate(stats[stat.ST_MTIME])
- mtype = mimetypes.guess_type(file)[0]
- if host:
- host, port = urllib.splitport(host)
- if port or socket.gethostbyname(host) not in self.get_names():
- raise urllib2.URLError('file not on local host')
- fo = open(localfile,'rb')
- brange = req.headers.get('Range',None)
- brange = range_header_to_tuple(brange)
- assert brange != ()
- if brange:
- (fb,lb) = brange
- if lb == '': lb = size
- if fb < 0 or fb > size or lb > size:
- raise RangeError('Requested Range Not Satisfiable')
- size = (lb - fb)
- fo = RangeableFileObject(fo, (fb,lb))
- headers = mimetools.Message(StringIO(
- 'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' %
- (mtype or 'text/plain', size, modified)))
- return urllib.addinfourl(fo, headers, 'file:'+file)
-
-
-# FTP Range Support
-# Unfortunately, a large amount of base FTP code had to be copied
-# from urllib and urllib2 in order to insert the FTP REST command.
-# Code modifications for range support have been commented as
-# follows:
-# -- range support modifications start/end here
-
-from urllib import splitport, splituser, splitpasswd, splitattr, \
- unquote, addclosehook, addinfourl
-import ftplib
-import socket
-import sys
-import ftplib
-import mimetypes
-import mimetools
-
-class FTPRangeHandler(urllib2.FTPHandler):
- def ftp_open(self, req):
- host = req.get_host()
- if not host:
- raise IOError, ('ftp error', 'no host given')
- host, port = splitport(host)
- if port is None:
- port = ftplib.FTP_PORT
-
- # username/password handling
- user, host = splituser(host)
- if user:
- user, passwd = splitpasswd(user)
- else:
- passwd = None
- host = unquote(host)
- user = unquote(user or '')
- passwd = unquote(passwd or '')
-
- try:
- host = socket.gethostbyname(host)
- except socket.error, msg:
- raise urllib2.URLError(msg)
- path, attrs = splitattr(req.get_selector())
- dirs = path.split('/')
- dirs = map(unquote, dirs)
- dirs, file = dirs[:-1], dirs[-1]
- if dirs and not dirs[0]:
- dirs = dirs[1:]
- try:
- fw = self.connect_ftp(user, passwd, host, port, dirs)
- type = file and 'I' or 'D'
- for attr in attrs:
- attr, value = splitattr(attr)
- if attr.lower() == 'type' and \
- value in ('a', 'A', 'i', 'I', 'd', 'D'):
- type = value.upper()
-
- # -- range support modifications start here
- rest = None
- range_tup = range_header_to_tuple(req.headers.get('Range',None))
- assert range_tup != ()
- if range_tup:
- (fb,lb) = range_tup
- if fb > 0: rest = fb
- # -- range support modifications end here
-
- fp, retrlen = fw.retrfile(file, type, rest)
-
- # -- range support modifications start here
- if range_tup:
- (fb,lb) = range_tup
- if lb == '':
- if retrlen is None or retrlen == 0:
- raise RangeError('Requested Range Not Satisfiable due to unobtainable file length.')
- lb = retrlen
- retrlen = lb - fb
- if retrlen < 0:
- # beginning of range is larger than file
- raise RangeError('Requested Range Not Satisfiable')
- else:
- retrlen = lb - fb
- fp = RangeableFileObject(fp, (0,retrlen))
- # -- range support modifications end here
-
- headers = ""
- mtype = mimetypes.guess_type(req.get_full_url())[0]
- if mtype:
- headers += "Content-Type: %s\n" % mtype
- if retrlen is not None and retrlen >= 0:
- headers += "Content-Length: %d\n" % retrlen
- sf = StringIO(headers)
- headers = mimetools.Message(sf)
- return addinfourl(fp, headers, req.get_full_url())
- except ftplib.all_errors, msg:
- raise IOError, ('ftp error', msg), sys.exc_info()[2]
-
- def connect_ftp(self, user, passwd, host, port, dirs):
- fw = ftpwrapper(user, passwd, host, port, dirs)
- return fw
-
-class ftpwrapper(urllib.ftpwrapper):
- # range support note:
- # this ftpwrapper code is copied directly from
- # urllib. The only enhancement is to add the rest
- # argument and pass it on to ftp.ntransfercmd
- def retrfile(self, file, type, rest=None):
- self.endtransfer()
- if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1
- else: cmd = 'TYPE ' + type; isdir = 0
- try:
- self.ftp.voidcmd(cmd)
- except ftplib.all_errors:
- self.init()
- self.ftp.voidcmd(cmd)
- conn = None
- if file and not isdir:
- # Use nlst to see if the file exists at all
- try:
- self.ftp.nlst(file)
- except ftplib.error_perm, reason:
- raise IOError, ('ftp error', reason), sys.exc_info()[2]
- # Restore the transfer mode!
- self.ftp.voidcmd(cmd)
- # Try to retrieve as a file
- try:
- cmd = 'RETR ' + file
- conn = self.ftp.ntransfercmd(cmd, rest)
- except ftplib.error_perm, reason:
- if str(reason)[:3] == '501':
- # workaround for REST not supported error
- fp, retrlen = self.retrfile(file, type)
- fp = RangeableFileObject(fp, (rest,''))
- return (fp, retrlen)
- elif str(reason)[:3] != '550':
- raise IOError, ('ftp error', reason), sys.exc_info()[2]
- if not conn:
- # Set transfer mode to ASCII!
- self.ftp.voidcmd('TYPE A')
- # Try a directory listing
- if file: cmd = 'LIST ' + file
- else: cmd = 'LIST'
- conn = self.ftp.ntransfercmd(cmd)
- self.busy = 1
- # Pass back both a suitably decorated object and a retrieval length
- return (addclosehook(conn[0].makefile('rb'),
- self.endtransfer), conn[1])
-
-
-####################################################################
-# Range Tuple Functions
-# XXX: These range tuple functions might go better in a class.
-
-_rangere = None
-def range_header_to_tuple(range_header):
- """Get a (firstbyte,lastbyte) tuple from a Range header value.
-
- Range headers have the form "bytes=<firstbyte>-<lastbyte>". This
- function pulls the firstbyte and lastbyte values and returns
- a (firstbyte,lastbyte) tuple. If lastbyte is not specified in
- the header value, it is returned as an empty string in the
- tuple.
-
- Return None if range_header is None
- Return () if range_header does not conform to the range spec
- pattern.
-
- """
- global _rangere
- if range_header is None: return None
- if _rangere is None:
- import re
- _rangere = re.compile(r'^bytes=(\d{1,})-(\d*)')
- match = _rangere.match(range_header)
- if match:
- tup = range_tuple_normalize(match.group(1,2))
- if tup and tup[1]:
- tup = (tup[0],tup[1]+1)
- return tup
- return ()
-
-def range_tuple_to_header(range_tup):
- """Convert a range tuple to a Range header value.
- Return a string of the form "bytes=<firstbyte>-<lastbyte>" or None
- if no range is needed.
- """
- if range_tup is None: return None
- range_tup = range_tuple_normalize(range_tup)
- if range_tup:
- if range_tup[1]:
- range_tup = (range_tup[0],range_tup[1] - 1)
- return 'bytes=%s-%s' % range_tup
-
-def range_tuple_normalize(range_tup):
- """Normalize a (first_byte,last_byte) range tuple.
- Return a tuple whose first element is guaranteed to be an int
- and whose second element will be '' (meaning: the last byte) or
- an int. Finally, return None if the normalized tuple == (0,'')
- as that is equivelant to retrieving the entire file.
- """
- if range_tup is None: return None
- # handle first byte
- fb = range_tup[0]
- if fb in (None,''): fb = 0
- else: fb = int(fb)
- # handle last byte
- try: lb = range_tup[1]
- except IndexError: lb = ''
- else:
- if lb is None: lb = ''
- elif lb != '': lb = int(lb)
- # check if range is over the entire file
- if (fb,lb) == (0,''): return None
- # check that the range is valid
- if lb < fb: raise RangeError('Invalid byte range: %s-%s' % (fb,lb))
- return (fb,lb)
-
+++ /dev/null
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330,
-# Boston, MA 02111-1307 USA
-
-# This file is part of urlgrabber, a high-level cross-protocol url-grabber
-# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
-
-"""A high-level cross-protocol url-grabber.
-
-GENERAL ARGUMENTS (kwargs)
-
- Where possible, the module-level default is indicated, and legal
- values are provided.
-
- copy_local = 0 [0|1]
-
- ignored except for file:// urls, in which case it specifies
- whether urlgrab should still make a copy of the file, or simply
- point to the existing copy. The module level default for this
- option is 0.
-
- close_connection = 0 [0|1]
-
- tells URLGrabber to close the connection after a file has been
- transfered. This is ignored unless the download happens with the
- http keepalive handler (keepalive=1). Otherwise, the connection
- is left open for further use. The module level default for this
- option is 0 (keepalive connections will not be closed).
-
- keepalive = 1 [0|1]
-
- specifies whether keepalive should be used for HTTP/1.1 servers
- that support it. The module level default for this option is 1
- (keepalive is enabled).
-
- progress_obj = None
-
- a class instance that supports the following methods:
- po.start(filename, url, basename, length, text)
- # length will be None if unknown
- po.update(read) # read == bytes read so far
- po.end()
-
- text = None
-
- specifies an alternativ text item in the beginning of the progress
- bar line. If not given, the basename of the file is used.
-
- throttle = 1.0
-
- a number - if it's an int, it's the bytes/second throttle limit.
- If it's a float, it is first multiplied by bandwidth. If throttle
- == 0, throttling is disabled. If None, the module-level default
- (which can be set on default_grabber.throttle) is used. See
- BANDWIDTH THROTTLING for more information.
-
- timeout = None
-
- a positive float expressing the number of seconds to wait for socket
- operations. If the value is None or 0.0, socket operations will block
- forever. Setting this option causes urlgrabber to call the settimeout
- method on the Socket object used for the request. See the Python
- documentation on settimeout for more information.
- http://www.python.org/doc/current/lib/socket-objects.html
-
- bandwidth = 0
-
- the nominal max bandwidth in bytes/second. If throttle is a float
- and bandwidth == 0, throttling is disabled. If None, the
- module-level default (which can be set on
- default_grabber.bandwidth) is used. See BANDWIDTH THROTTLING for
- more information.
-
- range = None
-
- a tuple of the form (first_byte, last_byte) describing a byte
- range to retrieve. Either or both of the values may set to
- None. If first_byte is None, byte offset 0 is assumed. If
- last_byte is None, the last byte available is assumed. Note that
- the range specification is python-like in that (0,10) will yeild
- the first 10 bytes of the file.
-
- If set to None, no range will be used.
-
- reget = None [None|'simple'|'check_timestamp']
-
- whether to attempt to reget a partially-downloaded file. Reget
- only applies to .urlgrab and (obviously) only if there is a
- partially downloaded file. Reget has two modes:
-
- 'simple' -- the local file will always be trusted. If there
- are 100 bytes in the local file, then the download will always
- begin 100 bytes into the requested file.
-
- 'check_timestamp' -- the timestamp of the server file will be
- compared to the timestamp of the local file. ONLY if the
- local file is newer than or the same age as the server file
- will reget be used. If the server file is newer, or the
- timestamp is not returned, the entire file will be fetched.
-
- NOTE: urlgrabber can do very little to verify that the partial
- file on disk is identical to the beginning of the remote file.
- You may want to either employ a custom "checkfunc" or simply avoid
- using reget in situations where corruption is a concern.
-
- user_agent = 'urlgrabber/VERSION'
-
- a string, usually of the form 'AGENT/VERSION' that is provided to
- HTTP servers in the User-agent header. The module level default
- for this option is "urlgrabber/VERSION".
-
- http_headers = None
-
- a tuple of 2-tuples, each containing a header and value. These
- will be used for http and https requests only. For example, you
- can do
- http_headers = (('Pragma', 'no-cache'),)
-
- ftp_headers = None
-
- this is just like http_headers, but will be used for ftp requests.
-
- proxies = None
-
- a dictionary that maps protocol schemes to proxy hosts. For
- example, to use a proxy server on host "foo" port 3128 for http
- and https URLs:
- proxies={ 'http' : 'http://foo:3128', 'https' : 'http://foo:3128' }
- note that proxy authentication information may be provided using
- normal URL constructs:
- proxies={ 'http' : 'http://user:host@foo:3128' }
- Lastly, if proxies is None, the default environment settings will
- be used.
-
- prefix = None
-
- a url prefix that will be prepended to all requested urls. For
- example:
- g = URLGrabber(prefix='http://foo.com/mirror/')
- g.urlgrab('some/file.txt')
- ## this will fetch 'http://foo.com/mirror/some/file.txt'
- This option exists primarily to allow identical behavior to
- MirrorGroup (and derived) instances. Note: a '/' will be inserted
- if necessary, so you cannot specify a prefix that ends with a
- partial file or directory name.
-
- opener = None
-
- Overrides the default urllib2.OpenerDirector provided to urllib2
- when making requests. This option exists so that the urllib2
- handler chain may be customized. Note that the range, reget,
- proxy, and keepalive features require that custom handlers be
- provided to urllib2 in order to function properly. If an opener
- option is provided, no attempt is made by urlgrabber to ensure
- chain integrity. You are responsible for ensuring that any
- extension handlers are present if said features are required.
-
- data = None
-
- Only relevant for the HTTP family (and ignored for other
- protocols), this allows HTTP POSTs. When the data kwarg is
- present (and not None), an HTTP request will automatically become
- a POST rather than GET. This is done by direct passthrough to
- urllib2. If you use this, you may also want to set the
- 'Content-length' and 'Content-type' headers with the http_headers
- option. Note that python 2.2 handles the case of these
- badly and if you do not use the proper case (shown here), your
- values will be overridden with the defaults.
-
-
-RETRY RELATED ARGUMENTS
-
- retry = None
-
- the number of times to retry the grab before bailing. If this is
- zero, it will retry forever. This was intentional... really, it
- was :). If this value is not supplied or is supplied but is None
- retrying does not occur.
-
- retrycodes = [-1,2,4,5,6,7]
-
- a sequence of errorcodes (values of e.errno) for which it should
- retry. See the doc on URLGrabError for more details on this. You
- might consider modifying a copy of the default codes rather than
- building yours from scratch so that if the list is extended in the
- future (or one code is split into two) you can still enjoy the
- benefits of the default list. You can do that with something like
- this:
-
- retrycodes = urlgrabber.grabber.URLGrabberOptions().retrycodes
- if 12 not in retrycodes:
- retrycodes.append(12)
-
- checkfunc = None
-
- a function to do additional checks. This defaults to None, which
- means no additional checking. The function should simply return
- on a successful check. It should raise URLGrabError on an
- unsuccessful check. Raising of any other exception will be
- considered immediate failure and no retries will occur.
-
- If it raises URLGrabError, the error code will determine the retry
- behavior. Negative error numbers are reserved for use by these
- passed in functions, so you can use many negative numbers for
- different types of failure. By default, -1 results in a retry,
- but this can be customized with retrycodes.
-
- If you simply pass in a function, it will be given exactly one
- argument: a CallbackObject instance with the .url attribute
- defined and either .filename (for urlgrab) or .data (for urlread).
- For urlgrab, .filename is the name of the local file. For
- urlread, .data is the actual string data. If you need other
- arguments passed to the callback (program state of some sort), you
- can do so like this:
-
- checkfunc=(function, ('arg1', 2), {'kwarg': 3})
-
- if the downloaded file has filename /tmp/stuff, then this will
- result in this call (for urlgrab):
-
- function(obj, 'arg1', 2, kwarg=3)
- # obj.filename = '/tmp/stuff'
- # obj.url = 'http://foo.com/stuff'
-
- NOTE: both the "args" tuple and "kwargs" dict must be present if
- you use this syntax, but either (or both) can be empty.
-
- failure_callback = None
-
- The callback that gets called during retries when an attempt to
- fetch a file fails. The syntax for specifying the callback is
- identical to checkfunc, except for the attributes defined in the
- CallbackObject instance. The attributes for failure_callback are:
-
- exception = the raised exception
- url = the url we're trying to fetch
- tries = the number of tries so far (including this one)
- retry = the value of the retry option
-
- The callback is present primarily to inform the calling program of
- the failure, but if it raises an exception (including the one it's
- passed) that exception will NOT be caught and will therefore cause
- future retries to be aborted.
-
- The callback is called for EVERY failure, including the last one.
- On the last try, the callback can raise an alternate exception,
- but it cannot (without severe trickiness) prevent the exception
- from being raised.
-
- interrupt_callback = None
-
- This callback is called if KeyboardInterrupt is received at any
- point in the transfer. Basically, this callback can have three
- impacts on the fetch process based on the way it exits:
-
- 1) raise no exception: the current fetch will be aborted, but
- any further retries will still take place
-
- 2) raise a URLGrabError: if you're using a MirrorGroup, then
- this will prompt a failover to the next mirror according to
- the behavior of the MirrorGroup subclass. It is recommended
- that you raise URLGrabError with code 15, 'user abort'. If
- you are NOT using a MirrorGroup subclass, then this is the
- same as (3).
-
- 3) raise some other exception (such as KeyboardInterrupt), which
- will not be caught at either the grabber or mirror levels.
- That is, it will be raised up all the way to the caller.
-
- This callback is very similar to failure_callback. They are
- passed the same arguments, so you could use the same function for
- both.
-
- urlparser = URLParser()
-
- The URLParser class handles pre-processing of URLs, including
- auth-handling for user/pass encoded in http urls, file handing
- (that is, filenames not sent as a URL), and URL quoting. If you
- want to override any of this behavior, you can pass in a
- replacement instance. See also the 'quote' option.
-
- quote = None
-
- Whether or not to quote the path portion of a url.
- quote = 1 -> quote the URLs (they're not quoted yet)
- quote = 0 -> do not quote them (they're already quoted)
- quote = None -> guess what to do
-
- This option only affects proper urls like 'file:///etc/passwd'; it
- does not affect 'raw' filenames like '/etc/passwd'. The latter
- will always be quoted as they are converted to URLs. Also, only
- the path part of a url is quoted. If you need more fine-grained
- control, you should probably subclass URLParser and pass it in via
- the 'urlparser' option.
-
-BANDWIDTH THROTTLING
-
- urlgrabber supports throttling via two values: throttle and
- bandwidth Between the two, you can either specify and absolute
- throttle threshold or specify a theshold as a fraction of maximum
- available bandwidth.
-
- throttle is a number - if it's an int, it's the bytes/second
- throttle limit. If it's a float, it is first multiplied by
- bandwidth. If throttle == 0, throttling is disabled. If None, the
- module-level default (which can be set with set_throttle) is used.
-
- bandwidth is the nominal max bandwidth in bytes/second. If throttle
- is a float and bandwidth == 0, throttling is disabled. If None, the
- module-level default (which can be set with set_bandwidth) is used.
-
- THROTTLING EXAMPLES:
-
- Lets say you have a 100 Mbps connection. This is (about) 10^8 bits
- per second, or 12,500,000 Bytes per second. You have a number of
- throttling options:
-
- *) set_bandwidth(12500000); set_throttle(0.5) # throttle is a float
-
- This will limit urlgrab to use half of your available bandwidth.
-
- *) set_throttle(6250000) # throttle is an int
-
- This will also limit urlgrab to use half of your available
- bandwidth, regardless of what bandwidth is set to.
-
- *) set_throttle(6250000); set_throttle(1.0) # float
-
- Use half your bandwidth
-
- *) set_throttle(6250000); set_throttle(2.0) # float
-
- Use up to 12,500,000 Bytes per second (your nominal max bandwidth)
-
- *) set_throttle(6250000); set_throttle(0) # throttle = 0
-
- Disable throttling - this is more efficient than a very large
- throttle setting.
-
- *) set_throttle(0); set_throttle(1.0) # throttle is float, bandwidth = 0
-
- Disable throttling - this is the default when the module is loaded.
-
- SUGGESTED AUTHOR IMPLEMENTATION (THROTTLING)
-
- While this is flexible, it's not extremely obvious to the user. I
- suggest you implement a float throttle as a percent to make the
- distinction between absolute and relative throttling very explicit.
-
- Also, you may want to convert the units to something more convenient
- than bytes/second, such as kbps or kB/s, etc.
-
-"""
-
-# $Id: grabber.py,v 1.48 2006/09/22 00:58:05 mstenner Exp $
-
-import os
-import os.path
-import sys
-import urlparse
-import rfc822
-import time
-import string
-import urllib
-import urllib2
-from stat import * # S_* and ST_*
-
-########################################################################
-# MODULE INITIALIZATION
-########################################################################
-try:
- exec('from ' + (__name__.split('.'))[0] + ' import __version__')
-except:
- __version__ = '???'
-
-import sslfactory
-
-auth_handler = urllib2.HTTPBasicAuthHandler( \
- urllib2.HTTPPasswordMgrWithDefaultRealm())
-
-try:
- from i18n import _
-except ImportError, msg:
- def _(st): return st
-
-try:
- from httplib import HTTPException
-except ImportError, msg:
- HTTPException = None
-
-try:
- # This is a convenient way to make keepalive optional.
- # Just rename the module so it can't be imported.
- import keepalive
- from keepalive import HTTPHandler, HTTPSHandler
- have_keepalive = True
-except ImportError, msg:
- have_keepalive = False
-
-try:
- # add in range support conditionally too
- import byterange
- from byterange import HTTPRangeHandler, HTTPSRangeHandler, \
- FileRangeHandler, FTPRangeHandler, range_tuple_normalize, \
- range_tuple_to_header, RangeError
-except ImportError, msg:
- range_handlers = ()
- RangeError = None
- have_range = 0
-else:
- range_handlers = (HTTPRangeHandler(), HTTPSRangeHandler(),
- FileRangeHandler(), FTPRangeHandler())
- have_range = 1
-
-
-# check whether socket timeout support is available (Python >= 2.3)
-import socket
-try:
- TimeoutError = socket.timeout
- have_socket_timeout = True
-except AttributeError:
- TimeoutError = None
- have_socket_timeout = False
-
-########################################################################
-# functions for debugging output. These functions are here because they
-# are also part of the module initialization.
-DEBUG = None
-def set_logger(DBOBJ):
- """Set the DEBUG object. This is called by _init_default_logger when
- the environment variable URLGRABBER_DEBUG is set, but can also be
- called by a calling program. Basically, if the calling program uses
- the logging module and would like to incorporate urlgrabber logging,
- then it can do so this way. It's probably not necessary as most
- internal logging is only for debugging purposes.
-
- The passed-in object should be a logging.Logger instance. It will
- be pushed into the keepalive and byterange modules if they're
- being used. The mirror module pulls this object in on import, so
- you will need to manually push into it. In fact, you may find it
- tidier to simply push your logging object (or objects) into each
- of these modules independently.
- """
-
- global DEBUG
- DEBUG = DBOBJ
- if have_keepalive and keepalive.DEBUG is None:
- keepalive.DEBUG = DBOBJ
- if have_range and byterange.DEBUG is None:
- byterange.DEBUG = DBOBJ
- if sslfactory.DEBUG is None:
- sslfactory.DEBUG = DBOBJ
-
-def _init_default_logger():
- '''Examines the environment variable URLGRABBER_DEBUG and creates
- a logging object (logging.logger) based on the contents. It takes
- the form
-
- URLGRABBER_DEBUG=level,filename
-
- where "level" can be either an integer or a log level from the
- logging module (DEBUG, INFO, etc). If the integer is zero or
- less, logging will be disabled. Filename is the filename where
- logs will be sent. If it is "-", then stdout will be used. If
- the filename is empty or missing, stderr will be used. If the
- variable cannot be processed or the logging module cannot be
- imported (python < 2.3) then logging will be disabled. Here are
- some examples:
-
- URLGRABBER_DEBUG=1,debug.txt # log everything to debug.txt
- URLGRABBER_DEBUG=WARNING,- # log warning and higher to stdout
- URLGRABBER_DEBUG=INFO # log info and higher to stderr
-
- This funtion is called during module initialization. It is not
- intended to be called from outside. The only reason it is a
- function at all is to keep the module-level namespace tidy and to
- collect the code into a nice block.'''
-
- try:
- dbinfo = os.environ['URLGRABBER_DEBUG'].split(',')
- import logging
- level = logging._levelNames.get(dbinfo[0], int(dbinfo[0]))
- if level < 1: raise ValueError()
-
- formatter = logging.Formatter('%(asctime)s %(message)s')
- if len(dbinfo) > 1: filename = dbinfo[1]
- else: filename = ''
- if filename == '': handler = logging.StreamHandler(sys.stderr)
- elif filename == '-': handler = logging.StreamHandler(sys.stdout)
- else: handler = logging.FileHandler(filename)
- handler.setFormatter(formatter)
- DBOBJ = logging.getLogger('urlgrabber')
- DBOBJ.addHandler(handler)
- DBOBJ.setLevel(level)
- except (KeyError, ImportError, ValueError):
- DBOBJ = None
- set_logger(DBOBJ)
-
-_init_default_logger()
-########################################################################
-# END MODULE INITIALIZATION
-########################################################################
-
-
-
-class URLGrabError(IOError):
- """
- URLGrabError error codes:
-
- URLGrabber error codes (0 -- 255)
- 0 - everything looks good (you should never see this)
- 1 - malformed url
- 2 - local file doesn't exist
- 3 - request for non-file local file (dir, etc)
- 4 - IOError on fetch
- 5 - OSError on fetch
- 6 - no content length header when we expected one
- 7 - HTTPException
- 8 - Exceeded read limit (for urlread)
- 9 - Requested byte range not satisfiable.
- 10 - Byte range requested, but range support unavailable
- 11 - Illegal reget mode
- 12 - Socket timeout
- 13 - malformed proxy url
- 14 - HTTPError (includes .code and .exception attributes)
- 15 - user abort
-
- MirrorGroup error codes (256 -- 511)
- 256 - No more mirrors left to try
-
- Custom (non-builtin) classes derived from MirrorGroup (512 -- 767)
- [ this range reserved for application-specific error codes ]
-
- Retry codes (< 0)
- -1 - retry the download, unknown reason
-
- Note: to test which group a code is in, you can simply do integer
- division by 256: e.errno / 256
-
- Negative codes are reserved for use by functions passed in to
- retrygrab with checkfunc. The value -1 is built in as a generic
- retry code and is already included in the retrycodes list.
- Therefore, you can create a custom check function that simply
- returns -1 and the fetch will be re-tried. For more customized
- retries, you can use other negative number and include them in
- retry-codes. This is nice for outputting useful messages about
- what failed.
-
- You can use these error codes like so:
- try: urlgrab(url)
- except URLGrabError, e:
- if e.errno == 3: ...
- # or
- print e.strerror
- # or simply
- print e #### print '[Errno %i] %s' % (e.errno, e.strerror)
- """
- pass
-
-class CallbackObject:
- """Container for returned callback data.
-
- This is currently a dummy class into which urlgrabber can stuff
- information for passing to callbacks. This way, the prototype for
- all callbacks is the same, regardless of the data that will be
- passed back. Any function that accepts a callback function as an
- argument SHOULD document what it will define in this object.
-
- It is possible that this class will have some greater
- functionality in the future.
- """
- def __init__(self, **kwargs):
- self.__dict__.update(kwargs)
-
-def urlgrab(url, filename=None, **kwargs):
- """grab the file at <url> and make a local copy at <filename>
- If filename is none, the basename of the url is used.
- urlgrab returns the filename of the local file, which may be different
- from the passed-in filename if the copy_local kwarg == 0.
-
- See module documentation for a description of possible kwargs.
- """
- return default_grabber.urlgrab(url, filename, **kwargs)
-
-def urlopen(url, **kwargs):
- """open the url and return a file object
- If a progress object or throttle specifications exist, then
- a special file object will be returned that supports them.
- The file object can be treated like any other file object.
-
- See module documentation for a description of possible kwargs.
- """
- return default_grabber.urlopen(url, **kwargs)
-
-def urlread(url, limit=None, **kwargs):
- """read the url into a string, up to 'limit' bytes
- If the limit is exceeded, an exception will be thrown. Note that urlread
- is NOT intended to be used as a way of saying "I want the first N bytes"
- but rather 'read the whole file into memory, but don't use too much'
-
- See module documentation for a description of possible kwargs.
- """
- return default_grabber.urlread(url, limit, **kwargs)
-
-
-class URLParser:
- """Process the URLs before passing them to urllib2.
-
- This class does several things:
-
- * add any prefix
- * translate a "raw" file to a proper file: url
- * handle any http or https auth that's encoded within the url
- * quote the url
-
- Only the "parse" method is called directly, and it calls sub-methods.
-
- An instance of this class is held in the options object, which
- means that it's easy to change the behavior by sub-classing and
- passing the replacement in. It need only have a method like:
-
- url, parts = urlparser.parse(url, opts)
- """
-
- def parse(self, url, opts):
- """parse the url and return the (modified) url and its parts
-
- Note: a raw file WILL be quoted when it's converted to a URL.
- However, other urls (ones which come with a proper scheme) may
- or may not be quoted according to opts.quote
-
- opts.quote = 1 --> quote it
- opts.quote = 0 --> do not quote it
- opts.quote = None --> guess
- """
- quote = opts.quote
-
- if opts.prefix:
- url = self.add_prefix(url, opts.prefix)
-
- parts = urlparse.urlparse(url)
- (scheme, host, path, parm, query, frag) = parts
-
- if not scheme or (len(scheme) == 1 and scheme in string.letters):
- # if a scheme isn't specified, we guess that it's "file:"
- if url[0] not in '/\\': url = os.path.abspath(url)
- url = 'file:' + urllib.pathname2url(url)
- parts = urlparse.urlparse(url)
- quote = 0 # pathname2url quotes, so we won't do it again
-
- if scheme in ['http', 'https']:
- parts = self.process_http(parts)
-
- if quote is None:
- quote = self.guess_should_quote(parts)
- if quote:
- parts = self.quote(parts)
-
- url = urlparse.urlunparse(parts)
- return url, parts
-
- def add_prefix(self, url, prefix):
- if prefix[-1] == '/' or url[0] == '/':
- url = prefix + url
- else:
- url = prefix + '/' + url
- return url
-
- def process_http(self, parts):
- (scheme, host, path, parm, query, frag) = parts
-
- if '@' in host and auth_handler:
- try:
- user_pass, host = host.split('@', 1)
- if ':' in user_pass:
- user, password = user_pass.split(':', 1)
- except ValueError, e:
- raise URLGrabError(1, _('Bad URL: %s') % url)
- if DEBUG: DEBUG.info('adding HTTP auth: %s, %s', user, password)
- auth_handler.add_password(None, host, user, password)
-
- return (scheme, host, path, parm, query, frag)
-
- def quote(self, parts):
- """quote the URL
-
- This method quotes ONLY the path part. If you need to quote
- other parts, you should override this and pass in your derived
- class. The other alternative is to quote other parts before
- passing into urlgrabber.
- """
- (scheme, host, path, parm, query, frag) = parts
- path = urllib.quote(path)
- return (scheme, host, path, parm, query, frag)
-
- hexvals = '0123456789ABCDEF'
- def guess_should_quote(self, parts):
- """
- Guess whether we should quote a path. This amounts to
- guessing whether it's already quoted.
-
- find ' ' -> 1
- find '%' -> 1
- find '%XX' -> 0
- else -> 1
- """
- (scheme, host, path, parm, query, frag) = parts
- if ' ' in path:
- return 1
- ind = string.find(path, '%')
- if ind > -1:
- while ind > -1:
- if len(path) < ind+3:
- return 1
- code = path[ind+1:ind+3].upper()
- if code[0] not in self.hexvals or \
- code[1] not in self.hexvals:
- return 1
- ind = string.find(path, '%', ind+1)
- return 0
- return 1
-
-class URLGrabberOptions:
- """Class to ease kwargs handling."""
-
- def __init__(self, delegate=None, **kwargs):
- """Initialize URLGrabberOptions object.
- Set default values for all options and then update options specified
- in kwargs.
- """
- self.delegate = delegate
- if delegate is None:
- self._set_defaults()
- self._set_attributes(**kwargs)
-
- def __getattr__(self, name):
- if self.delegate and hasattr(self.delegate, name):
- return getattr(self.delegate, name)
- raise AttributeError, name
-
- def raw_throttle(self):
- """Calculate raw throttle value from throttle and bandwidth
- values.
- """
- if self.throttle <= 0:
- return 0
- elif type(self.throttle) == type(0):
- return float(self.throttle)
- else: # throttle is a float
- return self.bandwidth * self.throttle
-
- def derive(self, **kwargs):
- """Create a derived URLGrabberOptions instance.
- This method creates a new instance and overrides the
- options specified in kwargs.
- """
- return URLGrabberOptions(delegate=self, **kwargs)
-
- def _set_attributes(self, **kwargs):
- """Update object attributes with those provided in kwargs."""
- self.__dict__.update(kwargs)
- if have_range and kwargs.has_key('range'):
- # normalize the supplied range value
- self.range = range_tuple_normalize(self.range)
- if not self.reget in [None, 'simple', 'check_timestamp']:
- raise URLGrabError(11, _('Illegal reget mode: %s') \
- % (self.reget, ))
-
- def _set_defaults(self):
- """Set all options to their default values.
- When adding new options, make sure a default is
- provided here.
- """
- self.progress_obj = None
- self.throttle = 1.0
- self.bandwidth = 0
- self.retry = None
- self.retrycodes = [-1,2,4,5,6,7]
- self.checkfunc = None
- self.copy_local = 0
- self.close_connection = 0
- self.range = None
- self.user_agent = 'urlgrabber/%s' % __version__
- self.keepalive = 1
- self.proxies = None
- self.reget = None
- self.failure_callback = None
- self.interrupt_callback = None
- self.prefix = None
- self.opener = None
- self.cache_openers = True
- self.timeout = None
- self.text = None
- self.http_headers = None
- self.ftp_headers = None
- self.data = None
- self.urlparser = URLParser()
- self.quote = None
- self.ssl_ca_cert = None
- self.ssl_context = None
-
-class URLGrabber:
- """Provides easy opening of URLs with a variety of options.
-
- All options are specified as kwargs. Options may be specified when
- the class is created and may be overridden on a per request basis.
-
- New objects inherit default values from default_grabber.
- """
-
- def __init__(self, **kwargs):
- self.opts = URLGrabberOptions(**kwargs)
-
- def _retry(self, opts, func, *args):
- tries = 0
- while 1:
- # there are only two ways out of this loop. The second has
- # several "sub-ways"
- # 1) via the return in the "try" block
- # 2) by some exception being raised
- # a) an excepton is raised that we don't "except"
- # b) a callback raises ANY exception
- # c) we're not retry-ing or have run out of retries
- # d) the URLGrabError code is not in retrycodes
- # beware of infinite loops :)
- tries = tries + 1
- exception = None
- retrycode = None
- callback = None
- if DEBUG: DEBUG.info('attempt %i/%s: %s',
- tries, opts.retry, args[0])
- try:
- r = apply(func, (opts,) + args, {})
- if DEBUG: DEBUG.info('success')
- return r
- except URLGrabError, e:
- exception = e
- callback = opts.failure_callback
- retrycode = e.errno
- except KeyboardInterrupt, e:
- exception = e
- callback = opts.interrupt_callback
-
- if DEBUG: DEBUG.info('exception: %s', exception)
- if callback:
- if DEBUG: DEBUG.info('calling callback: %s', callback)
- cb_func, cb_args, cb_kwargs = self._make_callback(callback)
- obj = CallbackObject(exception=exception, url=args[0],
- tries=tries, retry=opts.retry)
- cb_func(obj, *cb_args, **cb_kwargs)
-
- if (opts.retry is None) or (tries == opts.retry):
- if DEBUG: DEBUG.info('retries exceeded, re-raising')
- raise
-
- if (retrycode is not None) and (retrycode not in opts.retrycodes):
- if DEBUG: DEBUG.info('retrycode (%i) not in list %s, re-raising',
- retrycode, opts.retrycodes)
- raise
-
- def urlopen(self, url, **kwargs):
- """open the url and return a file object
- If a progress object or throttle value specified when this
- object was created, then a special file object will be
- returned that supports them. The file object can be treated
- like any other file object.
- """
- opts = self.opts.derive(**kwargs)
- (url,parts) = opts.urlparser.parse(url, opts)
- def retryfunc(opts, url):
- return URLGrabberFileObject(url, filename=None, opts=opts)
- return self._retry(opts, retryfunc, url)
-
- def urlgrab(self, url, filename=None, **kwargs):
- """grab the file at <url> and make a local copy at <filename>
- If filename is none, the basename of the url is used.
- urlgrab returns the filename of the local file, which may be
- different from the passed-in filename if copy_local == 0.
- """
- opts = self.opts.derive(**kwargs)
- (url,parts) = opts.urlparser.parse(url, opts)
- (scheme, host, path, parm, query, frag) = parts
- if filename is None:
- filename = os.path.basename( urllib.unquote(path) )
- if scheme == 'file' and not opts.copy_local:
- # just return the name of the local file - don't make a
- # copy currently
- path = urllib.url2pathname(path)
- if host:
- path = os.path.normpath('//' + host + path)
- if not os.path.exists(path):
- raise URLGrabError(2,
- _('Local file does not exist: %s') % (path, ))
- elif not os.path.isfile(path):
- raise URLGrabError(3,
- _('Not a normal file: %s') % (path, ))
- elif not opts.range:
- return path
-
- def retryfunc(opts, url, filename):
- fo = URLGrabberFileObject(url, filename, opts)
- try:
- fo._do_grab()
- if not opts.checkfunc is None:
- cb_func, cb_args, cb_kwargs = \
- self._make_callback(opts.checkfunc)
- obj = CallbackObject()
- obj.filename = filename
- obj.url = url
- apply(cb_func, (obj, )+cb_args, cb_kwargs)
- finally:
- fo.close()
- return filename
-
- return self._retry(opts, retryfunc, url, filename)
-
- def urlread(self, url, limit=None, **kwargs):
- """read the url into a string, up to 'limit' bytes
- If the limit is exceeded, an exception will be thrown. Note
- that urlread is NOT intended to be used as a way of saying
- "I want the first N bytes" but rather 'read the whole file
- into memory, but don't use too much'
- """
- opts = self.opts.derive(**kwargs)
- (url,parts) = opts.urlparser.parse(url, opts)
- if limit is not None:
- limit = limit + 1
-
- def retryfunc(opts, url, limit):
- fo = URLGrabberFileObject(url, filename=None, opts=opts)
- s = ''
- try:
- # this is an unfortunate thing. Some file-like objects
- # have a default "limit" of None, while the built-in (real)
- # file objects have -1. They each break the other, so for
- # now, we just force the default if necessary.
- if limit is None: s = fo.read()
- else: s = fo.read(limit)
-
- if not opts.checkfunc is None:
- cb_func, cb_args, cb_kwargs = \
- self._make_callback(opts.checkfunc)
- obj = CallbackObject()
- obj.data = s
- obj.url = url
- apply(cb_func, (obj, )+cb_args, cb_kwargs)
- finally:
- fo.close()
- return s
-
- s = self._retry(opts, retryfunc, url, limit)
- if limit and len(s) > limit:
- raise URLGrabError(8,
- _('Exceeded limit (%i): %s') % (limit, url))
- return s
-
- def _make_callback(self, callback_obj):
- if callable(callback_obj):
- return callback_obj, (), {}
- else:
- return callback_obj
-
-# create the default URLGrabber used by urlXXX functions.
-# NOTE: actual defaults are set in URLGrabberOptions
-default_grabber = URLGrabber()
-
-class URLGrabberFileObject:
- """This is a file-object wrapper that supports progress objects
- and throttling.
-
- This exists to solve the following problem: lets say you want to
- drop-in replace a normal open with urlopen. You want to use a
- progress meter and/or throttling, but how do you do that without
- rewriting your code? Answer: urlopen will return a wrapped file
- object that does the progress meter and-or throttling internally.
- """
-
- def __init__(self, url, filename, opts):
- self.url = url
- self.filename = filename
- self.opts = opts
- self.fo = None
- self._rbuf = ''
- self._rbufsize = 1024*8
- self._ttime = time.time()
- self._tsize = 0
- self._amount_read = 0
- self._opener = None
- self._do_open()
-
- def __getattr__(self, name):
- """This effectively allows us to wrap at the instance level.
- Any attribute not found in _this_ object will be searched for
- in self.fo. This includes methods."""
- if hasattr(self.fo, name):
- return getattr(self.fo, name)
- raise AttributeError, name
-
- def _get_opener(self):
- """Build a urllib2 OpenerDirector based on request options."""
- if self.opts.opener:
- return self.opts.opener
- elif self._opener is None:
- handlers = []
- need_keepalive_handler = (have_keepalive and self.opts.keepalive)
- need_range_handler = (range_handlers and \
- (self.opts.range or self.opts.reget))
- # if you specify a ProxyHandler when creating the opener
- # it _must_ come before all other handlers in the list or urllib2
- # chokes.
- if self.opts.proxies:
- handlers.append( CachedProxyHandler(self.opts.proxies) )
-
- # -------------------------------------------------------
- # OK, these next few lines are a serious kludge to get
- # around what I think is a bug in python 2.2's
- # urllib2. The basic idea is that default handlers
- # get applied first. If you override one (like a
- # proxy handler), then the default gets pulled, but
- # the replacement goes on the end. In the case of
- # proxies, this means the normal handler picks it up
- # first and the proxy isn't used. Now, this probably
- # only happened with ftp or non-keepalive http, so not
- # many folks saw it. The simple approach to fixing it
- # is just to make sure you override the other
- # conflicting defaults as well. I would LOVE to see
- # these go way or be dealt with more elegantly. The
- # problem isn't there after 2.2. -MDS 2005/02/24
- if not need_keepalive_handler:
- handlers.append( urllib2.HTTPHandler() )
- if not need_range_handler:
- handlers.append( urllib2.FTPHandler() )
- # -------------------------------------------------------
-
- ssl_factory = sslfactory.get_factory(self.opts.ssl_ca_cert,
- self.opts.ssl_context)
-
- if need_keepalive_handler:
- handlers.append(HTTPHandler())
- handlers.append(HTTPSHandler(ssl_factory))
- if need_range_handler:
- handlers.extend( range_handlers )
- handlers.append( auth_handler )
- if self.opts.cache_openers:
- self._opener = CachedOpenerDirector(ssl_factory, *handlers)
- else:
- self._opener = ssl_factory.create_opener(*handlers)
- # OK, I don't like to do this, but otherwise, we end up with
- # TWO user-agent headers.
- self._opener.addheaders = []
- return self._opener
-
- def _do_open(self):
- opener = self._get_opener()
-
- req = urllib2.Request(self.url, self.opts.data) # build request object
- self._add_headers(req) # add misc headers that we need
- self._build_range(req) # take care of reget and byterange stuff
-
- fo, hdr = self._make_request(req, opener)
- if self.reget_time and self.opts.reget == 'check_timestamp':
- # do this if we have a local file with known timestamp AND
- # we're in check_timestamp reget mode.
- fetch_again = 0
- try:
- modified_tuple = hdr.getdate_tz('last-modified')
- modified_stamp = rfc822.mktime_tz(modified_tuple)
- if modified_stamp > self.reget_time: fetch_again = 1
- except (TypeError,):
- fetch_again = 1
-
- if fetch_again:
- # the server version is newer than the (incomplete) local
- # version, so we should abandon the version we're getting
- # and fetch the whole thing again.
- fo.close()
- self.opts.reget = None
- del req.headers['Range']
- self._build_range(req)
- fo, hdr = self._make_request(req, opener)
-
- (scheme, host, path, parm, query, frag) = urlparse.urlparse(self.url)
- path = urllib.unquote(path)
- if not (self.opts.progress_obj or self.opts.raw_throttle() \
- or self.opts.timeout):
- # if we're not using the progress_obj, throttling, or timeout
- # we can get a performance boost by going directly to
- # the underlying fileobject for reads.
- self.read = fo.read
- if hasattr(fo, 'readline'):
- self.readline = fo.readline
- elif self.opts.progress_obj:
- try:
- length = int(hdr['Content-Length'])
- length = length + self._amount_read # Account for regets
- except (KeyError, ValueError, TypeError):
- length = None
-
- self.opts.progress_obj.start(str(self.filename),
- urllib.unquote(self.url),
- os.path.basename(path),
- length, text=self.opts.text)
- self.opts.progress_obj.update(0)
- (self.fo, self.hdr) = (fo, hdr)
-
- def _add_headers(self, req):
- if self.opts.user_agent:
- req.add_header('User-agent', self.opts.user_agent)
- try: req_type = req.get_type()
- except ValueError: req_type = None
- if self.opts.http_headers and req_type in ('http', 'https'):
- for h, v in self.opts.http_headers:
- req.add_header(h, v)
- if self.opts.ftp_headers and req_type == 'ftp':
- for h, v in self.opts.ftp_headers:
- req.add_header(h, v)
-
- def _build_range(self, req):
- self.reget_time = None
- self.append = 0
- reget_length = 0
- rt = None
- if have_range and self.opts.reget and type(self.filename) == type(''):
- # we have reget turned on and we're dumping to a file
- try:
- s = os.stat(self.filename)
- except OSError:
- pass
- else:
- self.reget_time = s[ST_MTIME]
- reget_length = s[ST_SIZE]
-
- # Set initial length when regetting
- self._amount_read = reget_length
-
- rt = reget_length, ''
- self.append = 1
-
- if self.opts.range:
- if not have_range:
- raise URLGrabError(10, _('Byte range requested but range '\
- 'support unavailable'))
- rt = self.opts.range
- if rt[0]: rt = (rt[0] + reget_length, rt[1])
-
- if rt:
- header = range_tuple_to_header(rt)
- if header: req.add_header('Range', header)
-
- def _make_request(self, req, opener):
- try:
- if have_socket_timeout and self.opts.timeout:
- old_to = socket.getdefaulttimeout()
- socket.setdefaulttimeout(self.opts.timeout)
- try:
- fo = opener.open(req)
- finally:
- socket.setdefaulttimeout(old_to)
- else:
- fo = opener.open(req)
- hdr = fo.info()
- except ValueError, e:
- raise URLGrabError(1, _('Bad URL: %s') % (e, ))
- except RangeError, e:
- raise URLGrabError(9, str(e))
- except urllib2.HTTPError, e:
- new_e = URLGrabError(14, str(e))
- new_e.code = e.code
- new_e.exception = e
- raise new_e
- except IOError, e:
- if hasattr(e, 'reason') and have_socket_timeout and \
- isinstance(e.reason, TimeoutError):
- raise URLGrabError(12, _('Timeout: %s') % (e, ))
- else:
- raise URLGrabError(4, _('IOError: %s') % (e, ))
- except OSError, e:
- raise URLGrabError(5, _('OSError: %s') % (e, ))
- except HTTPException, e:
- raise URLGrabError(7, _('HTTP Exception (%s): %s') % \
- (e.__class__.__name__, e))
- else:
- return (fo, hdr)
-
- def _do_grab(self):
- """dump the file to self.filename."""
- if self.append: new_fo = open(self.filename, 'ab')
- else: new_fo = open(self.filename, 'wb')
- bs = 1024*8
- size = 0
-
- block = self.read(bs)
- size = size + len(block)
- while block:
- new_fo.write(block)
- block = self.read(bs)
- size = size + len(block)
-
- new_fo.close()
- try:
- modified_tuple = self.hdr.getdate_tz('last-modified')
- modified_stamp = rfc822.mktime_tz(modified_tuple)
- os.utime(self.filename, (modified_stamp, modified_stamp))
- except (TypeError,), e: pass
-
- return size
-
- def _fill_buffer(self, amt=None):
- """fill the buffer to contain at least 'amt' bytes by reading
- from the underlying file object. If amt is None, then it will
- read until it gets nothing more. It updates the progress meter
- and throttles after every self._rbufsize bytes."""
- # the _rbuf test is only in this first 'if' for speed. It's not
- # logically necessary
- if self._rbuf and not amt is None:
- L = len(self._rbuf)
- if amt > L:
- amt = amt - L
- else:
- return
-
- # if we've made it here, then we don't have enough in the buffer
- # and we need to read more.
-
- buf = [self._rbuf]
- bufsize = len(self._rbuf)
- while amt is None or amt:
- # first, delay if necessary for throttling reasons
- if self.opts.raw_throttle():
- diff = self._tsize/self.opts.raw_throttle() - \
- (time.time() - self._ttime)
- if diff > 0: time.sleep(diff)
- self._ttime = time.time()
-
- # now read some data, up to self._rbufsize
- if amt is None: readamount = self._rbufsize
- else: readamount = min(amt, self._rbufsize)
- try:
- new = self.fo.read(readamount)
- except socket.error, e:
- raise URLGrabError(4, _('Socket Error: %s') % (e, ))
- except TimeoutError, e:
- raise URLGrabError(12, _('Timeout: %s') % (e, ))
- except IOError, e:
- raise URLGrabError(4, _('IOError: %s') %(e,))
- newsize = len(new)
- if not newsize: break # no more to read
-
- if amt: amt = amt - newsize
- buf.append(new)
- bufsize = bufsize + newsize
- self._tsize = newsize
- self._amount_read = self._amount_read + newsize
- if self.opts.progress_obj:
- self.opts.progress_obj.update(self._amount_read)
-
- self._rbuf = string.join(buf, '')
- return
-
- def read(self, amt=None):
- self._fill_buffer(amt)
- if amt is None:
- s, self._rbuf = self._rbuf, ''
- else:
- s, self._rbuf = self._rbuf[:amt], self._rbuf[amt:]
- return s
-
- def readline(self, limit=-1):
- i = string.find(self._rbuf, '\n')
- while i < 0 and not (0 < limit <= len(self._rbuf)):
- L = len(self._rbuf)
- self._fill_buffer(L + self._rbufsize)
- if not len(self._rbuf) > L: break
- i = string.find(self._rbuf, '\n', L)
-
- if i < 0: i = len(self._rbuf)
- else: i = i+1
- if 0 <= limit < len(self._rbuf): i = limit
-
- s, self._rbuf = self._rbuf[:i], self._rbuf[i:]
- return s
-
- def close(self):
- if self.opts.progress_obj:
- self.opts.progress_obj.end(self._amount_read)
- self.fo.close()
- if self.opts.close_connection:
- try: self.fo.close_connection()
- except: pass
-
-_handler_cache = []
-def CachedOpenerDirector(ssl_factory = None, *handlers):
- for (cached_handlers, opener) in _handler_cache:
- if cached_handlers == handlers:
- for handler in opener.handlers:
- handler.add_parent(opener)
- return opener
- if not ssl_factory:
- ssl_factory = sslfactory.get_factory()
- opener = ssl_factory.create_opener(*handlers)
- _handler_cache.append( (handlers, opener) )
- return opener
-
-_proxy_cache = []
-def CachedProxyHandler(proxies):
- for (pdict, handler) in _proxy_cache:
- if pdict == proxies:
- if DEBUG: DEBUG.debug('re-using proxy settings: %s', proxies)
- break
- else:
- for k, v in proxies.items():
- utype, url = urllib.splittype(v)
- host, other = urllib.splithost(url)
- if (utype is None) or (host is None):
- raise URLGrabError(13, _('Bad proxy URL: %s') % v)
-
- if DEBUG: DEBUG.info('creating new proxy handler: %s', proxies)
- handler = urllib2.ProxyHandler(proxies)
- _proxy_cache.append( (proxies, handler) )
- return handler
-
-#####################################################################
-# DEPRECATED FUNCTIONS
-def set_throttle(new_throttle):
- """Deprecated. Use: default_grabber.throttle = new_throttle"""
- default_grabber.throttle = new_throttle
-
-def set_bandwidth(new_bandwidth):
- """Deprecated. Use: default_grabber.bandwidth = new_bandwidth"""
- default_grabber.bandwidth = new_bandwidth
-
-def set_progress_obj(new_progress_obj):
- """Deprecated. Use: default_grabber.progress_obj = new_progress_obj"""
- default_grabber.progress_obj = new_progress_obj
-
-def set_user_agent(new_user_agent):
- """Deprecated. Use: default_grabber.user_agent = new_user_agent"""
- default_grabber.user_agent = new_user_agent
-
-def retrygrab(url, filename=None, copy_local=0, close_connection=0,
- progress_obj=None, throttle=None, bandwidth=None,
- numtries=3, retrycodes=[-1,2,4,5,6,7], checkfunc=None):
- """Deprecated. Use: urlgrab() with the retry arg instead"""
- kwargs = {'copy_local' : copy_local,
- 'close_connection' : close_connection,
- 'progress_obj' : progress_obj,
- 'throttle' : throttle,
- 'bandwidth' : bandwidth,
- 'retry' : numtries,
- 'retrycodes' : retrycodes,
- 'checkfunc' : checkfunc
- }
- return urlgrab(url, filename, **kwargs)
-
-
-#####################################################################
-# TESTING
-def _main_test():
- import sys
- try: url, filename = sys.argv[1:3]
- except ValueError:
- print 'usage:', sys.argv[0], \
- '<url> <filename> [copy_local=0|1] [close_connection=0|1]'
- sys.exit()
-
- kwargs = {}
- for a in sys.argv[3:]:
- k, v = string.split(a, '=', 1)
- kwargs[k] = int(v)
-
- set_throttle(1.0)
- set_bandwidth(32 * 1024)
- print "throttle: %s, throttle bandwidth: %s B/s" % (default_grabber.throttle,
- default_grabber.bandwidth)
-
- try: from progress import text_progress_meter
- except ImportError, e: pass
- else: kwargs['progress_obj'] = text_progress_meter()
-
- try: name = apply(urlgrab, (url, filename), kwargs)
- except URLGrabError, e: print e
- else: print 'LOCAL FILE:', name
-
-
-def _retry_test():
- import sys
- try: url, filename = sys.argv[1:3]
- except ValueError:
- print 'usage:', sys.argv[0], \
- '<url> <filename> [copy_local=0|1] [close_connection=0|1]'
- sys.exit()
-
- kwargs = {}
- for a in sys.argv[3:]:
- k, v = string.split(a, '=', 1)
- kwargs[k] = int(v)
-
- try: from progress import text_progress_meter
- except ImportError, e: pass
- else: kwargs['progress_obj'] = text_progress_meter()
-
- def cfunc(filename, hello, there='foo'):
- print hello, there
- import random
- rnum = random.random()
- if rnum < .5:
- print 'forcing retry'
- raise URLGrabError(-1, 'forcing retry')
- if rnum < .75:
- print 'forcing failure'
- raise URLGrabError(-2, 'forcing immediate failure')
- print 'success'
- return
-
- kwargs['checkfunc'] = (cfunc, ('hello',), {'there':'there'})
- try: name = apply(retrygrab, (url, filename), kwargs)
- except URLGrabError, e: print e
- else: print 'LOCAL FILE:', name
-
-def _file_object_test(filename=None):
- import random, cStringIO, sys
- if filename is None:
- filename = __file__
- print 'using file "%s" for comparisons' % filename
- fo = open(filename)
- s_input = fo.read()
- fo.close()
-
- for testfunc in [_test_file_object_smallread,
- _test_file_object_readall,
- _test_file_object_readline,
- _test_file_object_readlines]:
- fo_input = cStringIO.StringIO(s_input)
- fo_output = cStringIO.StringIO()
- wrapper = URLGrabberFileObject(fo_input, None, 0)
- print 'testing %-30s ' % testfunc.__name__,
- testfunc(wrapper, fo_output)
- s_output = fo_output.getvalue()
- if s_output == s_input: print 'passed'
- else: print 'FAILED'
-
-def _test_file_object_smallread(wrapper, fo_output):
- while 1:
- s = wrapper.read(23)
- fo_output.write(s)
- if not s: return
-
-def _test_file_object_readall(wrapper, fo_output):
- s = wrapper.read()
- fo_output.write(s)
-
-def _test_file_object_readline(wrapper, fo_output):
- while 1:
- s = wrapper.readline()
- fo_output.write(s)
- if not s: return
-
-def _test_file_object_readlines(wrapper, fo_output):
- li = wrapper.readlines()
- fo_output.write(string.join(li, ''))
-
-if __name__ == '__main__':
- _main_test()
- _retry_test()
- _file_object_test('test')
+++ /dev/null
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330,
-# Boston, MA 02111-1307 USA
-
-# This file is part of urlgrabber, a high-level cross-protocol url-grabber
-# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
-
-"""An HTTP handler for urllib2 that supports HTTP 1.1 and keepalive.
-
->>> import urllib2
->>> from keepalive import HTTPHandler
->>> keepalive_handler = HTTPHandler()
->>> opener = urllib2.build_opener(keepalive_handler)
->>> urllib2.install_opener(opener)
->>>
->>> fo = urllib2.urlopen('http://www.python.org')
-
-If a connection to a given host is requested, and all of the existing
-connections are still in use, another connection will be opened. If
-the handler tries to use an existing connection but it fails in some
-way, it will be closed and removed from the pool.
-
-To remove the handler, simply re-run build_opener with no arguments, and
-install that opener.
-
-You can explicitly close connections by using the close_connection()
-method of the returned file-like object (described below) or you can
-use the handler methods:
-
- close_connection(host)
- close_all()
- open_connections()
-
-NOTE: using the close_connection and close_all methods of the handler
-should be done with care when using multiple threads.
- * there is nothing that prevents another thread from creating new
- connections immediately after connections are closed
- * no checks are done to prevent in-use connections from being closed
-
->>> keepalive_handler.close_all()
-
-EXTRA ATTRIBUTES AND METHODS
-
- Upon a status of 200, the object returned has a few additional
- attributes and methods, which should not be used if you want to
- remain consistent with the normal urllib2-returned objects:
-
- close_connection() - close the connection to the host
- readlines() - you know, readlines()
- status - the return status (ie 404)
- reason - english translation of status (ie 'File not found')
-
- If you want the best of both worlds, use this inside an
- AttributeError-catching try:
-
- >>> try: status = fo.status
- >>> except AttributeError: status = None
-
- Unfortunately, these are ONLY there if status == 200, so it's not
- easy to distinguish between non-200 responses. The reason is that
- urllib2 tries to do clever things with error codes 301, 302, 401,
- and 407, and it wraps the object upon return.
-
- For python versions earlier than 2.4, you can avoid this fancy error
- handling by setting the module-level global HANDLE_ERRORS to zero.
- You see, prior to 2.4, it's the HTTP Handler's job to determine what
- to handle specially, and what to just pass up. HANDLE_ERRORS == 0
- means "pass everything up". In python 2.4, however, this job no
- longer belongs to the HTTP Handler and is now done by a NEW handler,
- HTTPErrorProcessor. Here's the bottom line:
-
- python version < 2.4
- HANDLE_ERRORS == 1 (default) pass up 200, treat the rest as
- errors
- HANDLE_ERRORS == 0 pass everything up, error processing is
- left to the calling code
- python version >= 2.4
- HANDLE_ERRORS == 1 pass up 200, treat the rest as errors
- HANDLE_ERRORS == 0 (default) pass everything up, let the
- other handlers (specifically,
- HTTPErrorProcessor) decide what to do
-
- In practice, setting the variable either way makes little difference
- in python 2.4, so for the most consistent behavior across versions,
- you probably just want to use the defaults, which will give you
- exceptions on errors.
-
-"""
-
-# $Id: keepalive.py,v 1.16 2006/09/22 00:58:05 mstenner Exp $
-
-import urllib2
-import httplib
-import socket
-import thread
-
-DEBUG = None
-
-import sslfactory
-
-import sys
-if sys.version_info < (2, 4): HANDLE_ERRORS = 1
-else: HANDLE_ERRORS = 0
-
-class ConnectionManager:
- """
- The connection manager must be able to:
- * keep track of all existing
- """
- def __init__(self):
- self._lock = thread.allocate_lock()
- self._hostmap = {} # map hosts to a list of connections
- self._connmap = {} # map connections to host
- self._readymap = {} # map connection to ready state
-
- def add(self, host, connection, ready):
- self._lock.acquire()
- try:
- if not self._hostmap.has_key(host): self._hostmap[host] = []
- self._hostmap[host].append(connection)
- self._connmap[connection] = host
- self._readymap[connection] = ready
- finally:
- self._lock.release()
-
- def remove(self, connection):
- self._lock.acquire()
- try:
- try:
- host = self._connmap[connection]
- except KeyError:
- pass
- else:
- del self._connmap[connection]
- del self._readymap[connection]
- self._hostmap[host].remove(connection)
- if not self._hostmap[host]: del self._hostmap[host]
- finally:
- self._lock.release()
-
- def set_ready(self, connection, ready):
- try: self._readymap[connection] = ready
- except KeyError: pass
-
- def get_ready_conn(self, host):
- conn = None
- self._lock.acquire()
- try:
- if self._hostmap.has_key(host):
- for c in self._hostmap[host]:
- if self._readymap[c]:
- self._readymap[c] = 0
- conn = c
- break
- finally:
- self._lock.release()
- return conn
-
- def get_all(self, host=None):
- if host:
- return list(self._hostmap.get(host, []))
- else:
- return dict(self._hostmap)
-
-class KeepAliveHandler:
- def __init__(self):
- self._cm = ConnectionManager()
-
- #### Connection Management
- def open_connections(self):
- """return a list of connected hosts and the number of connections
- to each. [('foo.com:80', 2), ('bar.org', 1)]"""
- return [(host, len(li)) for (host, li) in self._cm.get_all().items()]
-
- def close_connection(self, host):
- """close connection(s) to <host>
- host is the host:port spec, as in 'www.cnn.com:8080' as passed in.
- no error occurs if there is no connection to that host."""
- for h in self._cm.get_all(host):
- self._cm.remove(h)
- h.close()
-
- def close_all(self):
- """close all open connections"""
- for host, conns in self._cm.get_all().items():
- for h in conns:
- self._cm.remove(h)
- h.close()
-
- def _request_closed(self, request, host, connection):
- """tells us that this request is now closed and the the
- connection is ready for another request"""
- self._cm.set_ready(connection, 1)
-
- def _remove_connection(self, host, connection, close=0):
- if close: connection.close()
- self._cm.remove(connection)
-
- #### Transaction Execution
- def do_open(self, req):
- host = req.get_host()
- if not host:
- raise urllib2.URLError('no host given')
-
- try:
- h = self._cm.get_ready_conn(host)
- while h:
- r = self._reuse_connection(h, req, host)
-
- # if this response is non-None, then it worked and we're
- # done. Break out, skipping the else block.
- if r: break
-
- # connection is bad - possibly closed by server
- # discard it and ask for the next free connection
- h.close()
- self._cm.remove(h)
- h = self._cm.get_ready_conn(host)
- else:
- # no (working) free connections were found. Create a new one.
- h = self._get_connection(host)
- if DEBUG: DEBUG.info("creating new connection to %s (%d)",
- host, id(h))
- self._cm.add(host, h, 0)
- self._start_transaction(h, req)
- r = h.getresponse()
- except (socket.error, httplib.HTTPException), err:
- raise urllib2.URLError(err)
-
- # if not a persistent connection, don't try to reuse it
- if r.will_close: self._cm.remove(h)
-
- if DEBUG: DEBUG.info("STATUS: %s, %s", r.status, r.reason)
- r._handler = self
- r._host = host
- r._url = req.get_full_url()
- r._connection = h
- r.code = r.status
- r.headers = r.msg
- r.msg = r.reason
-
- if r.status == 200 or not HANDLE_ERRORS:
- return r
- else:
- return self.parent.error('http', req, r,
- r.status, r.msg, r.headers)
-
- def _reuse_connection(self, h, req, host):
- """start the transaction with a re-used connection
- return a response object (r) upon success or None on failure.
- This DOES not close or remove bad connections in cases where
- it returns. However, if an unexpected exception occurs, it
- will close and remove the connection before re-raising.
- """
- try:
- self._start_transaction(h, req)
- r = h.getresponse()
- # note: just because we got something back doesn't mean it
- # worked. We'll check the version below, too.
- except (socket.error, httplib.HTTPException):
- r = None
- except:
- # adding this block just in case we've missed
- # something we will still raise the exception, but
- # lets try and close the connection and remove it
- # first. We previously got into a nasty loop
- # where an exception was uncaught, and so the
- # connection stayed open. On the next try, the
- # same exception was raised, etc. The tradeoff is
- # that it's now possible this call will raise
- # a DIFFERENT exception
- if DEBUG: DEBUG.error("unexpected exception - closing " + \
- "connection to %s (%d)", host, id(h))
- self._cm.remove(h)
- h.close()
- raise
-
- if r is None or r.version == 9:
- # httplib falls back to assuming HTTP 0.9 if it gets a
- # bad header back. This is most likely to happen if
- # the socket has been closed by the server since we
- # last used the connection.
- if DEBUG: DEBUG.info("failed to re-use connection to %s (%d)",
- host, id(h))
- r = None
- else:
- if DEBUG: DEBUG.info("re-using connection to %s (%d)", host, id(h))
-
- return r
-
- def _start_transaction(self, h, req):
- try:
- if req.has_data():
- data = req.get_data()
- h.putrequest('POST', req.get_selector())
- if not req.headers.has_key('Content-type'):
- h.putheader('Content-type',
- 'application/x-www-form-urlencoded')
- if not req.headers.has_key('Content-length'):
- h.putheader('Content-length', '%d' % len(data))
- else:
- h.putrequest('GET', req.get_selector())
- except (socket.error, httplib.HTTPException), err:
- raise urllib2.URLError(err)
-
- for args in self.parent.addheaders:
- h.putheader(*args)
- for k, v in req.headers.items():
- h.putheader(k, v)
- h.endheaders()
- if req.has_data():
- h.send(data)
-
- def _get_connection(self, host):
- return NotImplementedError
-
-class HTTPHandler(KeepAliveHandler, urllib2.HTTPHandler):
- def __init__(self):
- KeepAliveHandler.__init__(self)
-
- def http_open(self, req):
- return self.do_open(req)
-
- def _get_connection(self, host):
- return HTTPConnection(host)
-
-class HTTPSHandler(KeepAliveHandler, urllib2.HTTPSHandler):
- def __init__(self, ssl_factory=None):
- KeepAliveHandler.__init__(self)
- if not ssl_factory:
- ssl_factory = sslfactory.get_factory()
- self._ssl_factory = ssl_factory
-
- def https_open(self, req):
- return self.do_open(req)
-
- def _get_connection(self, host):
- return self._ssl_factory.get_https_connection(host)
-
-class HTTPResponse(httplib.HTTPResponse):
- # we need to subclass HTTPResponse in order to
- # 1) add readline() and readlines() methods
- # 2) add close_connection() methods
- # 3) add info() and geturl() methods
-
- # in order to add readline(), read must be modified to deal with a
- # buffer. example: readline must read a buffer and then spit back
- # one line at a time. The only real alternative is to read one
- # BYTE at a time (ick). Once something has been read, it can't be
- # put back (ok, maybe it can, but that's even uglier than this),
- # so if you THEN do a normal read, you must first take stuff from
- # the buffer.
-
- # the read method wraps the original to accomodate buffering,
- # although read() never adds to the buffer.
- # Both readline and readlines have been stolen with almost no
- # modification from socket.py
-
-
- def __init__(self, sock, debuglevel=0, strict=0, method=None):
- if method: # the httplib in python 2.3 uses the method arg
- httplib.HTTPResponse.__init__(self, sock, debuglevel, method)
- else: # 2.2 doesn't
- httplib.HTTPResponse.__init__(self, sock, debuglevel)
- self.fileno = sock.fileno
- self.code = None
- self._rbuf = ''
- self._rbufsize = 8096
- self._handler = None # inserted by the handler later
- self._host = None # (same)
- self._url = None # (same)
- self._connection = None # (same)
-
- _raw_read = httplib.HTTPResponse.read
-
- def close(self):
- if self.fp:
- self.fp.close()
- self.fp = None
- if self._handler:
- self._handler._request_closed(self, self._host,
- self._connection)
-
- def close_connection(self):
- self._handler._remove_connection(self._host, self._connection, close=1)
- self.close()
-
- def info(self):
- return self.headers
-
- def geturl(self):
- return self._url
-
- def read(self, amt=None):
- # the _rbuf test is only in this first if for speed. It's not
- # logically necessary
- if self._rbuf and not amt is None:
- L = len(self._rbuf)
- if amt > L:
- amt -= L
- else:
- s = self._rbuf[:amt]
- self._rbuf = self._rbuf[amt:]
- return s
-
- s = self._rbuf + self._raw_read(amt)
- self._rbuf = ''
- return s
-
- def readline(self, limit=-1):
- data = ""
- i = self._rbuf.find('\n')
- while i < 0 and not (0 < limit <= len(self._rbuf)):
- new = self._raw_read(self._rbufsize)
- if not new: break
- i = new.find('\n')
- if i >= 0: i = i + len(self._rbuf)
- self._rbuf = self._rbuf + new
- if i < 0: i = len(self._rbuf)
- else: i = i+1
- if 0 <= limit < len(self._rbuf): i = limit
- data, self._rbuf = self._rbuf[:i], self._rbuf[i:]
- return data
-
- def readlines(self, sizehint = 0):
- total = 0
- list = []
- while 1:
- line = self.readline()
- if not line: break
- list.append(line)
- total += len(line)
- if sizehint and total >= sizehint:
- break
- return list
-
-
-class HTTPConnection(httplib.HTTPConnection):
- # use the modified response class
- response_class = HTTPResponse
-
-class HTTPSConnection(httplib.HTTPSConnection):
- response_class = HTTPResponse
-
-#########################################################################
-##### TEST FUNCTIONS
-#########################################################################
-
-def error_handler(url):
- global HANDLE_ERRORS
- orig = HANDLE_ERRORS
- keepalive_handler = HTTPHandler()
- opener = urllib2.build_opener(keepalive_handler)
- urllib2.install_opener(opener)
- pos = {0: 'off', 1: 'on'}
- for i in (0, 1):
- print " fancy error handling %s (HANDLE_ERRORS = %i)" % (pos[i], i)
- HANDLE_ERRORS = i
- try:
- fo = urllib2.urlopen(url)
- foo = fo.read()
- fo.close()
- try: status, reason = fo.status, fo.reason
- except AttributeError: status, reason = None, None
- except IOError, e:
- print " EXCEPTION: %s" % e
- raise
- else:
- print " status = %s, reason = %s" % (status, reason)
- HANDLE_ERRORS = orig
- hosts = keepalive_handler.open_connections()
- print "open connections:", hosts
- keepalive_handler.close_all()
-
-def continuity(url):
- import md5
- format = '%25s: %s'
-
- # first fetch the file with the normal http handler
- opener = urllib2.build_opener()
- urllib2.install_opener(opener)
- fo = urllib2.urlopen(url)
- foo = fo.read()
- fo.close()
- m = md5.new(foo)
- print format % ('normal urllib', m.hexdigest())
-
- # now install the keepalive handler and try again
- opener = urllib2.build_opener(HTTPHandler())
- urllib2.install_opener(opener)
-
- fo = urllib2.urlopen(url)
- foo = fo.read()
- fo.close()
- m = md5.new(foo)
- print format % ('keepalive read', m.hexdigest())
-
- fo = urllib2.urlopen(url)
- foo = ''
- while 1:
- f = fo.readline()
- if f: foo = foo + f
- else: break
- fo.close()
- m = md5.new(foo)
- print format % ('keepalive readline', m.hexdigest())
-
-def comp(N, url):
- print ' making %i connections to:\n %s' % (N, url)
-
- sys.stdout.write(' first using the normal urllib handlers')
- # first use normal opener
- opener = urllib2.build_opener()
- urllib2.install_opener(opener)
- t1 = fetch(N, url)
- print ' TIME: %.3f s' % t1
-
- sys.stdout.write(' now using the keepalive handler ')
- # now install the keepalive handler and try again
- opener = urllib2.build_opener(HTTPHandler())
- urllib2.install_opener(opener)
- t2 = fetch(N, url)
- print ' TIME: %.3f s' % t2
- print ' improvement factor: %.2f' % (t1/t2, )
-
-def fetch(N, url, delay=0):
- import time
- lens = []
- starttime = time.time()
- for i in range(N):
- if delay and i > 0: time.sleep(delay)
- fo = urllib2.urlopen(url)
- foo = fo.read()
- fo.close()
- lens.append(len(foo))
- diff = time.time() - starttime
-
- j = 0
- for i in lens[1:]:
- j = j + 1
- if not i == lens[0]:
- print "WARNING: inconsistent length on read %i: %i" % (j, i)
-
- return diff
-
-def test_timeout(url):
- global DEBUG
- dbbackup = DEBUG
- class FakeLogger:
- def debug(self, msg, *args): print msg % args
- info = warning = error = debug
- DEBUG = FakeLogger()
- print " fetching the file to establish a connection"
- fo = urllib2.urlopen(url)
- data1 = fo.read()
- fo.close()
-
- i = 20
- print " waiting %i seconds for the server to close the connection" % i
- while i > 0:
- sys.stdout.write('\r %2i' % i)
- sys.stdout.flush()
- time.sleep(1)
- i -= 1
- sys.stderr.write('\r')
-
- print " fetching the file a second time"
- fo = urllib2.urlopen(url)
- data2 = fo.read()
- fo.close()
-
- if data1 == data2:
- print ' data are identical'
- else:
- print ' ERROR: DATA DIFFER'
-
- DEBUG = dbbackup
-
-
-def test(url, N=10):
- print "checking error hander (do this on a non-200)"
- try: error_handler(url)
- except IOError, e:
- print "exiting - exception will prevent further tests"
- sys.exit()
- print
- print "performing continuity test (making sure stuff isn't corrupted)"
- continuity(url)
- print
- print "performing speed comparison"
- comp(N, url)
- print
- print "performing dropped-connection check"
- test_timeout(url)
-
-if __name__ == '__main__':
- import time
- import sys
- try:
- N = int(sys.argv[1])
- url = sys.argv[2]
- except:
- print "%s <integer> <url>" % sys.argv[0]
- else:
- test(url, N)
+++ /dev/null
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330,
-# Boston, MA 02111-1307 USA
-
-# This file is part of urlgrabber, a high-level cross-protocol url-grabber
-# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
-
-"""Module for downloading files from a pool of mirrors
-
-DESCRIPTION
-
- This module provides support for downloading files from a pool of
- mirrors with configurable failover policies. To a large extent, the
- failover policy is chosen by using different classes derived from
- the main class, MirrorGroup.
-
- Instances of MirrorGroup (and cousins) act very much like URLGrabber
- instances in that they have urlread, urlgrab, and urlopen methods.
- They can therefore, be used in very similar ways.
-
- from urlgrabber.grabber import URLGrabber
- from urlgrabber.mirror import MirrorGroup
- gr = URLGrabber()
- mg = MirrorGroup(gr, ['http://foo.com/some/directory/',
- 'http://bar.org/maybe/somewhere/else/',
- 'ftp://baz.net/some/other/place/entirely/']
- mg.urlgrab('relative/path.zip')
-
- The assumption is that all mirrors are identical AFTER the base urls
- specified, so that any mirror can be used to fetch any file.
-
-FAILOVER
-
- The failover mechanism is designed to be customized by subclassing
- from MirrorGroup to change the details of the behavior. In general,
- the classes maintain a master mirror list and a "current mirror"
- index. When a download is initiated, a copy of this list and index
- is created for that download only. The specific failover policy
- depends on the class used, and so is documented in the class
- documentation. Note that ANY behavior of the class can be
- overridden, so any failover policy at all is possible (although
- you may need to change the interface in extreme cases).
-
-CUSTOMIZATION
-
- Most customization of a MirrorGroup object is done at instantiation
- time (or via subclassing). There are four major types of
- customization:
-
- 1) Pass in a custom urlgrabber - The passed in urlgrabber will be
- used (by default... see #2) for the grabs, so options to it
- apply for the url-fetching
-
- 2) Custom mirror list - Mirror lists can simply be a list of
- stings mirrors (as shown in the example above) but each can
- also be a dict, allowing for more options. For example, the
- first mirror in the list above could also have been:
-
- {'mirror': 'http://foo.com/some/directory/',
- 'grabber': <a custom grabber to be used for this mirror>,
- 'kwargs': { <a dict of arguments passed to the grabber> }}
-
- All mirrors are converted to this format internally. If
- 'grabber' is omitted, the default grabber will be used. If
- kwargs are omitted, then (duh) they will not be used.
-
- 3) Pass keyword arguments when instantiating the mirror group.
- See, for example, the failure_callback argument.
-
- 4) Finally, any kwargs passed in for the specific file (to the
- urlgrab method, for example) will be folded in. The options
- passed into the grabber's urlXXX methods will override any
- options specified in a custom mirror dict.
-
-"""
-
-# $Id: mirror.py,v 1.14 2006/02/22 18:26:46 mstenner Exp $
-
-import random
-import thread # needed for locking to make this threadsafe
-
-from grabber import URLGrabError, CallbackObject, DEBUG
-
-try:
- from i18n import _
-except ImportError, msg:
- def _(st): return st
-
-class GrabRequest:
- """This is a dummy class used to hold information about the specific
- request. For example, a single file. By maintaining this information
- separately, we can accomplish two things:
-
- 1) make it a little easier to be threadsafe
- 2) have request-specific parameters
- """
- pass
-
-class MirrorGroup:
- """Base Mirror class
-
- Instances of this class are built with a grabber object and a list
- of mirrors. Then all calls to urlXXX should be passed relative urls.
- The requested file will be searched for on the first mirror. If the
- grabber raises an exception (possibly after some retries) then that
- mirror will be removed from the list, and the next will be attempted.
- If all mirrors are exhausted, then an exception will be raised.
-
- MirrorGroup has the following failover policy:
-
- * downloads begin with the first mirror
-
- * by default (see default_action below) a failure (after retries)
- causes it to increment the local AND master indices. Also,
- the current mirror is removed from the local list (but NOT the
- master list - the mirror can potentially be used for other
- files)
-
- * if the local list is ever exhausted, a URLGrabError will be
- raised (errno=256, no more mirrors)
-
- OPTIONS
-
- In addition to the required arguments "grabber" and "mirrors",
- MirrorGroup also takes the following optional arguments:
-
- default_action
-
- A dict that describes the actions to be taken upon failure
- (after retries). default_action can contain any of the
- following keys (shown here with their default values):
-
- default_action = {'increment': 1,
- 'increment_master': 1,
- 'remove': 1,
- 'remove_master': 0,
- 'fail': 0}
-
- In this context, 'increment' means "use the next mirror" and
- 'remove' means "never use this mirror again". The two
- 'master' values refer to the instance-level mirror list (used
- for all files), whereas the non-master values refer to the
- current download only.
-
- The 'fail' option will cause immediate failure by re-raising
- the exception and no further attempts to get the current
- download.
-
- This dict can be set at instantiation time,
- mg = MirrorGroup(grabber, mirrors, default_action={'fail':1})
- at method-execution time (only applies to current fetch),
- filename = mg.urlgrab(url, default_action={'increment': 0})
- or by returning an action dict from the failure_callback
- return {'fail':0}
- in increasing precedence.
-
- If all three of these were done, the net result would be:
- {'increment': 0, # set in method
- 'increment_master': 1, # class default
- 'remove': 1, # class default
- 'remove_master': 0, # class default
- 'fail': 0} # set at instantiation, reset
- # from callback
-
- failure_callback
-
- this is a callback that will be called when a mirror "fails",
- meaning the grabber raises some URLGrabError. If this is a
- tuple, it is interpreted to be of the form (cb, args, kwargs)
- where cb is the actual callable object (function, method,
- etc). Otherwise, it is assumed to be the callable object
- itself. The callback will be passed a grabber.CallbackObject
- instance along with args and kwargs (if present). The following
- attributes are defined withing the instance:
-
- obj.exception = < exception that was raised >
- obj.mirror = < the mirror that was tried >
- obj.relative_url = < url relative to the mirror >
- obj.url = < full url that failed >
- # .url is just the combination of .mirror
- # and .relative_url
-
- The failure callback can return an action dict, as described
- above.
-
- Like default_action, the failure_callback can be set at
- instantiation time or when the urlXXX method is called. In
- the latter case, it applies only for that fetch.
-
- The callback can re-raise the exception quite easily. For
- example, this is a perfectly adequate callback function:
-
- def callback(obj): raise obj.exception
-
- WARNING: do not save the exception object (or the
- CallbackObject instance). As they contain stack frame
- references, they can lead to circular references.
-
- Notes:
- * The behavior can be customized by deriving and overriding the
- 'CONFIGURATION METHODS'
- * The 'grabber' instance is kept as a reference, not copied.
- Therefore, the grabber instance can be modified externally
- and changes will take effect immediately.
- """
-
- # notes on thread-safety:
-
- # A GrabRequest should never be shared by multiple threads because
- # it's never saved inside the MG object and never returned outside it.
- # therefore, it should be safe to access/modify grabrequest data
- # without a lock. However, accessing the mirrors and _next attributes
- # of the MG itself must be done when locked to prevent (for example)
- # removal of the wrong mirror.
-
- ##############################################################
- # CONFIGURATION METHODS - intended to be overridden to
- # customize behavior
- def __init__(self, grabber, mirrors, **kwargs):
- """Initialize the MirrorGroup object.
-
- REQUIRED ARGUMENTS
-
- grabber - URLGrabber instance
- mirrors - a list of mirrors
-
- OPTIONAL ARGUMENTS
-
- failure_callback - callback to be used when a mirror fails
- default_action - dict of failure actions
-
- See the module-level and class level documentation for more
- details.
- """
-
- # OVERRIDE IDEAS:
- # shuffle the list to randomize order
- self.grabber = grabber
- self.mirrors = self._parse_mirrors(mirrors)
- self._next = 0
- self._lock = thread.allocate_lock()
- self.default_action = None
- self._process_kwargs(kwargs)
-
- # if these values are found in **kwargs passed to one of the urlXXX
- # methods, they will be stripped before getting passed on to the
- # grabber
- options = ['default_action', 'failure_callback']
-
- def _process_kwargs(self, kwargs):
- self.failure_callback = kwargs.get('failure_callback')
- self.default_action = kwargs.get('default_action')
-
- def _parse_mirrors(self, mirrors):
- parsed_mirrors = []
- for m in mirrors:
- if type(m) == type(''): m = {'mirror': m}
- parsed_mirrors.append(m)
- return parsed_mirrors
-
- def _load_gr(self, gr):
- # OVERRIDE IDEAS:
- # shuffle gr list
- self._lock.acquire()
- gr.mirrors = list(self.mirrors)
- gr._next = self._next
- self._lock.release()
-
- def _get_mirror(self, gr):
- # OVERRIDE IDEAS:
- # return a random mirror so that multiple mirrors get used
- # even without failures.
- if not gr.mirrors:
- raise URLGrabError(256, _('No more mirrors to try.'))
- return gr.mirrors[gr._next]
-
- def _failure(self, gr, cb_obj):
- # OVERRIDE IDEAS:
- # inspect the error - remove=1 for 404, remove=2 for connection
- # refused, etc. (this can also be done via
- # the callback)
- cb = gr.kw.get('failure_callback') or self.failure_callback
- if cb:
- if type(cb) == type( () ):
- cb, args, kwargs = cb
- else:
- args, kwargs = (), {}
- action = cb(cb_obj, *args, **kwargs) or {}
- else:
- action = {}
- # XXXX - decide - there are two ways to do this
- # the first is action-overriding as a whole - use the entire action
- # or fall back on module level defaults
- #action = action or gr.kw.get('default_action') or self.default_action
- # the other is to fall through for each element in the action dict
- a = dict(self.default_action or {})
- a.update(gr.kw.get('default_action', {}))
- a.update(action)
- action = a
- self.increment_mirror(gr, action)
- if action and action.get('fail', 0): raise
-
- def increment_mirror(self, gr, action={}):
- """Tell the mirror object increment the mirror index
-
- This increments the mirror index, which amounts to telling the
- mirror object to use a different mirror (for this and future
- downloads).
-
- This is a SEMI-public method. It will be called internally,
- and you may never need to call it. However, it is provided
- (and is made public) so that the calling program can increment
- the mirror choice for methods like urlopen. For example, with
- urlopen, there's no good way for the mirror group to know that
- an error occurs mid-download (it's already returned and given
- you the file object).
-
- remove --- can have several values
- 0 do not remove the mirror from the list
- 1 remove the mirror for this download only
- 2 remove the mirror permanently
-
- beware of remove=0 as it can lead to infinite loops
- """
- badmirror = gr.mirrors[gr._next]
-
- self._lock.acquire()
- try:
- ind = self.mirrors.index(badmirror)
- except ValueError:
- pass
- else:
- if action.get('remove_master', 0):
- del self.mirrors[ind]
- elif self._next == ind and action.get('increment_master', 1):
- self._next += 1
- if self._next >= len(self.mirrors): self._next = 0
- self._lock.release()
-
- if action.get('remove', 1):
- del gr.mirrors[gr._next]
- elif action.get('increment', 1):
- gr._next += 1
- if gr._next >= len(gr.mirrors): gr._next = 0
-
- if DEBUG:
- grm = [m['mirror'] for m in gr.mirrors]
- DEBUG.info('GR mirrors: [%s] %i', ' '.join(grm), gr._next)
- selfm = [m['mirror'] for m in self.mirrors]
- DEBUG.info('MAIN mirrors: [%s] %i', ' '.join(selfm), self._next)
-
- #####################################################################
- # NON-CONFIGURATION METHODS
- # these methods are designed to be largely workhorse methods that
- # are not intended to be overridden. That doesn't mean you can't;
- # if you want to, feel free, but most things can be done by
- # by overriding the configuration methods :)
-
- def _join_url(self, base_url, rel_url):
- if base_url.endswith('/') or rel_url.startswith('/'):
- return base_url + rel_url
- else:
- return base_url + '/' + rel_url
-
- def _mirror_try(self, func, url, kw):
- gr = GrabRequest()
- gr.func = func
- gr.url = url
- gr.kw = dict(kw)
- self._load_gr(gr)
-
- for k in self.options:
- try: del kw[k]
- except KeyError: pass
-
- while 1:
- mirrorchoice = self._get_mirror(gr)
- fullurl = self._join_url(mirrorchoice['mirror'], gr.url)
- kwargs = dict(mirrorchoice.get('kwargs', {}))
- kwargs.update(kw)
- grabber = mirrorchoice.get('grabber') or self.grabber
- func_ref = getattr(grabber, func)
- if DEBUG: DEBUG.info('MIRROR: trying %s -> %s', url, fullurl)
- try:
- return func_ref( *(fullurl,), **kwargs )
- except URLGrabError, e:
- if DEBUG: DEBUG.info('MIRROR: failed')
- obj = CallbackObject()
- obj.exception = e
- obj.mirror = mirrorchoice['mirror']
- obj.relative_url = gr.url
- obj.url = fullurl
- self._failure(gr, obj)
-
- def urlgrab(self, url, filename=None, **kwargs):
- kw = dict(kwargs)
- kw['filename'] = filename
- func = 'urlgrab'
- return self._mirror_try(func, url, kw)
-
- def urlopen(self, url, **kwargs):
- kw = dict(kwargs)
- func = 'urlopen'
- return self._mirror_try(func, url, kw)
-
- def urlread(self, url, limit=None, **kwargs):
- kw = dict(kwargs)
- kw['limit'] = limit
- func = 'urlread'
- return self._mirror_try(func, url, kw)
-
-
-class MGRandomStart(MirrorGroup):
- """A mirror group that starts at a random mirror in the list.
-
- This behavior of this class is identical to MirrorGroup, except that
- it starts at a random location in the mirror list.
- """
-
- def __init__(self, grabber, mirrors, **kwargs):
- """Initialize the object
-
- The arguments for intialization are the same as for MirrorGroup
- """
- MirrorGroup.__init__(self, grabber, mirrors, **kwargs)
- self._next = random.randrange(len(mirrors))
-
-class MGRandomOrder(MirrorGroup):
- """A mirror group that uses mirrors in a random order.
-
- This behavior of this class is identical to MirrorGroup, except that
- it uses the mirrors in a random order. Note that the order is set at
- initialization time and fixed thereafter. That is, it does not pick a
- random mirror after each failure.
- """
-
- def __init__(self, grabber, mirrors, **kwargs):
- """Initialize the object
-
- The arguments for intialization are the same as for MirrorGroup
- """
- MirrorGroup.__init__(self, grabber, mirrors, **kwargs)
- random.shuffle(self.mirrors)
-
-if __name__ == '__main__':
- pass
+++ /dev/null
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330,
-# Boston, MA 02111-1307 USA
-
-# This file is part of urlgrabber, a high-level cross-protocol url-grabber
-# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
-
-# $Id: progress.py,v 1.7 2005/08/19 21:59:07 mstenner Exp $
-
-import sys
-import time
-import math
-import thread
-
-class BaseMeter:
- def __init__(self):
- self.update_period = 0.3 # seconds
-
- self.filename = None
- self.url = None
- self.basename = None
- self.text = None
- self.size = None
- self.start_time = None
- self.last_amount_read = 0
- self.last_update_time = None
- self.re = RateEstimator()
-
- def start(self, filename=None, url=None, basename=None,
- size=None, now=None, text=None):
- self.filename = filename
- self.url = url
- self.basename = basename
- self.text = text
-
- #size = None ######### TESTING
- self.size = size
- if not size is None: self.fsize = format_number(size) + 'B'
-
- if now is None: now = time.time()
- self.start_time = now
- self.re.start(size, now)
- self.last_amount_read = 0
- self.last_update_time = now
- self._do_start(now)
-
- def _do_start(self, now=None):
- pass
-
- def update(self, amount_read, now=None):
- # for a real gui, you probably want to override and put a call
- # to your mainloop iteration function here
- if now is None: now = time.time()
- if (now >= self.last_update_time + self.update_period) or \
- not self.last_update_time:
- self.re.update(amount_read, now)
- self.last_amount_read = amount_read
- self.last_update_time = now
- self._do_update(amount_read, now)
-
- def _do_update(self, amount_read, now=None):
- pass
-
- def end(self, amount_read, now=None):
- if now is None: now = time.time()
- self.re.update(amount_read, now)
- self.last_amount_read = amount_read
- self.last_update_time = now
- self._do_end(amount_read, now)
-
- def _do_end(self, amount_read, now=None):
- pass
-
-class TextMeter(BaseMeter):
- def __init__(self, fo=sys.stderr):
- BaseMeter.__init__(self)
- self.fo = fo
-
- def _do_update(self, amount_read, now=None):
- etime = self.re.elapsed_time()
- fetime = format_time(etime)
- fread = format_number(amount_read)
- #self.size = None
- if self.text is not None:
- text = self.text
- else:
- text = self.basename
- if self.size is None:
- out = '\r%-60.60s %5sB %s ' % \
- (text, fread, fetime)
- else:
- rtime = self.re.remaining_time()
- frtime = format_time(rtime)
- frac = self.re.fraction_read()
- bar = '='*int(25 * frac)
-
- out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s ETA ' % \
- (text, frac*100, bar, fread, frtime)
-
- self.fo.write(out)
- self.fo.flush()
-
- def _do_end(self, amount_read, now=None):
- total_time = format_time(self.re.elapsed_time())
- total_size = format_number(amount_read)
- if self.text is not None:
- text = self.text
- else:
- text = self.basename
- if self.size is None:
- out = '\r%-60.60s %5sB %s ' % \
- (text, total_size, total_time)
- else:
- bar = '='*25
- out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s ' % \
- (text, 100, bar, total_size, total_time)
- self.fo.write(out + '\n')
- self.fo.flush()
-
-text_progress_meter = TextMeter
-
-class MultiFileHelper(BaseMeter):
- def __init__(self, master):
- BaseMeter.__init__(self)
- self.master = master
-
- def _do_start(self, now):
- self.master.start_meter(self, now)
-
- def _do_update(self, amount_read, now):
- # elapsed time since last update
- self.master.update_meter(self, now)
-
- def _do_end(self, amount_read, now):
- self.ftotal_time = format_time(now - self.start_time)
- self.ftotal_size = format_number(self.last_amount_read)
- self.master.end_meter(self, now)
-
- def failure(self, message, now=None):
- self.master.failure_meter(self, message, now)
-
- def message(self, message):
- self.master.message_meter(self, message)
-
-class MultiFileMeter:
- helperclass = MultiFileHelper
- def __init__(self):
- self.meters = []
- self.in_progress_meters = []
- self._lock = thread.allocate_lock()
- self.update_period = 0.3 # seconds
-
- self.numfiles = None
- self.finished_files = 0
- self.failed_files = 0
- self.open_files = 0
- self.total_size = None
- self.failed_size = 0
- self.start_time = None
- self.finished_file_size = 0
- self.last_update_time = None
- self.re = RateEstimator()
-
- def start(self, numfiles=None, total_size=None, now=None):
- if now is None: now = time.time()
- self.numfiles = numfiles
- self.finished_files = 0
- self.failed_files = 0
- self.open_files = 0
- self.total_size = total_size
- self.failed_size = 0
- self.start_time = now
- self.finished_file_size = 0
- self.last_update_time = now
- self.re.start(total_size, now)
- self._do_start(now)
-
- def _do_start(self, now):
- pass
-
- def end(self, now=None):
- if now is None: now = time.time()
- self._do_end(now)
-
- def _do_end(self, now):
- pass
-
- def lock(self): self._lock.acquire()
- def unlock(self): self._lock.release()
-
- ###########################################################
- # child meter creation and destruction
- def newMeter(self):
- newmeter = self.helperclass(self)
- self.meters.append(newmeter)
- return newmeter
-
- def removeMeter(self, meter):
- self.meters.remove(meter)
-
- ###########################################################
- # child functions - these should only be called by helpers
- def start_meter(self, meter, now):
- if not meter in self.meters:
- raise ValueError('attempt to use orphaned meter')
- self._lock.acquire()
- try:
- if not meter in self.in_progress_meters:
- self.in_progress_meters.append(meter)
- self.open_files += 1
- finally:
- self._lock.release()
- self._do_start_meter(meter, now)
-
- def _do_start_meter(self, meter, now):
- pass
-
- def update_meter(self, meter, now):
- if not meter in self.meters:
- raise ValueError('attempt to use orphaned meter')
- if (now >= self.last_update_time + self.update_period) or \
- not self.last_update_time:
- self.re.update(self._amount_read(), now)
- self.last_update_time = now
- self._do_update_meter(meter, now)
-
- def _do_update_meter(self, meter, now):
- pass
-
- def end_meter(self, meter, now):
- if not meter in self.meters:
- raise ValueError('attempt to use orphaned meter')
- self._lock.acquire()
- try:
- try: self.in_progress_meters.remove(meter)
- except ValueError: pass
- self.open_files -= 1
- self.finished_files += 1
- self.finished_file_size += meter.last_amount_read
- finally:
- self._lock.release()
- self._do_end_meter(meter, now)
-
- def _do_end_meter(self, meter, now):
- pass
-
- def failure_meter(self, meter, message, now):
- if not meter in self.meters:
- raise ValueError('attempt to use orphaned meter')
- self._lock.acquire()
- try:
- try: self.in_progress_meters.remove(meter)
- except ValueError: pass
- self.open_files -= 1
- self.failed_files += 1
- if meter.size and self.failed_size is not None:
- self.failed_size += meter.size
- else:
- self.failed_size = None
- finally:
- self._lock.release()
- self._do_failure_meter(meter, message, now)
-
- def _do_failure_meter(self, meter, message, now):
- pass
-
- def message_meter(self, meter, message):
- pass
-
- ########################################################
- # internal functions
- def _amount_read(self):
- tot = self.finished_file_size
- for m in self.in_progress_meters:
- tot += m.last_amount_read
- return tot
-
-
-class TextMultiFileMeter(MultiFileMeter):
- def __init__(self, fo=sys.stderr):
- self.fo = fo
- MultiFileMeter.__init__(self)
-
- # files: ###/### ###% data: ######/###### ###% time: ##:##:##/##:##:##
- def _do_update_meter(self, meter, now):
- self._lock.acquire()
- try:
- format = "files: %3i/%-3i %3i%% data: %6.6s/%-6.6s %3i%% " \
- "time: %8.8s/%8.8s"
- df = self.finished_files
- tf = self.numfiles or 1
- pf = 100 * float(df)/tf + 0.49
- dd = self.re.last_amount_read
- td = self.total_size
- pd = 100 * (self.re.fraction_read() or 0) + 0.49
- dt = self.re.elapsed_time()
- rt = self.re.remaining_time()
- if rt is None: tt = None
- else: tt = dt + rt
-
- fdd = format_number(dd) + 'B'
- ftd = format_number(td) + 'B'
- fdt = format_time(dt, 1)
- ftt = format_time(tt, 1)
-
- out = '%-79.79s' % (format % (df, tf, pf, fdd, ftd, pd, fdt, ftt))
- self.fo.write('\r' + out)
- self.fo.flush()
- finally:
- self._lock.release()
-
- def _do_end_meter(self, meter, now):
- self._lock.acquire()
- try:
- format = "%-30.30s %6.6s %8.8s %9.9s"
- fn = meter.basename
- size = meter.last_amount_read
- fsize = format_number(size) + 'B'
- et = meter.re.elapsed_time()
- fet = format_time(et, 1)
- frate = format_number(size / et) + 'B/s'
-
- out = '%-79.79s' % (format % (fn, fsize, fet, frate))
- self.fo.write('\r' + out + '\n')
- finally:
- self._lock.release()
- self._do_update_meter(meter, now)
-
- def _do_failure_meter(self, meter, message, now):
- self._lock.acquire()
- try:
- format = "%-30.30s %6.6s %s"
- fn = meter.basename
- if type(message) in (type(''), type(u'')):
- message = message.splitlines()
- if not message: message = ['']
- out = '%-79s' % (format % (fn, 'FAILED', message[0] or ''))
- self.fo.write('\r' + out + '\n')
- for m in message[1:]: self.fo.write(' ' + m + '\n')
- self._lock.release()
- finally:
- self._do_update_meter(meter, now)
-
- def message_meter(self, meter, message):
- self._lock.acquire()
- try:
- pass
- finally:
- self._lock.release()
-
- def _do_end(self, now):
- self._do_update_meter(None, now)
- self._lock.acquire()
- try:
- self.fo.write('\n')
- self.fo.flush()
- finally:
- self._lock.release()
-
-######################################################################
-# support classes and functions
-
-class RateEstimator:
- def __init__(self, timescale=5.0):
- self.timescale = timescale
-
- def start(self, total=None, now=None):
- if now is None: now = time.time()
- self.total = total
- self.start_time = now
- self.last_update_time = now
- self.last_amount_read = 0
- self.ave_rate = None
-
- def update(self, amount_read, now=None):
- if now is None: now = time.time()
- if amount_read == 0:
- # if we just started this file, all bets are off
- self.last_update_time = now
- self.last_amount_read = 0
- self.ave_rate = None
- return
-
- #print 'times', now, self.last_update_time
- time_diff = now - self.last_update_time
- read_diff = amount_read - self.last_amount_read
- self.last_update_time = now
- self.last_amount_read = amount_read
- self.ave_rate = self._temporal_rolling_ave(\
- time_diff, read_diff, self.ave_rate, self.timescale)
- #print 'results', time_diff, read_diff, self.ave_rate
-
- #####################################################################
- # result methods
- def average_rate(self):
- "get the average transfer rate (in bytes/second)"
- return self.ave_rate
-
- def elapsed_time(self):
- "the time between the start of the transfer and the most recent update"
- return self.last_update_time - self.start_time
-
- def remaining_time(self):
- "estimated time remaining"
- if not self.ave_rate or not self.total: return None
- return (self.total - self.last_amount_read) / self.ave_rate
-
- def fraction_read(self):
- """the fraction of the data that has been read
- (can be None for unknown transfer size)"""
- if self.total is None: return None
- elif self.total == 0: return 1.0
- else: return float(self.last_amount_read)/self.total
-
- #########################################################################
- # support methods
- def _temporal_rolling_ave(self, time_diff, read_diff, last_ave, timescale):
- """a temporal rolling average performs smooth averaging even when
- updates come at irregular intervals. This is performed by scaling
- the "epsilon" according to the time since the last update.
- Specifically, epsilon = time_diff / timescale
-
- As a general rule, the average will take on a completely new value
- after 'timescale' seconds."""
- epsilon = time_diff / timescale
- if epsilon > 1: epsilon = 1.0
- return self._rolling_ave(time_diff, read_diff, last_ave, epsilon)
-
- def _rolling_ave(self, time_diff, read_diff, last_ave, epsilon):
- """perform a "rolling average" iteration
- a rolling average "folds" new data into an existing average with
- some weight, epsilon. epsilon must be between 0.0 and 1.0 (inclusive)
- a value of 0.0 means only the old value (initial value) counts,
- and a value of 1.0 means only the newest value is considered."""
-
- try:
- recent_rate = read_diff / time_diff
- except ZeroDivisionError:
- recent_rate = None
- if last_ave is None: return recent_rate
- elif recent_rate is None: return last_ave
-
- # at this point, both last_ave and recent_rate are numbers
- return epsilon * recent_rate + (1 - epsilon) * last_ave
-
- def _round_remaining_time(self, rt, start_time=15.0):
- """round the remaining time, depending on its size
- If rt is between n*start_time and (n+1)*start_time round downward
- to the nearest multiple of n (for any counting number n).
- If rt < start_time, round down to the nearest 1.
- For example (for start_time = 15.0):
- 2.7 -> 2.0
- 25.2 -> 25.0
- 26.4 -> 26.0
- 35.3 -> 34.0
- 63.6 -> 60.0
- """
-
- if rt < 0: return 0.0
- shift = int(math.log(rt/start_time)/math.log(2))
- rt = int(rt)
- if shift <= 0: return rt
- return float(int(rt) >> shift << shift)
-
-
-def format_time(seconds, use_hours=0):
- if seconds is None or seconds < 0:
- if use_hours: return '--:--:--'
- else: return '--:--'
- else:
- seconds = int(seconds)
- minutes = seconds / 60
- seconds = seconds % 60
- if use_hours:
- hours = minutes / 60
- minutes = minutes % 60
- return '%02i:%02i:%02i' % (hours, minutes, seconds)
- else:
- return '%02i:%02i' % (minutes, seconds)
-
-def format_number(number, SI=0, space=' '):
- """Turn numbers into human-readable metric-like numbers"""
- symbols = ['', # (none)
- 'k', # kilo
- 'M', # mega
- 'G', # giga
- 'T', # tera
- 'P', # peta
- 'E', # exa
- 'Z', # zetta
- 'Y'] # yotta
-
- if SI: step = 1000.0
- else: step = 1024.0
-
- thresh = 999
- depth = 0
- max_depth = len(symbols) - 1
-
- # we want numbers between 0 and thresh, but don't exceed the length
- # of our list. In that event, the formatting will be screwed up,
- # but it'll still show the right number.
- while number > thresh and depth < max_depth:
- depth = depth + 1
- number = number / step
-
- if type(number) == type(1) or type(number) == type(1L):
- # it's an int or a long, which means it didn't get divided,
- # which means it's already short enough
- format = '%i%s%s'
- elif number < 9.95:
- # must use 9.95 for proper sizing. For example, 9.99 will be
- # rounded to 10.0 with the .1f format string (which is too long)
- format = '%.1f%s%s'
- else:
- format = '%.0f%s%s'
-
- return(format % (float(number or 0), space, symbols[depth]))
+++ /dev/null
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330,
-# Boston, MA 02111-1307 USA
-
-# This file is part of urlgrabber, a high-level cross-protocol url-grabber
-
-import httplib
-import urllib2
-
-try:
- from M2Crypto import SSL
- from M2Crypto import httpslib
- from M2Crypto import m2urllib2
-
- have_m2crypto = True
-except ImportError:
- have_m2crypto = False
-
-DEBUG = None
-
-if have_m2crypto:
-
- class M2SSLFactory:
-
- def __init__(self, ssl_ca_cert, ssl_context):
- self.ssl_context = self._get_ssl_context(ssl_ca_cert, ssl_context)
-
- def _get_ssl_context(self, ssl_ca_cert, ssl_context):
- """
- Create an ssl context using the CA cert file or ssl context.
-
- The CA cert is used first if it was passed as an option. If not,
- then the supplied ssl context is used. If no ssl context was supplied,
- None is returned.
- """
- if ssl_ca_cert:
- context = SSL.Context()
- context.load_verify_locations(ssl_ca_cert)
- context.set_verify(SSL.verify_peer, -1)
- return context
- else:
- return ssl_context
-
- def create_https_connection(self, host, response_class = None):
- connection = httplib.HTTPSConnection(host, self.ssl_context)
- if response_class:
- connection.response_class = response_class
- return connection
-
- def create_opener(self, *handlers):
- return m2urllib2.build_opener(self.ssl_context, *handlers)
-
-
-class SSLFactory:
-
- def create_https_connection(self, host, response_class = None):
- connection = httplib.HTTPSConnection(host)
- if response_class:
- connection.response_class = response_class
- return connection
-
- def create_opener(self, *handlers):
- return urllib2.build_opener(*handlers)
-
-
-
-def get_factory(ssl_ca_cert = None, ssl_context = None):
- """ Return an SSLFactory, based on if M2Crypto is available. """
- if have_m2crypto:
- return M2SSLFactory(ssl_ca_cert, ssl_context)
- else:
- # Log here if someone provides the args but we don't use them.
- if ssl_ca_cert or ssl_context:
- if DEBUG:
- DEBUG.warning("SSL arguments supplied, but M2Crypto is not available. "
- "Using Python SSL.")
- return SSLFactory()
+++ /dev/null
-#!/usr/bin/python
-
-import ctypes
-import fcntl
-import os
-import random
-import select
-import shutil
-import string
-import subprocess
-import sys
-import time
-
-from constants import *
-from exception import *
-from logger import getLog
-
-_libc = ctypes.cdll.LoadLibrary(None)
-_errno = ctypes.c_int.in_dll(_libc, "errno")
-_libc.personality.argtypes = [ctypes.c_ulong]
-_libc.personality.restype = ctypes.c_int
-_libc.unshare.argtypes = [ctypes.c_int,]
-_libc.unshare.restype = ctypes.c_int
-CLONE_NEWNS = 0x00020000
-
-# taken from sys/personality.h
-PER_LINUX32=0x0008
-PER_LINUX=0x0000
-personality_defs = {
- 'linux64': PER_LINUX,
- 'linux32': PER_LINUX32,
-}
-
-def touch(filename):
- getLog().debug("touching file: %s" % filename)
- f = open(filename, "w")
- f.close()
-
-def mkdir(*args):
- for dirName in args:
- getLog().debug("ensuring that dir exists: %s" % dirName)
- if not os.path.exists(dirName):
- try:
- getLog().debug("creating dir: %s" % dirName)
- os.makedirs(dirName)
- except OSError, e:
- getLog().exception("Could not create dir %s. Error: %s" % (dirName, e))
- raise Error, "Could not create dir %s. Error: %s" % (dirName, e)
-
-def rm(path, *args, **kargs):
- """version os shutil.rmtree that ignores no-such-file-or-directory errors,
- and tries harder if it finds immutable files"""
- tryAgain = 1
- failedFilename = None
- getLog().debug("remove tree: %s" % path)
- while tryAgain:
- tryAgain = 0
- try:
- shutil.rmtree(path, *args, **kargs)
- except OSError, e:
- if e.errno == 2: # no such file or directory
- pass
- elif e.errno==1 or e.errno==13:
- tryAgain = 1
- if failedFilename == e.filename:
- raise
- failedFilename = e.filename
- os.system("chattr -R -i %s" % path)
- else:
- raise
-
-def logOutput(fds, logger, returnOutput=1, start=0, timeout=0):
- output=""
- done = 0
-
- # set all fds to nonblocking
- for fd in fds:
- flags = fcntl.fcntl(fd, fcntl.F_GETFL)
- if not fd.closed:
- fcntl.fcntl(fd, fcntl.F_SETFL, flags| os.O_NONBLOCK)
-
- tail = ""
- while not done:
- if (time.time() - start)>timeout and timeout!=0:
- done = 1
- break
-
- i_rdy,o_rdy,e_rdy = select.select(fds,[],[],1)
- for s in i_rdy:
- # slurp as much input as is ready
- input = s.read()
- if input == "":
- done = 1
- break
- if logger is not None:
- lines = input.split("\n")
- if tail:
- lines[0] = tail + lines[0]
- # we may not have all of the last line
- tail = lines.pop()
- for line in lines:
- if line == '': continue
- logger.info(line)
- for h in logger.handlers:
- h.flush()
- if returnOutput:
- output += input
- if tail and logger is not None:
- logger.debug(tail)
- return output
-
-# these are called in child process, so no logging
-def condChroot(chrootPath):
- if chrootPath is not None:
- os.chdir(chrootPath)
- os.chroot(chrootPath)
-
-def condChdir(cwd):
- if cwd is not None:
- os.chdir(cwd)
-
-def condPersonality(per=None):
- if not per:
- return
- if personality_defs.get(per, None) is None:
- return
- res = _libc.personality(personality_defs[per])
- if res == -1:
- raise OSError(_errno.value, os.strerror(_errno.value))
-
-def do(command, shell=False, chrootPath=None, cwd=None, timeout=0, raiseExc=True, returnOutput=0, personality=None, logger=None, *args, **kargs):
- output = ""
- start = time.time()
- env = kargs.get("env", None)
- preexec = ChildPreExec(personality, chrootPath, cwd)
-
- try:
- child = None
- if logger:
- logger.debug("Executing command: %s" % command)
- child = subprocess.Popen(
- command,
- shell=shell,
- bufsize=0, close_fds=True,
- stdin=open("/dev/null", "r"),
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- preexec_fn = preexec,
- env=env
- )
-
- # use select() to poll for output so we dont block
- output = logOutput([child.stdout, child.stderr],
- logger, returnOutput, start, timeout)
-
- except:
- # kill children if they arent done
- if child is not None and child.returncode is None:
- os.killpg(child.pid, 9)
- try:
- if child is not None:
- os.waitpid(child.pid, 0)
- except:
- pass
- raise
-
- # wait until child is done, kill it if it passes timeout
- niceExit=1
- while child.poll() is None:
- if (time.time() - start)>timeout and timeout!=0:
- niceExit=0
- os.killpg(child.pid, 15)
- if (time.time() - start)>(timeout+1) and timeout!=0:
- niceExit=0
- os.killpg(child.pid, 9)
-
- if not niceExit:
- raise commandTimeoutExpired, ("Timeout(%s) expired for command:\n # %s\n%s" % (timeout, command, output))
-
- if logger:
- logger.debug("Child returncode was: %s" % str(child.returncode))
- if raiseExc and child.returncode:
- if returnOutput:
- raise Error, ("Command failed: \n # %s\n%s" % (command, output), child.returncode)
- else:
- raise Error, ("Command failed. See logs for output.\n # %s" % (command,), child.returncode)
-
- return output
-
-class ChildPreExec(object):
- def __init__(self, personality, chrootPath, cwd):
- self.personality = personality
- self.chrootPath = chrootPath
- self.cwd = cwd
-
- def __call__(self, *args, **kargs):
- os.setpgrp()
- condPersonality(self.personality)
- condChroot(self.chrootPath)
- condChdir(self.cwd)
-
-def random_string(length=5):
- ret = ""
- while length:
- ret += random.choice(string.hexdigits)
- length -= 1
-
- return ret
+++ /dev/null
-
-###############################################################################
-#
-# Constant definitions of the naoki build system
-#
-###############################################################################
-
-# If we are running in a chroot, we map all directories to $(BASEDIR), otherwise
-# BASEDIR will be the absolute path (for gathering information from the source
-# packages)
-ifeq "$(CHROOT)" "1"
- BASEDIR = /usr/src
-endif
-
-# Set default directories
-DIR_APP = $(DIR_SRC)/$(THISAPP)
-DIR_DL = $(BASEDIR)/cache
-DIR_PACKAGES = /usr/src/packages/$(PKG_ARCH)
-DIR_PATCHES = $(DIR_SOURCE)/patches
-DIR_SRC = /usr/src
-DIR_TMP = /tmp
-DIR_SOURCE = $(CURDIR)
-DIR_TOOLS = $(BASEDIR)/tools
-
-# Directory where to search for object files
-VPATH = $(DIR_DL)
-
-# Paths to scripts
-DO_EXTRACT = tar xaf
-DO_QUALITY_AGENT = $(DIR_TOOLS)/quality-agent
-
-###############################################################################
-#
-# Build environment
-#
-###############################################################################
-
-# Export CFLAGS + CXXFLAGS
-export CFLAGS
-export CXXFLAGS
-
-# Options that get passed to configure by default
-CONFIGURE_OPTIONS = \
- --prefix=/usr \
- --build=$(TARGET) \
- --host=$(TARGET)
-
-# For compatibility with pakfire3
-DISTRO_MACHINE=$(TARGET)
-
-###############################################################################
-#
-# Packager variables
-#
-###############################################################################
-# Variables that get exported (and expanded for the sub-packages) to the
-# packager process
-PKG_VARIABLES = \
- CONTROL_PREIN \
- CONTROL_PREUN \
- CONTROL_POSTIN \
- CONTROL_POSTUN \
- \
- PKG_ARCH \
- PKG_BUILD_DEPS \
- PKG_DEPS \
- PKG_DESCRIPTION \
- PKG_EPOCH \
- PKG_FILES \
- PKG_GROUP \
- PKG_MAINTAINER \
- PKG_LICENSE \
- PKG_PROVIDES \
- PKG_REL \
- PKG_SUMMARY \
- PKG_URL \
- PKG_VER
-
-# Variables that exported to the packager process
-# These reflect settings from the build system
-export BUILD_DATE ?= $(shell date -u)
-export BUILD_HOST ?= $(shell cat /proc/sys/kernel/hostname)
-export BUILD_ID
-
-###############################################################################
-#
-# Package variables
-#
-###############################################################################
-#
-# Variables
-#
-
-# The actual package name (the name of the directory)
-PKG_NAME_REAL = $(notdir $(CURDIR))
-PKG_NAME = $(PKG_NAME_REAL)
-
-# Set default epoch
-PKG_EPOCH = 0
-
-# Shortcut to package name + version
-THISAPP = $(PKG_NAME)-$(PKG_VER)
-THISVER = $(PKG_EPOCH):$(PKG_VER)-$(PKG_REL)
-
-# All packages depend on gcc and headers by default.
-PKG_BUILD_DEPS+= gcc glibc-devel kernel-headers
-PKG_DEPS +=
-
-# All PKG_OBJECTS are downloaded. This is in most cases the tarball.
-PKG_OBJECTS = $(PKG_TARBALL)
-OBJECTS = $(PKG_OBJECTS)
-
-# List of packages to build
-PKG_PACKAGES = $(PKG_NAME_REAL)
-
-#
-# Macros
-#
-
-# Abstract variable that translates the package names to a list of filenames
-PKG_PACKAGES_FILES = $(foreach package,$(PKG_PACKAGES),$(call DO_PACKAGE_FILENAME,$(package)))
-
-# Automatically detect all patches in "patches"
-PKG_PATCHES = \
- $(foreach patch,$(wildcard $(DIR_PATCHES)/*.patch),$(notdir $(patch)))
-PKG_PATCHES += \
- $(foreach patch,$(wildcard $(DIR_PATCHES)/*.patch0),$(notdir $(patch)))
-PKG_PATCHES += \
- $(foreach patch,$(wildcard $(DIR_PATCHES)/*.diff),$(notdir $(patch)))
-
-# Dynamic command that applies all patches
-DO_PATCHES = cd $(DIR_APP) && $(DIR_TOOLS)/patch $(foreach patch,$(PKG_PATCHES),$(DIR_PATCHES)/$(patch))
-
-# Get a list of files that are installed automatically
-PKG_DEFAULT_FILES = $(wildcard *.default)
-PKG_DEFAULT_FILES += $(wildcard default/*)
-PKG_INIT_FILES = $(wildcard *.init)
-PKG_INIT_FILES += $(wildcard init/*.conf)
-PKG_PAM_FILES = $(wildcard *.pam)
-PKG_PAM_FILES += $(wildcard pam.d/*)
-
-###############################################################################
-#
-# Quality agent
-#
-###############################################################################
-
-export QUALITY_AGENT_PERMIT_NOT_FULL_RELRO
-export QUALITY_AGENT_RPATH_ALLOW_ORIGIN
-export QUALITY_AGENT_WHITELIST_EXECSTACK
-export QUALITY_AGENT_WHITELIST_NX
-export QUALITY_AGENT_WHITELIST_RPATH
-export QUALITY_AGENT_WHITELIST_SONAME
+++ /dev/null
-
-###############################################################################
-#
-# Function definitions of the naoki build system
-#
-###############################################################################
-
-include $(PKGROOT)/gmsl
-
-DO_PACKAGE_FILENAME = $(1)$(call DO_PKG_SUFFIX,$(1))
-
-DO_PKG_SUFFIX = \
- -$(if $(PKG_VER-$(1)),$(PKG_VER-$(1)),$(PKG_VER))-$(DISTRO_SNAME)$(DISTRO_EPOCH)-$(PKG_ARCH).$(if $(PKG_REL-$(1)),$(PKG_REL-$(1)),$(PKG_REL)).ipk
-
-define DO_INIT
- # Run ldconfig
- ldconfig
-endef
-
-define DO_PACKAGE
- @echo "#####################################################################"
- @echo "# $(1) - Package build started"
- @echo "#####################################################################"
-
- @$(foreach var,$(PKG_VARIABLES),$(if $($(var)-$(1)),$(var)="$(strip $($(var)-$(1)))",$(var)="$(strip $($(var)))")) \
- $(DIR_TOOLS)/packager $(1) $(DIR_PACKAGES)/$(call DO_PACKAGE_FILENAME,$(1))
-
- @echo "#####################################################################"
- @echo "# $(1) - Package build finished"
- @echo "#####################################################################"
-
-endef
-
-define DO_FILELIST
- @echo "# Filelist dump"
- @cd $(BUILDROOT) && find -ls
-endef
-
-define __INSTALL_DEFAULT
- -mkdir -pv $(BUILDROOT)/etc/default
- cd $(DIR_APP) && cp -vf $(DIR_SOURCE)/$(1) $(BUILDROOT)/etc/default/$(subst .default,,$(notdir $(1)))
-
-endef
-
-define DO_INSTALL_DEFAULT
- $(foreach file,$(PKG_DEFAULT_FILES),$(call __INSTALL_DEFAULT,$(file)))
-endef
-
-define __INSTALL_INIT
- -mkdir -pv $(BUILDROOT)/etc/init
- cd $(DIR_APP) && cp -vf $(DIR_SOURCE)/$(1) $(BUILDROOT)/etc/init/$(subst .init,.conf,$(notdir $(1)))
-
-endef
-
-define DO_INSTALL_INIT
- $(foreach file,$(PKG_INIT_FILES),$(call __INSTALL_INIT,$(file)))
-endef
-
-define __INSTALL_PAM
- -mkdir -pv $(BUILDROOT)/etc/pam.d
- cd $(DIR_APP) && cp -vf $(DIR_SOURCE)/$(1) $(BUILDROOT)/etc/pam.d/$(subst .pam,,$(notdir $(1)))
-
-endef
-
-define DO_INSTALL_PAM
- $(foreach file,$(PKG_PAM_FILES),$(call __INSTALL_PAM,$(file)))
-endef
-
-define DO_PYTHON_COMPILE
- @find $(BUILDROOT) -name "*.py" | xargs $(DIR_TOOLS)/py-compile
-endef
-
-define DO_FIX_LIBTOOL
- # remove rpath from libtool
- @if [ -e "libtool" ]; then \
- sed -i libtool \
- -e 's|^hardcode_libdir_flag_spec=.*|hardcode_libdir_flag_spec=""|g' \
- -e 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g'; \
- fi
-endef
-
-define DO_PREPARE
- # Initialize the environment at the beginning
- $(DO_INIT)
-
- @echo "#####################################################################"
- @echo "# $(PKG_NAME) - Preparation started"
- @echo "#####################################################################"
-
- $(STAGE_PREPARE)
-
- @echo "#####################################################################"
- @echo "# $(PKG_NAME) - Preparation finished"
- @echo "#####################################################################"
-endef
-
-define DO_BUILD
- @echo "#####################################################################"
- @echo "# $(PKG_NAME) - Build started"
- @echo "#####################################################################"
-
- $(STAGE_BUILD)
-
- @echo "#####################################################################"
- @echo "# $(PKG_NAME) - Build finished"
- @echo "#####################################################################"
-endef
-
-define DO_TEST
- @echo "#####################################################################"
- @echo "# $(PKG_NAME) - Test started"
- @echo "#####################################################################"
-
- $(STAGE_TEST)
-
- @echo "#####################################################################"
- @echo "# $(PKG_NAME) - Test finished"
- @echo "#####################################################################"
-endef
-
-define DO_INSTALL
- @echo "#####################################################################"
- @echo "# $(PKG_NAME) - Install started"
- @echo "#####################################################################"
-
- -mkdir -pv $(BUILDROOT)
-
- $(STAGE_INSTALL)
-
- $(DO_INSTALL_DEFAULT)
- $(DO_INSTALL_INIT)
- $(DO_INSTALL_PAM)
-
- $(DO_PYTHON_COMPILE)
-
- @echo "#####################################################################"
- @echo "# $(PKG_NAME) - Install finished"
- @echo "#####################################################################"
-
- $(DO_QUALITY_AGENT)
- $(DO_FILELIST)
-endef
-
-STAGE_PACKAGE_TARGETS = $(call reverse,$(PKG_PACKAGES_FILES))
-STAGE_DONE = $(ROOT)/.done
-
-define STAGE_PREPARE
- $(if $(PKG_TARBALL),cd $(DIR_SRC) && $(DO_EXTRACT) $(DIR_DL)/$(PKG_TARBALL))
-
- $(if $(PKG_PATCHES),$(DO_PATCHES))
-
- $(STAGE_PREPARE_CMDS)
- $(STAGE_PREPARE_CMDS2)
-endef
-
-STAGE_BUILD_TARGETS =
-
-define STAGE_BUILD
- cd $(DIR_APP) && \
- $(CONFIGURE_ENVIRONMENT) \
- ./configure \
- $(CONFIGURE_OPTIONS)
-
- $(DO_FIX_LIBTOOL)
- $(STAGE_CONFIGURE_CMDS)
-
- cd $(DIR_APP) && make $(STAGE_BUILD_TARGETS) $(PARALLELISMFLAGS)
- $(STAGE_BUILD_CMDS)
-endef
-
-STAGE_INSTALL_TARGETS = install
-
-define STAGE_INSTALL
- cd $(DIR_APP) && make $(STAGE_INSTALL_TARGETS) DESTDIR=$(BUILDROOT)
-
- $(STAGE_INSTALL_CMDS)
-endef
+++ /dev/null
-###############################################################################
-# #
-# IPFire.org - A linux based firewall #
-# Copyright (C) 2007, 2008, 2009 Michael Tremer & Christian Schmidt #
-# #
-# This program is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published by #
-# the Free Software Foundation, either version 3 of the License, or #
-# (at your option) any later version. #
-# #
-# This program is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with this program. If not, see <http://www.gnu.org/licenses/>. #
-# #
-###############################################################################
-
-.SECONDEXPANSION:
-
-include $(PKGROOT)/Constants
-include $(PKGROOT)/Functions
-include $(PKGROOT)/Targets
-include $(PKGROOT)/Templates
+++ /dev/null
-
-###############################################################################
-#
-# Target definitions of the naoki build system
-#
-###############################################################################
-
-.PHONY: info
-info:
- @echo "PKG_ARCH=\"$(PKG_ARCH)\""
- @echo "PKG_BUILD_DEPENDENCIES=\"$(PKG_BUILD_DEPS)\""
- @echo "PKG_DEPENDENCIES=\"$(PKG_DEPS)\""
- @echo "PKG_DESCRIPTION=\"$(strip $(PKG_DESCRIPTION))\""
- @echo "PKG_EPOCH=\"$(PKG_EPOCH)\""
- @echo "PKG_GROUP=\"$(PKG_GROUP)\""
- @echo "PKG_LICENSE=\"$(PKG_LICENSE)\""
- @echo "PKG_MAINTAINER=\"$(PKG_MAINTAINER)\""
- @echo "PKG_NAME=\"$(PKG_NAME_REAL)\""
- @echo "PKG_OBJECTS=\"$(strip $(OBJECTS))\""
- @echo "PKG_PACKAGES=\"$(PKG_PACKAGES)\""
- @echo "PKG_PACKAGES_FILES=\"$(PKG_PACKAGES_FILES)\""
- @echo "PKG_PATCHES=\"$(PKG_PATCHES)\""
- @echo "PKG_VER=\"$(PKG_VER)\""
- @echo "PKG_REL=\"$(PKG_REL)\""
- @echo "PKG_SUMMARY=\"$(strip $(PKG_SUMMARY))\""
- @echo "PKG_URL=\"$(PKG_URL)\""
- @echo "SOURCE_DIR=\"$(DIR_APP)\""
-
-$(OBJECTS):
- @echo "Object file \"$@\" is required." >&2
- @exit 1
-
-.PHONY: package
-package: $(STAGE_DONE)
- $(foreach package,$(call reverse,$(PKG_PACKAGES)),$(call DO_PACKAGE,$(package)))
-
-.PHONY: shell
-shell: $(OBJECTS)
- $(if $(STAGE_PREPARE),$(DO_PREPARE))
-
-prepare:
- $(if $(STAGE_PREPARE),$(DO_PREPARE))
-
-$(STAGE_DONE): $(OBJECTS)
- $(if $(STAGE_PREPARE),$(DO_PREPARE))
- $(if $(STAGE_BUILD),$(DO_BUILD))
- $(if $(STAGE_TEST),$(DO_TEST))
- $(if $(STAGE_INSTALL),$(DO_INSTALL))
- @touch $@
+++ /dev/null
-
-###############################################################################
-#
-# Template definitions of the naoki build system
-#
-###############################################################################
-
-###############################################################################
-# Default template
-###############################################################################
-
-PKG_PROVIDES-$(PKG_NAME_REAL) = $(PKG_NAME_REAL)=$(THISVER)
-
-# Package all files by default
-PKG_FILES = /
-
-###############################################################################
-# Devel template
-###############################################################################
-
-PKG_DESCRIPTION-$(PKG_NAME_REAL)-devel = Development files of $(THISAPP).
-PKG_SUMMARY-$(PKG_NAME_REAL)-devel = $(PKG_DESCRIPTION-$(PKG_NAME_REAL)-devel)
-
-PKG_DEPS-$(PKG_NAME_REAL)-devel = $(PKG_NAME_REAL)=$(THISVER)
-
-define PKG_FILES-$(PKG_NAME_REAL)-devel
- /usr/bin/*-config
- /usr/include
- /usr/lib/*.a
- /usr/lib/pkgconfig
- /usr/share/aclocal
- */lib/*.so
- /usr/share/*/cmake
- /usr/share/man/man2
- /usr/share/man/man3
- /usr/share/vala
-endef
-
-
-###############################################################################
-# Library template
-###############################################################################
-
-PKG_DESCRIPTION-$(PKG_NAME_REAL)-libs = Library files of $(THISAPP).
-PKG_SUMMARY-$(PKG_NAME_REAL)-libs = $(PKG_DESCRIPTION-$(PKG_NAME_REAL)-libs)
-
-PKG_DEPS-$(PKG_NAME_REAL)-libs = $(PKG_NAME_REAL)=$(THISVER)
-
-define PKG_FILES-$(PKG_NAME_REAL)-libs
- /lib/lib*.so.*
- /usr/lib/lib*.so.*
-endef
-
-# Another naming scheme template...
-PKG_DESCRIPTION-lib$(PKG_NAME_REAL) = $(PKG_DESCRIPTION-$(PKG_NAME_REAL)-libs)
-PKG_SUMMARY-lib$(PKG_NAME_REAL) = $(PKG_SUMMARY-$(PKG_NAME_REAL)-libs)
-PKG_FILES-lib$(PKG_NAME_REAL) = $(PKG_FILES-$(PKG_NAME_REAL)-libs)
-
-PKG_DESCRIPTION-lib$(PKG_NAME_REAL)-devel = $(PKG_DESCRIPTION-$(PKG_NAME_REAL)-devel)
-PKG_SUMMARY-lib$(PKG_NAME_REAL)-devel = $(PKG_SUMMARY-$(PKG_NAME_REAL)-devel)
-PKG_FILES-lib$(PKG_NAME_REAL)-devel = $(PKG_FILES-$(PKG_NAME_REAL)-devel) blah
-PKG_BUILD_DEPS-lib$(PKG_NAME_REAL)-devel = $(PKG_BUILD_DEPS-$(PKG_NAME_REAL)-devel)
+++ /dev/null
-# ----------------------------------------------------------------------------
-#
-# GNU Make Standard Library (GMSL)
-#
-# A library of functions to be used with GNU Make's $(call) that
-# provides functionality not available in standard GNU Make.
-#
-# Copyright (c) 2005-2007 John Graham-Cumming
-#
-# This file is part of GMSL
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#
-# Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-#
-# Neither the name of the John Graham-Cumming nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-#
-# ----------------------------------------------------------------------------
-
-# This is the GNU Make Standard Library version number as a list with
-# three items: major, minor, revision
-
-gmsl_version := 1 0 11
-
-# Used to output warnings and error from the library, it's possible to
-# disable any warnings or errors by overriding these definitions
-# manually or by setting GMSL_NO_WARNINGS or GMSL_NO_ERRORS
-
-__gmsl_name := GNU Make Standard Library
-__gmsl_warning = $(warning $(__gmsl_name): $1)
-__gmsl_error = $(error $(__gmsl_name): $1)
-
-ifdef GMSL_NO_WARNINGS
-__gmsl_warning :=
-endif
-ifdef GMSL_NO_ERRORS
-__gmsl_error :=
-endif
-
-# If GMSL_TRACE is enabled then calls to the library functions are
-# traced to stdout using warning messages with their arguments
-
-ifdef GMSL_TRACE
-__gmsl_tr1 = $(warning $0('$1'))
-__gmsl_tr2 = $(warning $0('$1','$2'))
-__gmsl_tr3 = $(warning $0('$1','$2','$3'))
-else
-__gmsl_tr1 :=
-__gmsl_tr2 :=
-__gmsl_tr3 :=
-endif
-
-# Figure out whether we have $(eval) or not (GNU Make 3.80 and above)
-# if we do not then output a warning message, if we do then some
-# functions will be enabled.
-
-__gmsl_have_eval := $(false)
-__gmsl_ignore := $(eval __gmsl_have_eval := $(true))
-
-# If this is being run with Electric Cloud's emake then warn that
-# their $(eval) support is incomplete.
-
-ifdef ECLOUD_BUILD_ID
-$(warning You are using Electric Cloud's emake which has incomplete $$(eval) support)
-__gmsl_have_eval := $(false)
-endif
-
-# See if we have $(lastword) (GNU Make 3.81 and above)
-
-__gmsl_have_lastword := $(lastword $(false) $(true))
-
-# See if we have native or and and (GNU Make 3.81 and above)
-
-__gmsl_have_or := $(if $(filter-out undefined, \
- $(origin or)),$(call or,$(true),$(false)))
-__gmsl_have_and := $(if $(filter-out undefined, \
- $(origin and)),$(call and,$(true),$(true)))
-
-ifneq ($(__gmsl_have_eval),$(true))
-$(call __gmsl_warning,GNU Make $(MAKE_VERSION) does not support $$$$(eval): some functions disabled)
-endif
-
-# ----------------------------------------------------------------------------
-# Function: gmsl_compatible
-# Arguments: List containing the desired library version number (maj min rev)
-# Returns: $(true) if this version of the library is compatible
-# with the requested version number, otherwise $(false)
-# ----------------------------------------------------------------------------
-gmsl_compatible = $(strip \
- $(if $(call gt,$(word 1,$1),$(word 1,$(gmsl_version))), \
- $(false), \
- $(if $(call lt,$(word 1,$1),$(word 1,$(gmsl_version))), \
- $(true), \
- $(if $(call gt,$(word 2,$1),$(word 2,$(gmsl_version))), \
- $(false), \
- $(if $(call lt,$(word 2,$1),$(word 2,$(gmsl_version))), \
- $(true), \
- $(call lte,$(word 3,$1),$(word 3,$(gmsl_version))))))))
-
-# ###########################################################################
-# LOGICAL OPERATORS
-# ###########################################################################
-
-# not is defined in gmsl
-
-# ----------------------------------------------------------------------------
-# Function: and
-# Arguments: Two boolean values
-# Returns: Returns $(true) if both of the booleans are true
-# ----------------------------------------------------------------------------
-ifneq ($(__gmsl_have_and),$(true))
-and = $(__gmsl_tr2)$(if $1,$(if $2,$(true),$(false)),$(false))
-endif
-
-# ----------------------------------------------------------------------------
-# Function: or
-# Arguments: Two boolean values
-# Returns: Returns $(true) if either of the booleans is true
-# ----------------------------------------------------------------------------
-ifneq ($(__gmsl_have_or),$(true))
-or = $(__gmsl_tr2)$(if $1$2,$(true),$(false))
-endif
-
-# ----------------------------------------------------------------------------
-# Function: xor
-# Arguments: Two boolean values
-# Returns: Returns $(true) if exactly one of the booleans is true
-# ----------------------------------------------------------------------------
-xor = $(__gmsl_tr2)$(if $1,$(if $2,$(false),$(true)),$(if $2,$(true),$(false)))
-
-# ----------------------------------------------------------------------------
-# Function: nand
-# Arguments: Two boolean values
-# Returns: Returns value of 'not and'
-# ----------------------------------------------------------------------------
-nand = $(__gmsl_tr2)$(if $1,$(if $2,$(false),$(true)),$(true))
-
-# ----------------------------------------------------------------------------
-# Function: nor
-# Arguments: Two boolean values
-# Returns: Returns value of 'not or'
-# ----------------------------------------------------------------------------
-nor = $(__gmsl_tr2)$(if $1$2,$(false),$(true))
-
-# ----------------------------------------------------------------------------
-# Function: xnor
-# Arguments: Two boolean values
-# Returns: Returns value of 'not xor'
-# ----------------------------------------------------------------------------
-xnor =$(__gmsl_tr2)$(if $1,$(if $2,$(true),$(false)),$(if $2,$(false),$(true)))
-
-# ###########################################################################
-# LIST MANIPULATION FUNCTIONS
-# ###########################################################################
-
-# ----------------------------------------------------------------------------
-# Function: first (same as LISP's car, or head)
-# Arguments: 1: A list
-# Returns: Returns the first element of a list
-# ----------------------------------------------------------------------------
-first = $(__gmsl_tr1)$(firstword $1)
-
-# ----------------------------------------------------------------------------
-# Function: last
-# Arguments: 1: A list
-# Returns: Returns the last element of a list
-# ----------------------------------------------------------------------------
-ifeq ($(__gmsl_have_lastword),$(true))
-last = $(__gmsl_tr1)$(lastword $1)
-else
-last = $(__gmsl_tr1)$(if $1,$(word $(words $1),$1))
-endif
-
-# ----------------------------------------------------------------------------
-# Function: rest (same as LISP's cdr, or tail)
-# Arguments: 1: A list
-# Returns: Returns the list with the first element removed
-# ----------------------------------------------------------------------------
-rest = $(__gmsl_tr1)$(wordlist 2,$(words $1),$1)
-
-# ----------------------------------------------------------------------------
-# Function: chop
-# Arguments: 1: A list
-# Returns: Returns the list with the last element removed
-# ----------------------------------------------------------------------------
-chop = $(__gmsl_tr1)$(wordlist 2,$(words $1),x $1)
-
-# ----------------------------------------------------------------------------
-# Function: map
-# Arguments: 1: Name of function to $(call) for each element of list
-# 2: List to iterate over calling the function in 1
-# Returns: The list after calling the function on each element
-# ----------------------------------------------------------------------------
-map = $(__gmsl_tr2)$(strip $(foreach a,$2,$(call $1,$a)))
-
-# ----------------------------------------------------------------------------
-# Function: pairmap
-# Arguments: 1: Name of function to $(call) for each pair of elements
-# 2: List to iterate over calling the function in 1
-# 3: Second list to iterate over calling the function in 1
-# Returns: The list after calling the function on each pair of elements
-# ----------------------------------------------------------------------------
-pairmap = $(strip $(__gmsl_tr3)\
- $(if $2$3,$(call $1,$(call first,$2),$(call first,$3)) \
- $(call pairmap,$1,$(call rest,$2),$(call rest,$3))))
-
-# ----------------------------------------------------------------------------
-# Function: leq
-# Arguments: 1: A list to compare against...
-# 2: ...this list
-# Returns: Returns $(true) if the two lists are identical
-# ----------------------------------------------------------------------------
-leq = $(__gmsl_tr2)$(strip $(if $(call seq,$(words $1),$(words $2)), \
- $(call __gmsl_list_equal,$1,$2),$(false)))
-
-__gmsl_list_equal = $(if $(strip $1), \
- $(if $(call seq,$(call first,$1),$(call first,$2)), \
- $(call __gmsl_list_equal, \
- $(call rest,$1), \
- $(call rest,$2)), \
- $(false)), \
- $(true))
-
-# ----------------------------------------------------------------------------
-# Function: lne
-# Arguments: 1: A list to compare against...
-# 2: ...this list
-# Returns: Returns $(true) if the two lists are different
-# ----------------------------------------------------------------------------
-lne = $(__gmsl_tr2)$(call not,$(call leq,$1,$2))
-
-# ----------------------------------------------------------------------------
-# Function: reverse
-# Arguments: 1: A list to reverse
-# Returns: The list with its elements in reverse order
-# ----------------------------------------------------------------------------
-reverse =$(__gmsl_tr1)$(strip $(if $1,$(call reverse,$(call rest,$1)) \
- $(call first,$1)))
-
-# ----------------------------------------------------------------------------
-# Function: uniq
-# Arguments: 1: A list from which to remove repeated elements
-# Returns: The list with duplicate elements removed without reordering
-# ----------------------------------------------------------------------------
-uniq = $(strip $(__gmsl_tr1)$(if $1,$(call uniq,$(call chop,$1)) \
- $(if $(filter $(call last,$1),$(call chop,$1)),,$(call last,$1))))
-
-# ----------------------------------------------------------------------------
-# Function: length
-# Arguments: 1: A list
-# Returns: The number of elements in the list
-# ----------------------------------------------------------------------------
-length = $(__gmsl_tr1)$(words $1)
-
-# ###########################################################################
-# STRING MANIPULATION FUNCTIONS
-# ###########################################################################
-
-# Helper function that translates any GNU Make 'true' value (i.e. a
-# non-empty string) to our $(true)
-
-__gmsl_make_bool = $(if $(strip $1),$(true),$(false))
-
-# ----------------------------------------------------------------------------
-# Function: seq
-# Arguments: 1: A string to compare against...
-# 2: ...this string
-# Returns: Returns $(true) if the two strings are identical
-# ----------------------------------------------------------------------------
-seq = $(__gmsl_tr2)$(if $(filter-out xx,x$(subst $1,,$2)$(subst $2,,$1)x),$(false),$(true))
-
-# ----------------------------------------------------------------------------
-# Function: sne
-# Arguments: 1: A string to compare against...
-# 2: ...this string
-# Returns: Returns $(true) if the two strings are not the same
-# ----------------------------------------------------------------------------
-sne = $(__gmsl_tr2)$(call not,$(call seq,$1,$2))
-
-# ----------------------------------------------------------------------------
-# Function: split
-# Arguments: 1: The character to split on
-# 2: A string to split
-# Returns: Splits a string into a list separated by spaces at the split
-# character in the first argument
-# ----------------------------------------------------------------------------
-split = $(__gmsl_tr2)$(strip $(subst $1, ,$2))
-
-# ----------------------------------------------------------------------------
-# Function: merge
-# Arguments: 1: The character to put between fields
-# 2: A list to merge into a string
-# Returns: Merges a list into a single string, list elements are separated
-# by the character in the first argument
-# ----------------------------------------------------------------------------
-merge = $(__gmsl_tr2)$(strip $(if $2, \
- $(if $(call seq,1,$(words $2)), \
- $2,$(call first,$2)$1$(call merge,$1,$(call rest,$2)))))
-
-ifdef __gmsl_have_eval
-# ----------------------------------------------------------------------------
-# Function: tr
-# Arguments: 1: The list of characters to translate from
-# 2: The list of characters to translate to
-# 3: The text to translate
-# Returns: Returns the text after translating characters
-# ----------------------------------------------------------------------------
-tr = $(strip $(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3) \
- $(eval __gmsl_t := $3) \
- $(foreach c, \
- $(join $(addsuffix :,$1),$2), \
- $(eval __gmsl_t := \
- $(subst $(word 1,$(subst :, ,$c)),$(word 2,$(subst :, ,$c)), \
- $(__gmsl_t))))$(__gmsl_t))
-
-# Common character classes for use with the tr function. Each of
-# these is actually a variable declaration and must be wrapped with
-# $() or ${} to be used.
-
-[A-Z] := A B C D E F G H I J K L M N O P Q R S T U V W X Y Z #
-[a-z] := a b c d e f g h i j k l m n o p q r s t u v w x y z #
-[0-9] := 0 1 2 3 4 5 6 7 8 9 #
-[A-F] := A B C D E F #
-
-# ----------------------------------------------------------------------------
-# Function: uc
-# Arguments: 1: Text to upper case
-# Returns: Returns the text in upper case
-# ----------------------------------------------------------------------------
-uc = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call tr,$([a-z]),$([A-Z]),$1)
-
-# ----------------------------------------------------------------------------
-# Function: lc
-# Arguments: 1: Text to lower case
-# Returns: Returns the text in lower case
-# ----------------------------------------------------------------------------
-lc = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call tr,$([A-Z]),$([a-z]),$1)
-
-# ----------------------------------------------------------------------------
-# Function: strlen
-# Arguments: 1: A string
-# Returns: Returns the length of the string
-# ----------------------------------------------------------------------------
-__gmsl_characters := A B C D E F G H I J K L M N O P Q R S T U V W X Y Z
-__gmsl_characters += a b c d e f g h i j k l m n o p q r s t u v w x y z
-__gmsl_characters += 0 1 2 3 4 5 6 7 8 9
-__gmsl_characters += ` ~ ! @ \# $$ % ^ & * ( ) - _ = +
-__gmsl_characters += { } [ ] \ : ; ' " < > , . / ? |
-
-# Aside: if you read the above you might think that the lower-case
-# letter x is missing, and that that's an error. It is missing, but
-# it's not an error. __gmsl_characters is used by the strlen
-# function. strlen works by transforming every character and space
-# into the letter x and then counting the x's. Since there's no need
-# to transform x into x I omitted it.
-
-# This results in __gmsl_space containing just a space
-
-__gmsl_space :=
-__gmsl_space +=
-
-strlen = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(strip $(eval __temp := $(subst $(__gmsl_space),x,$1))$(foreach a,$(__gmsl_characters),$(eval __temp := $$(subst $$a,x,$(__temp))))$(eval __temp := $(subst x,x ,$(__temp)))$(words $(__temp)))
-
-# This results in __gmsl_newline containing just a newline
-
-define __gmsl_newline
-
-
-endef
-
-# This results in __gmsl_tab containing a tab
-
-__gmsl_tab := #
-
-# ----------------------------------------------------------------------------
-# Function: substr
-# Arguments: 1: A string
-# 2: Start position (first character is 1)
-# 3: End position (inclusive)
-# Returns: A substring.
-# Note: The string in $1 must not contain a §
-# ----------------------------------------------------------------------------
-
-substr = $(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3)$(strip $(eval __temp := $$(subst $$(__gmsl_space),§ ,$$1))$(foreach a,$(__gmsl_characters),$(eval __temp := $$(subst $$a,$$a$$(__gmsl_space),$(__temp))))$(eval __temp := $(wordlist $2,$3,$(__temp))))$(subst §,$(__gmsl_space),$(subst $(__gmsl_space),,$(__temp)))
-
-endif # __gmsl_have_eval
-
-# ###########################################################################
-# SET MANIPULATION FUNCTIONS
-# ###########################################################################
-
-# Sets are represented by sorted, deduplicated lists. To create a set
-# from a list use set_create, or start with the empty_set and
-# set_insert individual elements
-
-# This is the empty set
-empty_set :=
-
-# ----------------------------------------------------------------------------
-# Function: set_create
-# Arguments: 1: A list of set elements
-# Returns: Returns the newly created set
-# ----------------------------------------------------------------------------
-set_create = $(__gmsl_tr1)$(sort $1)
-
-# ----------------------------------------------------------------------------
-# Function: set_insert
-# Arguments: 1: A single element to add to a set
-# 2: A set
-# Returns: Returns the set with the element added
-# ----------------------------------------------------------------------------
-set_insert = $(__gmsl_tr2)$(sort $1 $2)
-
-# ----------------------------------------------------------------------------
-# Function: set_remove
-# Arguments: 1: A single element to remove from a set
-# 2: A set
-# Returns: Returns the set with the element removed
-# ----------------------------------------------------------------------------
-set_remove = $(__gmsl_tr2)$(filter-out $1,$2)
-
-# ----------------------------------------------------------------------------
-# Function: set_is_member
-# Arguments: 1: A single element
-# 2: A set
-# Returns: Returns $(true) if the element is in the set
-# ----------------------------------------------------------------------------
-set_is_member = $(__gmsl_tr2)$(if $(filter $1,$2),$(true),$(false))
-
-# ----------------------------------------------------------------------------
-# Function: set_union
-# Arguments: 1: A set
-# 2: Another set
-# Returns: Returns the union of the two sets
-# ----------------------------------------------------------------------------
-set_union = $(__gmsl_tr2)$(sort $1 $2)
-
-# ----------------------------------------------------------------------------
-# Function: set_intersection
-# Arguments: 1: A set
-# 2: Another set
-# Returns: Returns the intersection of the two sets
-# ----------------------------------------------------------------------------
-set_intersection = $(__gmsl_tr2)$(filter $1,$2)
-
-# ----------------------------------------------------------------------------
-# Function: set_is_subset
-# Arguments: 1: A set
-# 2: Another set
-# Returns: Returns $(true) if the first set is a subset of the second
-# ----------------------------------------------------------------------------
-set_is_subset = $(__gmsl_tr2)$(call set_equal,$(call set_intersection,$1,$2),$1)
-
-# ----------------------------------------------------------------------------
-# Function: set_equal
-# Arguments: 1: A set
-# 2: Another set
-# Returns: Returns $(true) if the two sets are identical
-# ----------------------------------------------------------------------------
-set_equal = $(__gmsl_tr2)$(call seq,$1,$2)
-
-# ###########################################################################
-# ARITHMETIC LIBRARY
-# ###########################################################################
-
-# Integers a represented by lists with the equivalent number of x's.
-# For example the number 4 is x x x x. The maximum integer that the
-# library can handle as _input_ is __gmsl_input_int which is defined
-# here as 65536
-
-__gmsl_sixteen := x x x x x x x x x x x x x x x x
-__gmsl_input_int := $(foreach a,$(__gmsl_sixteen), \
- $(foreach b,$(__gmsl_sixteen), \
- $(foreach c,$(__gmsl_sixteen), \
- $(__gmsl_sixteen)))))
-
-# ----------------------------------------------------------------------------
-# Function: int_decode
-# Arguments: 1: A number of x's representation
-# Returns: Returns the integer for human consumption that is represented
-# by the string of x's
-# ----------------------------------------------------------------------------
-int_decode = $(__gmsl_tr1)$(words $1)
-
-# ----------------------------------------------------------------------------
-# Function: int_encode
-# Arguments: 1: A number in human-readable integer form
-# Returns: Returns the integer encoded as a string of x's
-# ----------------------------------------------------------------------------
-int_encode = $(__gmsl_tr1)$(wordlist 1,$1,$(__gmsl_input_int))
-
-# The arithmetic library functions come in two forms: one form of each
-# function takes integers as arguments and the other form takes the
-# encoded form (x's created by a call to int_encode). For example,
-# there are two plus functions:
-#
-# plus Called with integer arguments and returns an integer
-# int_plus Called with encoded arguments and returns an encoded result
-#
-# plus will be slower than int_plus because its arguments and result
-# have to be translated between the x's format and integers. If doing
-# a complex calculation use the int_* forms with a single encoding of
-# inputs and single decoding of the output. For simple calculations
-# the direct forms can be used.
-
-# Helper function used to wrap an int_* function into a function that
-# takes a pair of integers, perhaps a function and returns an integer
-# result
-__gmsl_int_wrap = $(call int_decode,$(call $1,$(call int_encode,$2),$(call int_encode,$3)))
-__gmsl_int_wrap1 = $(call int_decode,$(call $1,$(call int_encode,$2)))
-__gmsl_int_wrap2 = $(call $1,$(call int_encode,$2),$(call int_encode,$3))
-
-# ----------------------------------------------------------------------------
-# Function: int_plus
-# Arguments: 1: A number in x's representation
-# 2: Another number in x's represntation
-# Returns: Returns the sum of the two numbers in x's representation
-# ----------------------------------------------------------------------------
-int_plus = $(strip $(__gmsl_tr2)$1 $2)
-
-# ----------------------------------------------------------------------------
-# Function: plus (wrapped version of int_plus)
-# Arguments: 1: An integer
-# 2: Another integer
-# Returns: Returns the sum of the two integers
-# ----------------------------------------------------------------------------
-plus = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_plus,$1,$2)
-
-# ----------------------------------------------------------------------------
-# Function: int_subtract
-# Arguments: 1: A number in x's representation
-# 2: Another number in x's represntation
-# Returns: Returns the difference of the two numbers in x's representation,
-# or outputs an error on a numeric underflow
-# ----------------------------------------------------------------------------
-int_subtract = $(strip $(__gmsl_tr2)$(if $(call int_gte,$1,$2), \
- $(filter-out xx,$(join $1,$2)), \
- $(call __gmsl_warning,Subtraction underflow)))
-
-# ----------------------------------------------------------------------------
-# Function: subtract (wrapped version of int_subtract)
-# Arguments: 1: An integer
-# 2: Another integer
-# Returns: Returns the difference of the two integers,
-# or outputs an error on a numeric underflow
-# ----------------------------------------------------------------------------
-subtract = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_subtract,$1,$2)
-
-# ----------------------------------------------------------------------------
-# Function: int_multiply
-# Arguments: 1: A number in x's representation
-# 2: Another number in x's represntation
-# Returns: Returns the product of the two numbers in x's representation
-# ----------------------------------------------------------------------------
-int_multiply = $(strip $(__gmsl_tr2)$(foreach a,$1,$2))
-
-# ----------------------------------------------------------------------------
-# Function: multiply (wrapped version of int_multiply)
-# Arguments: 1: An integer
-# 2: Another integer
-# Returns: Returns the product of the two integers
-# ----------------------------------------------------------------------------
-multiply = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_multiply,$1,$2)
-
-# ----------------------------------------------------------------------------
-# Function: int_divide
-# Arguments: 1: A number in x's representation
-# 2: Another number in x's represntation
-# Returns: Returns the result of integer division of argument 1 divided
-# by argument 2 in x's representation
-# ----------------------------------------------------------------------------
-int_divide = $(__gmsl_tr2)$(strip $(if $2, \
- $(if $(call int_gte,$1,$2), \
- x $(call int_divide,$(call int_subtract,$1,$2),$2),), \
- $(call __gmsl_error,Division by zero)))
-
-# ----------------------------------------------------------------------------
-# Function: divide (wrapped version of int_divide)
-# Arguments: 1: An integer
-# 2: Another integer
-# Returns: Returns the integer division of the first argument by the second
-# ----------------------------------------------------------------------------
-divide = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_divide,$1,$2)
-
-# ----------------------------------------------------------------------------
-# Function: int_max, int_min
-# Arguments: 1: A number in x's representation
-# 2: Another number in x's represntation
-# Returns: Returns the maximum or minimum of its arguments in x's
-# representation
-# ----------------------------------------------------------------------------
-int_max = $(__gmsl_tr2)$(subst xx,x,$(join $1,$2))
-int_min = $(__gmsl_tr2)$(subst xx,x,$(filter xx,$(join $1,$2)))
-
-# ----------------------------------------------------------------------------
-# Function: max, min
-# Arguments: 1: An integer
-# 2: Another integer
-# Returns: Returns the maximum or minimum of its integer arguments
-# ----------------------------------------------------------------------------
-max = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_max,$1,$2)
-min = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_min,$1,$2)
-
-# ----------------------------------------------------------------------------
-# Function: int_gt, int_gte, int_lt, int_lte, int_eq, int_ne
-# Arguments: Two x's representation numbers to be compared
-# Returns: $(true) or $(false)
-#
-# int_gt First argument greater than second argument
-# int_gte First argument greater than or equal to second argument
-# int_lt First argument less than second argument
-# int_lte First argument less than or equal to second argument
-# int_eq First argument is numerically equal to the second argument
-# int_ne First argument is not numerically equal to the second argument
-# ----------------------------------------------------------------------------
-int_gt = $(__gmsl_tr2)$(call __gmsl_make_bool, \
- $(filter-out $(words $2), \
- $(words $(call int_max,$1,$2))))
-int_gte = $(__gmsl_tr2)$(call __gmsl_make_bool, \
- $(call int_gt,$1,$2)$(call int_eq,$1,$2))
-int_lt = $(__gmsl_tr2)$(call __gmsl_make_bool, \
- $(filter-out $(words $1), \
- $(words $(call int_max,$1,$2))))
-int_lte = $(__gmsl_tr2)$(call __gmsl_make_bool, \
- $(call int_lt,$1,$2)$(call int_eq,$1,$2))
-int_eq = $(__gmsl_tr2)$(call __gmsl_make_bool, \
- $(filter $(words $1),$(words $2)))
-int_ne = $(__gmsl_tr2)$(call __gmsl_make_bool, \
- $(filter-out $(words $1),$(words $2)))
-
-# ----------------------------------------------------------------------------
-# Function: gt, gte, lt, lte, eq, ne
-# Arguments: Two integers to be compared
-# Returns: $(true) or $(false)
-#
-# gt First argument greater than second argument
-# gte First argument greater than or equal to second argument
-# lt First argument less than second argument
-# lte First argument less than or equal to second argument
-# eq First argument is numerically equal to the second argument
-# ne First argument is not numerically equal to the second argument
-# ----------------------------------------------------------------------------
-gt = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_gt,$1,$2)
-gte = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_gte,$1,$2)
-lt = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_lt,$1,$2)
-lte = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_lte,$1,$2)
-eq = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_eq,$1,$2)
-ne = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_ne,$1,$2)
-
-# increment adds 1 to its argument, decrement subtracts 1. Note that
-# decrement does not range check and hence will not underflow, but
-# will incorrectly say that 0 - 1 = 0
-
-# ----------------------------------------------------------------------------
-# Function: int_inc
-# Arguments: 1: A number in x's representation
-# Returns: The number incremented by 1 in x's representation
-# ----------------------------------------------------------------------------
-int_inc = $(strip $(__gmsl_tr1)$1 x)
-
-# ----------------------------------------------------------------------------
-# Function: inc
-# Arguments: 1: An integer
-# Returns: The argument incremented by 1
-# ----------------------------------------------------------------------------
-inc = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_inc,$1)
-
-# ----------------------------------------------------------------------------
-# Function: int_dec
-# Arguments: 1: A number in x's representation
-# Returns: The number decremented by 1 in x's representation
-# ----------------------------------------------------------------------------
-int_dec = $(__gmsl_tr1)$(strip $(if $(call sne,0,$(words $1)), \
- $(wordlist 2,$(words $1),$1), \
- $(call __gmsl_warning,Decrement underflow)))
-
-# ----------------------------------------------------------------------------
-# Function: dec
-# Arguments: 1: An integer
-# Returns: The argument decremented by 1
-# ----------------------------------------------------------------------------
-dec = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_dec,$1)
-
-# double doubles its argument, and halve halves it
-
-# ----------------------------------------------------------------------------
-# Function: int_double
-# Arguments: 1: A number in x's representation
-# Returns: The number doubled (i.e. * 2) and returned in x's representation
-# ----------------------------------------------------------------------------
-int_double = $(strip $(__gmsl_tr1)$1 $1)
-
-# ----------------------------------------------------------------------------
-# Function: double
-# Arguments: 1: An integer
-# Returns: The integer times 2
-# ----------------------------------------------------------------------------
-double = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_double,$1)
-
-# ----------------------------------------------------------------------------
-# Function: int_halve
-# Arguments: 1: A number in x's representation
-# Returns: The number halved (i.e. / 2) and returned in x's representation
-# ----------------------------------------------------------------------------
-int_halve = $(__gmsl_tr1)$(strip $(subst xx,x,$(filter-out xy x y, \
- $(join $1,$(foreach a,$1,y x)))))
-
-# ----------------------------------------------------------------------------
-# Function: halve
-# Arguments: 1: An integer
-# Returns: The integer divided by 2
-# ----------------------------------------------------------------------------
-halve = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_halve,$1)
-
-ifdef __gmsl_have_eval
-# ###########################################################################
-# ASSOCIATIVE ARRAYS
-# ###########################################################################
-
-# ----------------------------------------------------------------------------
-# Function: set
-# Arguments: 1: Name of associative array
-# 2: The key value to associate
-# 3: The value associated with the key
-# Returns: None
-# ----------------------------------------------------------------------------
-set = $(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3)$(eval __gmsl_aa_$1_$2 = $3)
-
-# ----------------------------------------------------------------------------
-# Function: get
-# Arguments: 1: Name of associative array
-# 2: The key to retrieve
-# Returns: The value stored in the array for that key
-# ----------------------------------------------------------------------------
-get = $(strip $(__gmsl_tr2)$(call assert_no_dollar,$0,$1$2)$(if $(filter-out undefined,$(origin __gmsl_aa_$1_$2)), \
- $(__gmsl_aa_$1_$2)))
-
-# ----------------------------------------------------------------------------
-# Function: keys
-# Arguments: 1: Name of associative array
-# Returns: Returns a list of all defined keys in the array
-# ----------------------------------------------------------------------------
-keys = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(sort $(patsubst __gmsl_aa_$1_%,%, \
- $(filter __gmsl_aa_$1_%,$(.VARIABLES))))
-
-# ----------------------------------------------------------------------------
-# Function: defined
-# Arguments: 1: Name of associative array
-# 2: The key to test
-# Returns: Returns true if the key is defined (i.e. not empty)
-# ----------------------------------------------------------------------------
-defined = $(__gmsl_tr2)$(call assert_no_dollar,$0,$1$2)$(call sne,$(call get,$1,$2),)
-
-endif # __gmsl_have_eval
-
-ifdef __gmsl_have_eval
-# ###########################################################################
-# NAMED STACKS
-# ###########################################################################
-
-# ----------------------------------------------------------------------------
-# Function: push
-# Arguments: 1: Name of stack
-# 2: Value to push onto the top of the stack (must not contain
-# a space)
-# Returns: None
-# ----------------------------------------------------------------------------
-push = $(__gmsl_tr2)$(call assert_no_dollar,$0,$1$2)$(eval __gmsl_stack_$1 := $2 $(if $(filter-out undefined,\
- $(origin __gmsl_stack_$1)),$(__gmsl_stack_$1)))
-
-# ----------------------------------------------------------------------------
-# Function: pop
-# Arguments: 1: Name of stack
-# Returns: Top element from the stack after removing it
-# ----------------------------------------------------------------------------
-pop = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(strip $(if $(filter-out undefined,$(origin __gmsl_stack_$1)), \
- $(call first,$(__gmsl_stack_$1)) \
- $(eval __gmsl_stack_$1 := $(call rest,$(__gmsl_stack_$1)))))
-
-# ----------------------------------------------------------------------------
-# Function: peek
-# Arguments: 1: Name of stack
-# Returns: Top element from the stack without removing it
-# ----------------------------------------------------------------------------
-peek = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call first,$(__gmsl_stack_$1))
-
-# ----------------------------------------------------------------------------
-# Function: depth
-# Arguments: 1: Name of stack
-# Returns: Number of items on the stack
-# ----------------------------------------------------------------------------
-depth = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(words $(__gmsl_stack_$1))
-
-endif # __gmsl_have_eval
-
-# ###########################################################################
-# DEBUGGING FACILITIES
-# ###########################################################################
-
-# ----------------------------------------------------------------------------
-# Target: gmsl-print-%
-# Arguments: The % should be replaced by the name of a variable that you
-# wish to print out.
-# Action: Echos the name of the variable that matches the % and its value.
-# For example, 'make gmsl-print-SHELL' will output the value of
-# the SHELL variable
-# ----------------------------------------------------------------------------
-gmsl-print-%: ; @echo $* = $($*)
-
-# ----------------------------------------------------------------------------
-# Function: assert
-# Arguments: 1: A boolean that must be true or the assertion will fail
-# 2: The message to print with the assertion
-# Returns: None
-# ----------------------------------------------------------------------------
-assert = $(if $1,,$(call __gmsl_error,Assertion failure: $2))
-
-# ----------------------------------------------------------------------------
-# Function: assert_exists
-# Arguments: 1: Name of file that must exist, if it is missing an assertion
-# will be generated
-# Returns: None
-# ----------------------------------------------------------------------------
-assert_exists = $(call assert,$(wildcard $1),file '$1' missing)
-
-# ----------------------------------------------------------------------------
-# Function: assert_no_dollar
-# Arguments: 1: Name of a function being executd
-# 2: Arguments to check
-# Returns: None
-# ----------------------------------------------------------------------------
-assert_no_dollar = $(call assert,$(call not,$(findstring $$,$2)),$1 called with a dollar sign in argument)
+++ /dev/null
-# ----------------------------------------------------------------------------
-#
-# GNU Make Standard Library (GMSL)
-#
-# A library of functions to be used with GNU Make's $(call) that
-# provides functionality not available in standard GNU Make.
-#
-# Copyright (c) 2005-2008 John Graham-Cumming
-#
-# This file is part of GMSL
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#
-# Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-#
-# Neither the name of the John Graham-Cumming nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-#
-# ----------------------------------------------------------------------------
-
-# Determine if the library has already been included and if so don't
-# bother including it again
-
-ifndef __gmsl_included
-
-# Standard definitions for true and false. true is any non-empty
-# string, false is an empty string. These are intended for use with
-# $(if).
-
-true := T
-false :=
-
-# ----------------------------------------------------------------------------
-# Function: not
-# Arguments: 1: A boolean value
-# Returns: Returns the opposite of the arg. (true -> false, false -> true)
-# ----------------------------------------------------------------------------
-not = $(if $1,$(false),$(true))
-
-# Prevent reinclusion of the library
-
-__gmsl_included := $(true)
-
-# Try to determine where this file is located. If the caller did
-# include /foo/gmsl then extract the /foo/ so that __gmsl gets
-# included transparently
-
-ifneq ($(MAKEFILE_LIST),)
-__gmsl_root := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST))
-
-# If there are any spaces in the path in __gmsl_root then give up
-
-ifeq (1,$(words $(__gmsl_root)))
-__gmsl_root := $(patsubst %gmsl,%,$(__gmsl_root))
-else
-__gmsl_root :=
-endif
-
-include $(__gmsl_root)__gmsl
-
-else
-
-include __gmsl
-
-endif
-
-endif # __gmsl_included
-
+++ /dev/null
-\f
-\18splash.lss
-
-
-
- - To install IPFire on your system, type \ f04install\ f07 and
- press the \ f04<ENTER>\ f07 key.
- - Use the function keys listed below for more information.
-
-\ f0f[F1-Main] [F2-Options]\ f07
+++ /dev/null
-
-DEFAULT vesamenu.c32
-#prompt 1
-TIMEOUT 100
-
-DISPLAY boot.msg
-F1 boot.msg
-F2 options.msg
-
-MENU BACKGROUND splash.jpg
-MENU TITLE Welcome to @NAME@ @VERSION@!
-MENU COLOR border 0 #ffffffff #00000000
-MENU COLOR sel 7 #ffffffff #ff000000
-MENU COLOR title 0 #ffffffff #00000000
-MENU COLOR tabmsg 0 #ffffffff #00000000
-MENU COLOR unsel 0 #ffffffff #00000000
-MENU COLOR hotsel 0 #ff000000 #ffffffff
-MENU COLOR hotkey 7 #ffffffff #ff000000
-MENU hidden
-MENU hiddenrow 5
-
-LABEL install
- MENU label ^Install a new @NAME@ system
- MENU default
- KERNEL vmlinuz0
- APPEND initrd=initrd0 root=live:CDLABEL=@NAME@_@VERSION@ rootfstype=auto liveimg mode=install quiet
-label rescue
- MENU label ^Rescue installed @NAME@ system
- KERNEL vmlinuz0
- APPEND initrd=initrd0 root=live:CDLABEL=@NAME@_@VERSION@ rootfstype=auto liveimg mode=rescue quiet
-label local
- MENU label Boot from ^local drive
- LOCALBOOT 0xffff
-label memtest
- MENU label ^Memory test
- KERNEL memtest
- APPEND -
+++ /dev/null
-\ f00
-\f
-
- \ f0fInstaller Boot Options\ f07
-
- - To run a rescue system, type: \ f04rescue <ENTER>\ f07.
-
- - To test the memory in your system type: \ f04memtest <ENTER>\ f07.
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-\ f0f[F1-Main] [F2-Options]\ f07
+++ /dev/null
-
-DEFAULT @SNAME@/vesamenu.c32
-#prompt 1
-TIMEOUT 100
-
-DISPLAY @SNAME@/boot.msg
-F1 @SNAME@/boot.msg
-F2 @SNAME@/options.msg
-
-MENU BACKGROUND @SNAME@/splash.jpg
-MENU TITLE Welcome to @NAME@ @VERSION@!
-MENU COLOR border 0 #ffffffff #00000000
-MENU COLOR sel 7 #ffffffff #ff000000
-MENU COLOR title 0 #ffffffff #00000000
-MENU COLOR tabmsg 0 #ffffffff #00000000
-MENU COLOR unsel 0 #ffffffff #00000000
-MENU COLOR hotsel 0 #ff000000 #ffffffff
-MENU COLOR hotkey 7 #ffffffff #ff000000
-MENU hidden
-MENU hiddenrow 5
-
-LABEL install
- MENU label ^Install a new @NAME@ system
- MENU default
- KERNEL @SNAME@/@SNAME@0
- APPEND initrd=@SNAME@/initrd0 root=http://pxe.ipfire.org/@IMAGENAME@.iso rootfstype=iso9660 vga=791 splash mode=install quiet ro net=dhcp
-label rescue
- MENU label ^Rescue installed @NAME@ system
- KERNEL @SNAME@/@SNAME@0
- APPEND initrd=@SNAME@/initrd0 root=http://pxe.ipfire.org/@IMAGENAME@.iso rootfstype=iso9660 vga=791 splash mode=rescue quiet ro net=dhcp
+++ /dev/null
-# Begin /etc/grsec/sysctl.conf
-
-# Locking all settings - must be the last line
-kernel.grsecurity.grsec_lock = 1
-
-# End /etc/grsec/sysctl.conf
+++ /dev/null
-#
-# file system mount-point type options dump fsck
-# order
-none / auto defaults 0 0
-proc /proc proc defaults 0 0
-sysfs /sys sysfs defaults 0 0
-devpts /dev/pts devpts gid=4,mode=620 0 0
-shm /dev/shm tmpfs defaults 0 0
+++ /dev/null
-root:x:0:root
-bin:x:1:bin,daemon,root
-daemon:x:2:bin,daemon,root
-sys:x:3:bin,adm,root
-adm:x:4:adm,daemon,root
-tty:x:5:
-disk:x:6:root
-lp:x:7:daemon,lp
-mem:x:8:
-kmem:x:9:
-wheel:x:10:root
-mail:x:12:mail
-news:x:13:news
-uucp:x:14:uucp
-man:x:15:
-games:x:20:
-gopher:x:30:
-dip:x:40:
-ftp:x:50:
-lock:x:54:
-nobody:x:99:
-users:x:100:
-dbus:x:81:
-utmp:x:22:
-smmsp:x:51:
-nscd:x:28:
-floppy:x:19:
-rpc:x:32:
-sshd:x:74:
-pcap:x:77:
-utempter:x:35:
-ntp:x:38:
-audio:x:63:
+++ /dev/null
-# Begin /etc/grsec/sysctl.conf
-
-# Disable chroot caps
-kernel.grsecurity.chroot_caps = 0
-
-# Disable chroot exec logging
-kernel.grsecurity.chroot_execlog = 0
-
-# Locking all settings - must be the last line
-kernel.grsecurity.grsec_lock = 1
-
-# End /etc/grsec/sysctl.conf
+++ /dev/null
-root:x:0:0:root:/root:/bin/bash
-bin:x:1:1:bin:/bin:/sbin/nologin
-daemon:x:2:2:daemon:/sbin:/sbin/nologin
-adm:x:3:4:adm:/var/adm:/sbin/nologin
-lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
-sync:x:5:0:sync:/sbin:/bin/sync
-shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
-halt:x:7:0:halt:/sbin:/sbin/halt
-mail:x:8:12:mail:/var/spool/mail:/sbin/nologin
-news:x:9:13:news:/etc/news:
-uucp:x:10:14:uucp:/var/spool/uucp:/sbin/nologin
-operator:x:11:0:operator:/root:/sbin/nologin
-games:x:12:100:games:/usr/games:/sbin/nologin
-gopher:x:13:30:gopher:/var/gopher:/sbin/nologin
-ftp:x:14:50:FTP User:/var/ftp:/sbin/nologin
-nobody:x:99:99:Nobody:/:/sbin/nologin
-dbus:x:81:81:System message bus:/:/sbin/nologin
-smmsp:x:51:51::/var/spool/mqueue:/sbin/nologin
-nscd:x:28:28:NSCD Daemon:/:/sbin/nologin
-rpc:x:32:32:Portmapper RPC user:/:/sbin/nologin
-sshd:x:74:74:Privilege-separated SSH:/var/empty/sshd:/sbin/nologin
-ntp:x:38:38::/etc/ntp:/sbin/nologin
+++ /dev/null
-# Just a file to skip the checkfs procdure on the live system
+++ /dev/null
-# You can use this to rotate the /var/log/radius/* files, simply copy
-# it to /etc/logrotate.d/radiusd
-
-# There are different detail-rotating strategies you can use. One is
-# to write to a single detail file per IP and use the rotate config
-# below. Another is to write to a daily detail file per IP with:
-# detailfile = ${radacctdir}/%{Client-IP-Address}/%Y%m%d-detail
-# (or similar) in radiusd.conf, without rotation. If you go with the
-# second technique, you will need another cron job that removes old
-# detail files. You do not need to comment out the below for method #2.
-/var/log/radius/radacct/*/detail {
- monthly
- rotate 4
- nocreate
- missingok
- compress
-}
-
-/var/log/radius/checkrad.log {
- monthly
- rotate 4
- create
- missingok
- compress
-}
-
-/var/log/radius/radius.log {
- monthly
- rotate 4
- create
- missingok
- compress
-}
-
-/var/log/radius/radutmp {
- monthly
- rotate 4
- create
- compress
- missingok
-}
-
-/var/log/radius/radwtmp {
- monthly
- rotate 4
- create
- compress
- missingok
-}
-/var/log/radius/sqltrace.sql {
- monthly
- rotate 4
- create
- compress
- missingok
-}
+++ /dev/null
-# Logrotate file for ppp RPM
-
-/var/log/ppp/connect-errors {
- missingok
- compress
- notifempty
- daily
- rotate 5
- create 0600 root root
-}
+++ /dev/null
-#!/bin/bash
-
-cat <<EOF
-
- You are now dropped to a chrooted shell of the package's environment.
-
- The sources have been extracted to /usr/src and maybe there are files left
- from a last (broken) build. Nothing of that content will be saved after
- you left the shell.
-
- You can leave the environment by typing "exit" or Ctrl-D.
-
-EOF
-
-# Setting nice environment
-export PS1="naoki-chroot \w> "
-
-# Change to directory the user will most likely work in
-if [ -z "${SOURCE_DIR}" ]; then
- SOURCE_DIR="/usr/src"
-fi
-cd "${SOURCE_DIR}"
-
-exec /bin/bash --login
+++ /dev/null
-functions-common
\ No newline at end of file
+++ /dev/null
-#!/bin/bash
-###############################################################################
-# #
-# IPFire.org - A linux based firewall #
-# Copyright (C) 2007, 2008, 2009 Michael Tremer & Christian Schmidt #
-# #
-# This program is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published by #
-# the Free Software Foundation, either version 3 of the License, or #
-# (at your option) any later version. #
-# #
-# This program is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with this program. If not, see <http://www.gnu.org/licenses/>. #
-# #
-###############################################################################
-
-FILES=
-ROOT=/
-
-while [ $# -gt 0 ]; do
- case "$1" in
- --root=*)
- ROOT=${1#--root=}
- ;;
- *.ipk)
- file=$1
- [ ${file:0:1} != / ] && file="$(pwd)/$file"
- if [ -e "$file" ]; then
- FILES="$FILES $file"
- else
- echo "File does not exist: $file" >&2
- #exit 1
- fi
- ;;
- esac
- shift
-done
-
-if [ "$ROOT" != "/" ]; then
- [ -d "$ROOT" ] || mkdir -p $ROOT
- cd $ROOT
- if [ "$(pwd)" != "$ROOT" ]; then
- echo "Could not change to root: $ROOT" >&2
- exit 1
- fi
-fi
-
-# Sort all packages
-FILES=$(for i in $FILES; do echo $i; done | sort -u)
-
-for file in $FILES; do
- #echo "Extracting $(basename ${file})..."
- cpio --extract --quiet -H newc --to-stdout data.img < $file | \
- tar --extract --use-compress-program=xz -C ${ROOT}
-done
+++ /dev/null
-#!/bin/bash
-
-# Simply import all files from this directory that
-# begin with functions-*.
-
-BASEDIR=$(dirname ${BASH_SOURCE[0]})
-
-for file in ${BASEDIR}/functions-*; do
- # Avoid infinite loop when importing this file again
- [ "$(basename ${file})" = "functions-common" ] && continue
-
- . ${file}
-done
-
+++ /dev/null
-#!/bin/bash
-
-# Debugging mode for these scripts
-DEBUG=0
-
-# Interpreters that should not be found by find_interpreters()
-INTERPRETERS_TO_BE_SKIPPED="/usr/bin/env"
-
-# Some path constants...
-LIBRARY_PATHS="/lib /usr/lib /libexec /usr/libexec"
-BINARY_PATHS="${LIBRARY_PATHS} /bin /sbin /usr/bin /usr/sbin"
-
-# List of directories that could probably empty and are removed automatically
-# so they won't appear in any package.
-ORPHAN_CANDIDATES="${BINARY_PATHS} /usr /usr/include /usr/share"
-for i in $(seq 0 9); do
- ORPHAN_CANDIDATES="${ORPHAN_CANDIDATES} /usr/share/man/man${i}"
-done
-ORPHAN_CANDIDATES="${ORPHAN_CANDIDATES} /usr/lib/pkgconfig"
-
+++ /dev/null
-#!/bin/bash
-
-function dir_is_empty() {
- [ "$(ls -A $@ 2>/dev/null | wc -l)" = "0" ]
-}
-
-function directory_remove_orphans() {
- local basedir=${1}
-
- log DEBUG "Removing orphans in ${basedir}"
-
- local dir
- for dir in ${ORPHAN_CANDIDATES}; do
- dir="${basedir}/${dir}"
-
- [ -d "${dir}" ] || continue
-
- if dir_is_empty ${dir}; then
- log DEBUG " Found orphaned directory: ${dir}"
- rm -rf ${dir}
- fi
- done
-}
-
+++ /dev/null
-#!/bin/bash
-
-# Check if a file is an ELF binary
-#
-function file_is_elf() {
- local file=${1}
-
- file "${file}" | grep -q "ELF"
-}
-
-# Check if a file is a script.
-# If the first line starts with #! this is sufficient.
-#
-function file_is_script() {
- local file=${1}
-
- local first_line=$(head -n1 ${file})
-
- [ "${first_line:0:2}" = "#!" ]
-}
-
-# Get the interpreter of a file.
-#
-function file_get_interpreter() {
- local file=${1}
-
- if file_is_elf ${file}; then
- _file_get_elf_interpreter ${file}
- elif file_is_script ${file}; then
- _file_get_script_interpreter ${file}
- fi
-}
-
-# Hidden function that gets the interpreter from an ELF file.
-#
-function _file_get_elf_interpreter() {
- local file=${1}
-
- readelf -l ${file} | grep "program interpreter" | \
- tr -d "]" | awk '{ print $NF }'
-}
-
-# Hidden fucntion that gets the interpreter from a script file.
-#
-function _file_get_script_interpreter() {
- local file=${1}
-
- # If the file is not executeable, no interpreter will be needed
- [ -x "${file}" ] || return
-
- local first_line=$(head -n1 ${file})
-
- first_line="${first_line:2:${#first_line}}"
-
- # Choose the first argument and strip any parameters if available
- local interpreter
- for interpreter in ${first_line}; do
- echo "${interpreter}"
- return
- done
-}
-
-# Check if a file is statically linked.
-#
-function file_is_static() {
- local file=${1}
-
- file ${file} | grep -q "statically linked"
-}
-
-# Get NEEDED from a file.
-#
-function file_get_needed() {
- local file=${1}
-
- readelf -d ${file} | grep NEEDED | \
- tr -d "[]" | awk '{ print $NF }'
-}
-
-# Get RPATH from a file.
-#
-function file_get_rpath() {
- local file=${1}
-
- readelf -d ${file} | grep RPATH | \
- tr -d "[]" | awk '{ print $NF }'
-}
-
-# Get SONAME from a file.
-#
-function file_get_soname() {
- local file=${1}
-
- local file_basename=$(basename ${file})
- if [ "${file_basename:0:3}" = "ld-" ]; then
- log DEBUG "Don't return a SONAME for linkers: ${file}"
- return
- fi
-
- readelf -d ${file} | grep SONAME | \
- tr -d "[]" | awk '{ print $NF }'
-}
-
-# Check if a file is a shared object.
-#
-function file_is_shared_object() {
- local file=${1}
-
- file ${file} | grep -q "shared object"
-}
-
-# Check if a file has the canary.
-#
-function file_has_canary() {
- local file=${1}
-
- readelf -s ${file} | grep -q "__stack_chk_fail"
-}
-
-# Check if a file has an executeable stack.
-#
-function file_has_execstack() {
- local file=${1}
-
- readelf -h ${file} | grep -qE "Type:[[:space:]]*EXEC"
-}
-
-# Check if a file has NX.
-#
-function file_has_nx() {
- local file=${1}
-
- readelf -l ${file} | grep "GNU_STACK" | grep -q "RWE"
- [ $? != 0 ]
-}
-
-# Check if a file is partly RELRO.
-#
-function file_is_relro_partly() {
- local file=${1}
-
- readelf -l ${file} | grep -q "GNU_RELRO"
-}
-
-# Check if a file is fully RELRO.
-#
-function file_is_relro_full() {
- local file=${1}
-
- if file_is_relro_partly ${file}; then
- readelf -d ${file} | grep -q "BIND_NOW"
- return $?
- fi
- return 1
-}
-
-# Find all ELF files.
-#
-function find_elf_files() {
- local dir
- local dirs
- local prefix
-
- while [ $# -gt 0 ]; do
- case "${1}" in
- --prefix=*)
- prefix="${1#--prefix=}/"
- ;;
- *)
- dirs="${dirs} ${1}"
- ;;
- esac
- shift
- done
-
- local file
- local files
-
- for dir in ${dirs}; do
- dir="${prefix}${dir}"
- for file in $(find ${dir} -type f 2>/dev/null); do
- if file_is_elf ${file} && file_is_shared_object ${file} && ! file_is_static ${file}; then
- files="${files} ${file}"
- fi
- done
- done
-
- echo ${files}
-}
-
-function filter_startfiles() {
- local file=${1}
-
- grep -qE "crt[1in]\.o$" <<<${file}
-}
+++ /dev/null
-#!/bin/bash
-
-function listsort() {
- local item
- for item in $@; do
- echo "${item}"
- done | sort -u | tr "\n" " "
-}
-
-function listmatch() {
- local arg=${1}
- shift
-
- local item
- for item in $@; do
- if [ "${arg}" = "${item}" ]; then
- return 0
- fi
- done
- return 1
-}
-
-function sort_by_length() {
- local c
- local i
- for i in $@; do
- echo "$(wc -c <<<${i}) ${i}"
- done | sort -n -r | while read c i; do
- echo "${i}"
- done
-}
+++ /dev/null
-#!/bin/bash
-
-function log() {
- local level=${1}
- shift
-
- if [ "${level}" = "DEBUG" ] && [ "${DEBUG}" != "1" ]; then
- return
- fi
-
- printf " %1s | %s\n" "${level:0:1}" "$@" >&2
-}
+++ /dev/null
-#!/bin/bash
-
-# A function that finds needed libraries and interpreters.
-#
-function find_requires() {
- local dir
- local dirs=$@
-
- # Find interpreters of all files in the dirs and skip those we provide
- # ourself.
- local interpreter
- local interpreters
- for interpreter in $(find_interpreters ${dirs}); do
- local found=0
- for dir in ${dirs}; do
- if [ -e "${dir}/${interpreter}" ]; then
- found=1
- break
- fi
- done
-
- [ "${found}" = "0" ] && interpreters="${interpreters} ${interpreter}"
- done
-
- # Find NEEDED libs and add them to a list if they are not provided by any
- # other file in dirs.
- local neededs
- for file in $(find_elf_files ${dirs}); do
- for needed in $(file_get_needed ${file}); do
- neededs="${neededs} ${needed}"
- done
- done
-
- # Find all symlink destinations
- local links=$(find_symlink_destinations ${dirs})
-
- # Others
- local others=$(find_python_requires ${dirs})
- others="${others} $(find_weak_symbols_requires ${dirs})"
- others="${others} $(find_perl_requires ${dirs})"
-
- # Get provides, because packages should not depend on features they provide
- # by themselves.
- local provides=$(find_provides ${dirs})
-
- # Return a sorted and unique(!) list
- local require
- local requires
- for require in $(listsort ${PKG_DEPS} ${interpreters} ${neededs} ${links} ${others}); do
- [ "${require:0:3}" = "ld-" ] && continue
- listmatch ${require} ${provides} || requires="${requires} ${require}"
- done
-
- echo ${requires}
-}
-
-function find_provides() {
- local dirs=$@
-
- local file
- local sonames
- for file in $(find_elf_files ${dirs}); do
- sonames="${sonames} $(file_get_soname ${file})"
- done
- sonames=$(listsort ${sonames})
-
- # Others
- local others=$(find_python_provides ${dirs})
- others="${others} $(find_weak_symbols_provides ${dirs})"
- others="${others} $(find_perl_provides ${dirs})"
-
- listsort ${PKG_PROVIDES} ${sonames} ${others}
-}
-
-function find_interpreters() {
- local dirs=$@
-
- log DEBUG "Searching for interpreters in ${dirs}"
-
- local file
- local interpreter
- local interpreters
- for file in $(find ${dirs} -type f 2>/dev/null); do
- # Get interpreter information from file.
- interpreter=$(file_get_interpreter ${file})
-
- # Skip the file silently if the result was empty.
- [ -z "${interpreter}" ] && continue
-
- # Skip invalid interpreters that don't start with a slash.
- if [ "${interpreter:0:1}" != "/" ]; then
- log WARNING "Skipping invalid interpreter \"${interpreter}\" from \"${file}\"."
- continue
- fi
-
- if ! listmatch ${interpreter} ${INTERPRETERS_TO_BE_SKIPPED}; then
- interpreters="${interpreters} ${interpreter}"
- fi
- done
-
- interpreters=$(listsort ${interpreters})
-
- log DEBUG "find_interpreters ${dirs}: ${interpreters}"
-
- echo "${interpreters}"
-}
-
-# Find the destinations of all symlinks and adds a dependency for that.
-#
-function find_symlink_destinations() {
- local dir=$@
-
- local link
- local links
- for link in $(find ${dir} -type l 2>/dev/null); do
- link="$(readlink -m ${link})"
- [ -e "${link}" ] && continue
-
- link="${link#${dir}}"
- links="${links} ${link}"
- done
-
- echo ${links}
-}
-
-function find_python_provides() {
- local dir=${1}
-
- local file
- for file in $(find ${dir}/usr/bin/python* 2>/dev/null); do
- file=$(basename ${file})
- file=${file#python}
-
- if [ -n "${file}" ]; then
- echo "python-api=${file}"
- fi
- done
-}
-
-function find_python_requires() {
- local dir=${1}
-
- local file
- for file in $(find ${dir}/usr/lib -maxdepth 1 2>/dev/null); do
- file=$(basename ${file})
-
- if [ "${file:0:6}" = "python" ]; then
- file=${file#python}
-
- if [ -n "${file}" ]; then
- echo "python-api=${file}"
- fi
- fi
- done
-}
-
-function find_perl_files() {
- local extension
- for extension in pm pl; do
- find $@ -name "*.${extension}" 2>/dev/null
- done
-}
-
-function find_perl_provides() {
- [ -x "/usr/bin/perl" ] || return 0
- ${BASEDIR}/perl.prov $(find_perl_files $@) | sort -u
-}
-
-function find_perl_requires() {
- [ -x "/usr/bin/perl" ] || return 0
- ${BASEDIR}/perl.req $(find_perl_files $@) | sort -u
-}
-
-function find_weak_symbols_provides() {
- local dirs=$@
-
- local file
- local soname
- local symbol
- for file in $(find_elf_files ${dirs}); do
- soname=$(file_get_soname ${file})
- [ -z "${soname}" ] && continue
-
- for symbol in $(objdump -p ${file} | grep -E "^[0-9]+" | awk '{ print $4 }'); do
- [ "${symbol}" = "${soname}" ] && continue
- [ "${symbol}" = "GLIBC_PRIVATE" ] && continue
-
- echo "${soname}(${symbol})"
- done
- done | sort -u
-}
-
-function find_weak_symbols_requires() {
- local dirs=$@
-
- local file
- for file in $(find_elf_files ${dirs}); do
- objdump -p ${file} | awk 'BEGIN { START=0; LIBNAME=""; }
- /^$/ { START=0; }
- /^Dynamic Section:$/ { START=1; }
- (START==1) && /NEEDED/ {
- print $2;
- }
- (START==2) && /^[A-Za-z]/ { START=3; }
- /^Version References:$/ { START=2; }
- (START==2) && /required from/ {
- sub(/:/, "", $3);
- LIBNAME=$3;
- }
- (START==2) && (LIBNAME!="") && ($4!="") && (($4~/^GLIBC_*/) || ($4~/^GCC_*/)) {
- print LIBNAME "(" $4 ")";
- }'
- done | grep -v "GLIBC_PRIVATE" | sort -u
-}
+++ /dev/null
-#!/bin/bash -x
-
-BOOTLOADER_DIR=/usr/src/src/bootloader
-IMAGES_DIR=/usr/src/images
-
-# Size of the ext filesystem in gigabytes
-FSSIZE=2
-FSTYPE=ext4
-
-ISO_FILENAME=${DISTRO_SNAME}-${DISTRO_VERSION}.${PKG_ARCH}.iso
-
-. $(dirname ${0})/common-functions
-
-function find_latest_kernel_release() {
- local i
- for i in /lib/modules/*; do
- i=$(basename ${i})
- done
-
- echo "${i}"
-}
-
-KERNEL_RELEASE=$(find_latest_kernel_release)
-
-function installer_image() {
- local target=${1}
- local dir=${2}
-
- # Create target directory if not existant
- mkdir -p ${target} 2>/dev/null
-
- # Create installer images:
- # (1) osmin.img (minimal cow image)
- # (2) squashfs.img (installer)
-
- local i
- for i in dev proc sys lib/modules; do
- mkdir -p ${dir}/${i} 2>/dev/null
- done
-
- # Copy kernel modules
- cp -vfr /lib/modules/${KERNEL_RELEASE} ${dir}/lib/modules/
-
- # Copy overlay
- cp -vfr /usr/src/src/install/* ${dir}
-
- _installer_image_ext3fs \
- ${target}/$(basename ${target})/ext3fs.img /installer
- _installer_image_osmin \
- ${target}/osmin.img \
- ${target}/$(basename ${target})/ext3fs.img
-
- cd ${target}
- mksquashfs * ${target}/squashfs.img -no-progress
- rm -fr $(basename ${target})
-}
-
-function _installer_image_osmin() {
- local file=${1}
- local image=${2}
-
- local tmp=$(mktemp -d)
-
- local image_loop=$(losetup -f)
-
- # Setting up loop for image
- losetup ${image_loop} ${image}
-
- # Setting up loop for an empty 64byte sparse file
- sparse=${tmp}/osmin
- sparse_loop=$(losetup -f)
-
- _ext3fs_sparse ${sparse} 128M
- losetup ${sparse_loop} ${sparse}
-
- # Create a DM snapshot device...
- local name="imgcreate-$$"
- local size=$(stat --format="%s" ${image})
- size=$(( ${size} / 512 ))
-
- dmsetup create ${name} \
- --table "0 ${size} snapshot ${image_loop} ${sparse_loop} p 8"
-
- # ...and resize it to its minimal size
- _ext3fs_resize /dev/mapper/${name}
-
- # Get size of the bytes used by the cow image
- local cow_size=( $(dmsetup status | grep "^${name}") )
- cow_size=${cow_size[4]}
- cow_size=$(awk -F"/" '{ print $1 }' <<<${cow_size})
- cow_size=$(( ${cow_size} * 512 ))
-
- sleep 2
- dmsetup remove ${name}
-
- # ... and truncate it to its minimal size
- truncate -s ${cow_size} ${sparse}
-
- losetup -d ${sparse_loop}
- losetup -d ${image_loop}
-
- rm -f ${file}
- ( cd ${tmp} && \
- mksquashfs * ${file} -no-progress )
-}
-
-function _ext3fs_blocks() {
- local device=${1}
-
- dumpe2fs -h ${device} 2>/dev/null | \
- grep "Block count" | awk '{ print $NF }'
-}
-
-function _ext3fs_bytes() {
- echo $(( $(_ext3fs_blocks $@) * 512 ))
-}
-
-function _ext3fs_resize() {
- local device=${1}
- local size=${2}
-
- [ -z "${size}" ] && size="-M"
-
- e2fsck -f -y ${device}
- resize2fs ${device} ${size}
-
- # Resparse after every resize operation
- _ext3fs_resparse ${device}
-}
-
-function _ext3fs_sparse() {
- local file=${1}
- local size=${2}
-
- # Create a sparse disk with given size
- dd if=/dev/zero of=${file} bs=1 count=0 seek=${size}
-}
-
-function _ext3fs_resparse() {
- local file=${1}
-
- zerofree ${file}
-}
-
-function _installer_image_ext3fs() {
- local file=${1}
- local dir=${2}
-
- local zero=$(mktemp)
-
- mkdir -p $(dirname ${file}) 2>/dev/null
-
- # Create a zeroed file
- _ext3fs_sparse ${file} ${FSSIZE}G
-
- # Create a temporary directory
- # and get a free loop device
- local tmp_dir=$(mktemp -d)
- local loop=$(losetup -f)
-
- # Set up the loop device
- losetup ${loop} ${file}
-
- # Create filesystem
- mkfs.${FSTYPE} \
- -L "$(basename ${file})" \
- -m 1 \
- ${loop}
-
- # Tune the FS
- tune2fs -c0 -i0 -Odir_index -ouser_xattr,acl ${loop}
-
- # Mount and copy all files to the FS
- mount ${loop} ${tmp_dir}
- cp -frp ${dir}/* ${tmp_dir}
- umount ${tmp_dir}
-
- losetup -d ${loop}
-
- _ext3fs_resparse ${file}
-
- rm -rf ${tmp_dir}
-}
-
-function install_config() {
- local src=${1}
- local dest=${2}
-
- sed \
- -e "s/@NAME@/${DISTRO_NAME}/g" \
- -e "s/@SNAME@/${DISTRO_SNAME}/g" \
- -e "s/@VERSION@/${DISTRO_VERSION}/g" \
- -e "s/@SLOGAN@/${DISTRO_SLOGAN}/g" \
- -e "s/@KERNEL@/${KERNEL_RELEASE}/g" \
- < ${src} > ${dest}
-}
-
-function install_isolinux() {
- local dest=${1}
-
- mkdir -p ${dest} 2>/dev/null
-
- local i
- for i in isolinux.bin vesamenu.c32; do
- cp -f /usr/share/syslinux/${i} ${dest}/
- done
-
- install_config ${BOOTLOADER_DIR}/installer.conf ${dest}/isolinux.cfg
- cp -f ${BOOTLOADER_DIR}/{*.msg,splash.*} ${dest}/
-
- install_kernel ${dest}
-}
-
-function install_kernel() {
- local dest=${1}
-
- mkdir -p ${dest} 2>/dev/null
-
- local file
- for file in vmlinuz; do
- cp -f /boot/${file}-${KERNEL_RELEASE}* ${dest}/${file}0
- done
-
- dracut -f ${dest}/initrd0 ${KERNEL_RELEASE}
-}
-
-case "${1}" in
- iso)
- log DEBUG "Creating ISO image..."
-
- ISO_DIR=$(mktemp -d)
- ISO_FILE=$(mktemp)
-
- # Copy installer image to ISO
- installer_image ${ISO_DIR}/LiveOS /installer
-
- # Install bootloader
- install_isolinux ${ISO_DIR}/isolinux
-
- # "Copy" packages
- mv /packages ${ISO_DIR}/Packages
-
- cd ${ISO_DIR} && \
- mkisofs -J -r -V "${DISTRO_NAME}_${DISTRO_VERSION}" \
- -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot \
- -boot-load-size 4 -boot-info-table . > ${ISO_FILE}
-
- cat ${ISO_FILE} > ${IMAGES_DIR}/${ISO_FILENAME}
- ;;
-
- *)
- log ERROR "Unknown type of image: ${1}"
- exit 1
- ;;
-esac
+++ /dev/null
-#!/bin/bash
-
-export PATH=${PATH}:/tools_i686/bin
-
-if [ -z "${1}" ]; then
- exit 1
-fi
-
-cpio --extract --to-stdout data.img <${1} | xzcat | \
- tar tvv
-
-cpio --extract --to-stdout info < ${1}
+++ /dev/null
-#!/bin/bash
-###############################################################################
-# #
-# IPFire.org - A linux based firewall #
-# Copyright (C) 2010 Michael Tremer & Christian Schmidt #
-# #
-# This program is free software: you can redistribute it and/or modify #
-# it under the terms of the GNU General Public License as published by #
-# the Free Software Foundation, either version 3 of the License, or #
-# (at your option) any later version. #
-# #
-# This program is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
-# GNU General Public License for more details. #
-# #
-# You should have received a copy of the GNU General Public License #
-# along with this program. If not, see <http://www.gnu.org/licenses/>. #
-# #
-###############################################################################
-
-BASEDIR=$(dirname ${0})
-
-PACKAGER_HASH_ALGORITHMS="md5 sha1 sha256 sha512"
-PACKAGER_VERSION="1"
-
-TAR_OPTIONS="--posix --acls --no-recursion --sparse --xz"
-
-. ${BASEDIR}/common-functions
-
-# Create an alias for PKG_ORIGIN to track which source package has built this
-# binary package.
-PKG_ORIGIN=${PKG_NAME}
-
-### MAIN ###
-
-if [ $# -ne 2 ]; then
- echo "${0}: Wrong number of arguments: $# - $@" >&2
- exit 2
-fi
-
-PKG_NAME=${1}
-PKG_TARGET=${2}
-
-
-### Function definitions ###
-
-function create_metafile() {
- local files=${1}
-
- cat <<EOF
-### ${PKG_NAME} package
-
-VERSION="${PACKAGER_VERSION}"
-
-# Build information
-BUILD_DATE="${BUILD_DATE}"
-BUILD_HOST="${BUILD_HOST}"
-BUILD_ID="${BUILD_ID}"
-
-# Distribution information
-DISTRO_NAME="${DISTRO_NAME}"
-DISTRO_EPOCH="${DISTRO_EPOCH}"
-DISTRO_VENDOR="${DISTRO_VENDOR}"
-
-# Package information
-PKG_NAME="${PKG_NAME}"
-PKG_VER="${PKG_VER}"
-PKG_REL="${PKG_REL}"
-PKG_EPOCH="${PKG_EPOCH}"
-
-PKG_GROUP="${PKG_GROUP}"
-PKG_ARCH="${PKG_ARCH}"
-PKG_ORIGIN="${PKG_ORIGIN}"
-
-PKG_MAINTAINER="${PKG_MAINTAINER}"
-PKG_LICENSE="${PKG_LICENSE}"
-
-PKG_SUMMARY="${PKG_SUMMARY}"
-PKG_DESCRIPTION="${PKG_DESCRIPTION}"
-
-# Dependency info
-PKG_DEPS="$(find_requires ${files})"
-PKG_PROVIDES="$(find_provides ${files})"
-
-PKG_DATA_SIZE="$(du -cb ${files} | tail -n1 | awk '{ print $1 }')"
-
-###
-EOF
-}
-
-function create_controlfile() {
- cat <<EOF
-#!/bin/sh
-
-function prein() {
-${CONTROL_PREIN}
-}
-
-function preun() {
-${CONTROL_PREUN}
-}
-
-function postin() {
-${CONTROL_POSTIN}
-}
-
-function postun() {
-${CONTROL_POSTUN}
-}
-
-###
-EOF
-}
-
-function find_files() {
- local paths=$@
- local file
-
- for file in $(find ${paths} 2>/dev/null | sort); do
- # Remove ${BUILDROOT}
- file="${file#${BUILDROOT}}"
-
- # Remove all leading slashes
- while [ "${file:0:1}" = "/" ]; do
- file="${file:1:${#file}}"
- done
-
- echo "${file}"
- done
-}
-
-function __filelist() {
- local paths
- local exclude_paths
-
- # Disable globbing
- set -f
-
- local path
- local exclude
- for path in ${PKG_FILES}; do
- if [ "${path:0:1}" = "!" ]; then
- exclude="1"
- path="${path:1:${#path}}"
- else
- exclude="0"
- fi
-
- if [ "${path:0:1}" != "/" ]; then
- path="/${path}"
- fi
-
- path="${BUILDROOT}${path}"
- if [ "${exclude}" = "0" ]; then
- paths="${paths} ${path}"
- else
- exclude_paths="${exclude_paths} ${path}"
- fi
- done
-
- # Enable globbing again
- set +f
-
- # If not paths were found -> break
- [ -z "${paths}" ] && return
-
- local excludes
- if [ -n "${exclude_paths}" ]; then
- excludes=$(find_files ${exclude_paths})
- fi
-
- for file in $(find_files ${paths}); do
- if [ -n "${excludes}" ]; then
- if ! listmatch ${file} ${excludes}; then
- echo "${file}"
- fi
- else
- echo "${file}"
- fi
- done
-}
-
-function create_dataimg() {
- local dir=${1}
- local img=${2}
-
- # First, remove all crappy directories
- directory_remove_orphans ${BUILDROOT}
-
- # Generate the filelist.
- local filelist=$(mktemp)
- __filelist > ${filelist}
-
- if [ "${DEBUG}" = "1" ]; then
- cat ${filelist} >&2
- fi
-
- # Create the tarball
- # Prints the filelist on fd 2.
- cd ${BUILDROOT} && \
- tar ${TAR_OPTIONS} \
- --create -vv \
- --file=${img} \
- --files-from=${filelist} >&2
-
- # Untar the tarball for further investigation
- cd ${dir} && \
- tar ${TAR_OPTIONS} \
- --extract \
- --file=${img}
-
- local file
-
- # Remove the just copied files
- sort_by_length $(<${filelist}) | \
- while read file; do
- [ -z "${file}" ] && continue
-
- if [ -d "${BUILDROOT}/${file}" ]; then
- if dir_is_empty ${BUILDROOT}/${file}; then
- rm -rf ${BUILDROOT}/${file}
- fi
- continue
- fi
- rm -rf ${BUILDROOT}/${file}
- done
-
- # Return the filelist
- cd ${dir} && \
- for file in $(find . -not -type d); do
- echo "${file:1:${#file}}"
- done
-}
-
-function create_package() {
- local target_file=${1}
-
- # Create temporary directory where to put in all files that will go into
- # the package.
- local tmp=$(mktemp -d)
-
- # A place where to copy the package files for further processing.
- local files=$(mktemp -d)
-
- # Create the package content
- create_dataimg ${files} ${tmp}/data.img > ${tmp}/filelist
- create_metafile ${files} > ${tmp}/info
- create_controlfile > ${tmp}/control
-
- # Add an empty file to hold place for the signature
- touch ${tmp}/signature
-
- # Calculate hash sums of the package files
- log DEBUG "Calculating hash sums."
- local algorithm
- local file
- local files=$(cd ${tmp} && ls)
- for algorithm in ${PACKAGER_HASH_ALGORITHMS}; do
- log DEBUG " Algorithm: ${algorithm}"
- for file in ${files}; do
- log DEBUG " ${file}"
- echo "${algorithm} $(cd ${tmp} && ${algorithm}sum ${file})" \
- >> ${tmp}/sums
- done
- done
-
- # Create the package
- cd ${tmp} && \
- find . | cpio -o -H newc --quiet > ${target_file}
-
- # Cleanup
- rm -rf ${tmp} ${files}
-}
-
-create_package ${PKG_TARGET}
-
-exit $?
-
+++ /dev/null
-#!/bin/bash
-
-for patch in $@; do
- echo "Applying file ${patch}..."
-
- if [ "${patch##*/*.}" = "patch0" ]; then
- cmd="patch -Np0"
- else
- cmd="patch -Np1"
- fi
-
- ${cmd} -i ${patch}
- ret=$?
-
- if [ ${ret} -ne 0 ]; then
- exit ${ret}
- fi
-done
+++ /dev/null
-#!/usr/bin/perl
-
-# RPM (and it's source code) is covered under two separate licenses.
-
-# The entire code base may be distributed under the terms of the GNU
-# General Public License (GPL), which appears immediately below.
-# Alternatively, all of the source code in the lib subdirectory of the
-# RPM source code distribution as well as any code derived from that
-# code may instead be distributed under the GNU Library General Public
-# License (LGPL), at the choice of the distributor. The complete text
-# of the LGPL appears at the bottom of this file.
-
-# This alternative is allowed to enable applications to be linked
-# against the RPM library (commonly called librpm) without forcing
-# such applications to be distributed under the GPL.
-
-# Any questions regarding the licensing of RPM should be addressed to
-# Erik Troan <ewt@redhat.com>.
-
-# a simple script to print the proper name for perl libraries.
-
-# To save development time I do not parse the perl grammmar but
-# instead just lex it looking for what I want. I take special care to
-# ignore comments and pod's.
-
-# it would be much better if perl could tell us the proper name of a
-# given script.
-
-# The filenames to scan are either passed on the command line or if
-# that is empty they are passed via stdin.
-
-# If there are lines in the file which match the pattern
-# (m/^\s*\$VERSION\s*=\s+/)
-# then these are taken to be the version numbers of the modules.
-# Special care is taken with a few known idioms for specifying version
-# numbers of files under rcs/cvs control.
-
-# If there are strings in the file which match the pattern
-# m/^\s*\$RPM_Provides\s*=\s*["'](.*)['"]/i
-# then these are treated as additional names which are provided by the
-# file and are printed as well.
-
-# I plan to rewrite this in C so that perl is not required by RPM at
-# build time.
-
-# by Ken Estes Mail.com kestes@staff.mail.com
-
-if ("@ARGV") {
- foreach (@ARGV) {
- process_file($_);
- }
-} else {
-
- # notice we are passed a list of filenames NOT as common in unix the
- # contents of the file.
-
- foreach (<>) {
- process_file($_);
- }
-}
-
-
-foreach $module (sort keys %require) {
- if (length($require{$module}) == 0) {
- print "perl($module)\n";
- } else {
-
- # I am not using rpm3.0 so I do not want spaces arround my
- # operators. Also I will need to change the processing of the
- # $RPM_* variable when I upgrade.
-
- print "perl($module)=$require{$module}\n";
- }
-}
-
-exit 0;
-
-
-
-sub process_file {
-
- my ($file) = @_;
- chomp $file;
-
- open(FILE, "<$file") || return;
-
- my ($package, $version, $incomment, $inover) = ();
-
- while (<FILE>) {
-
- # skip the documentation
-
- # we should not need to have item in this if statement (it
- # properly belongs in the over/back section) but people do not
- # read the perldoc.
-
- if (m/^=(head[1-4]|pod|item)/) {
- $incomment = 1;
- }
-
- if (m/^=(cut)/) {
- $incomment = 0;
- $inover = 0;
- }
-
- if (m/^=(over)/) {
- $inover = 1;
- }
-
- if (m/^=(back)/) {
- $inover = 0;
- }
-
- if ($incomment || $inover) {
- next;
- }
-
- # skip the data section
- if (m/^__(DATA|END)__$/) {
- last;
- }
-
- # not everyone puts the package name of the file as the first
- # package name so we report all namespaces except some common
- # false positives as if they were provided packages (really ugly).
-
- if (m/^\s*package\s+([_:a-zA-Z0-9]+)\s*;/) {
- $package=$1;
- undef $version;
- if ($package eq 'main') {
- undef $package;
- } else {
- # If $package already exists in the $require hash, it means
- # the package definition is broken up over multiple blocks.
- # In that case, don't stomp a previous $VERSION we might have
- # found. (See BZ#214496.)
- $require{$package}=undef unless (exists $require{$package});
- }
- }
-
- # after we found the package name take the first assignment to
- # $VERSION as the version number. Exporter requires that the
- # variable be called VERSION so we are safe.
-
- # here are examples of VERSION lines from the perl distribution
-
- #FindBin.pm:$VERSION = $VERSION = sprintf("%d.%02d", q$Revision: 1.9 $ =~ /(\d+)\.(\d+)/);
- #ExtUtils/Install.pm:$VERSION = substr q$Revision: 1.9 $, 10;
- #CGI/Apache.pm:$VERSION = (qw$Revision: 1.9 $)[1];
- #DynaLoader.pm:$VERSION = $VERSION = "1.03"; # avoid typo warning
- #General.pm:$Config::General::VERSION = 2.33;
- #
- # or with the new "our" pragma you could (read will) see:
- #
- # our $VERSION = '1.00'
- if (($package) && (m/^\s*(our\s+)?\$(\Q$package\E::)?VERSION\s*=\s+/)) {
-
- # first see if the version string contains the string
- # '$Revision' this often causes bizzare strings and is the most
- # common method of non static numbering.
-
- if (m/(\$Revision: (\d+[.0-9]+))/) {
- $version= $2;
- } elsif (m/[\'\"]?(\d+[.0-9]+)[\'\"]?/) {
-
- # look for a static number hard coded in the script
-
- $version= $1;
- }
- $require{$package}=$version;
- }
-
- # Allow someone to have a variable that defines virtual packages
- # The variable is called $RPM_Provides. It must be scoped with
- # "our", but not "local" or "my" (just would not make sense).
- #
- # For instance:
- #
- # $RPM_Provides = "blah bleah"
- #
- # Will generate provides for "blah" and "bleah".
- #
- # Each keyword can appear multiple times. Don't
- # bother with datastructures to store these strings,
- # if we need to print it print it now.
-
- if ( m/^\s*(our\s+)?\$RPM_Provides\s*=\s*["'](.*)['"]/i) {
- foreach $_ (split(/\s+/, $2)) {
- print "$_\n";
- }
- }
-
- }
-
- close(FILE) ||
- die("$0: Could not close file: '$file' : $!\n");
-
- return ;
-}
+++ /dev/null
-#!/usr/bin/perl
-
-# RPM (and its source code) is covered under two separate licenses.
-
-# The entire code base may be distributed under the terms of the GNU
-# General Public License (GPL), which appears immediately below.
-# Alternatively, all of the source code in the lib subdirectory of the
-# RPM source code distribution as well as any code derived from that
-# code may instead be distributed under the GNU Library General Public
-# License (LGPL), at the choice of the distributor. The complete text
-# of the LGPL appears at the bottom of this file.
-
-# This alternatively is allowed to enable applications to be linked
-# against the RPM library (commonly called librpm) without forcing
-# such applications to be distributed under the GPL.
-
-# Any questions regarding the licensing of RPM should be addressed to
-# Erik Troan <ewt@redhat.com>.
-
-# a simple makedepend like script for perl.
-
-# To save development time I do not parse the perl grammmar but
-# instead just lex it looking for what I want. I take special care to
-# ignore comments and pod's.
-
-# It would be much better if perl could tell us the dependencies of a
-# given script.
-
-# The filenames to scan are either passed on the command line or if
-# that is empty they are passed via stdin.
-
-# If there are strings in the file which match the pattern
-# m/^\s*\$RPM_Requires\s*=\s*["'](.*)['"]/i
-# then these are treated as additional names which are required by the
-# file and are printed as well.
-
-# I plan to rewrite this in C so that perl is not required by RPM at
-# build time.
-
-# by Ken Estes Mail.com kestes@staff.mail.com
-
-if ("@ARGV") {
- foreach (@ARGV) {
- process_file($_);
- }
-} else {
-
- # notice we are passed a list of filenames NOT as common in unix the
- # contents of the file.
-
- foreach (<>) {
- process_file($_);
- }
-}
-
-
-foreach $module (sort keys %require) {
- if (length($require{$module}) == 0) {
- print "perl($module)\n";
- } else {
-
- # I am not using rpm3.0 so I do not want spaces around my
- # operators. Also I will need to change the processing of the
- # $RPM_* variable when I upgrade.
-
- print "perl($module)>=$require{$module}\n";
- }
-}
-
-exit 0;
-
-
-
-sub process_file {
-
- my ($file) = @_;
- chomp $file;
-
- open(FILE, "<$file") || return;
-
- while (<FILE>) {
-
- # skip the "= <<" block
-
- if ( ( m/^\s*\$(?:.*)\s*=\s*<<\s*(["'`])(.*)\1/) ||
- ( m/^\s*\$(.*)\s*=\s*<<(\w*)\s*;/) ) {
- $tag = $2;
- while (<FILE>) {
- chomp;
- ( $_ eq $tag ) && last;
- }
- $_ = <FILE>;
- }
-
- # skip q{} quoted sections - just hope we don't have curly brackets
- # within the quote, nor an escaped hash mark that isn't a comment
- # marker, such as occurs right here. Draw the line somewhere.
- if ( m/^.*\Wq[qxwr]?\s*([\{\(\[#|\/])[^})\]#|\/]*$/ && ! m/^\s*(require|use)\s/ ) {
- $tag = $1;
- $tag =~ tr/{\(\[\#|\//})]#|\//;
- while (<FILE>) {
- ( $_ =~ m/\}/ ) && last;
- }
- }
-
- # skip the documentation
-
- # we should not need to have item in this if statement (it
- # properly belongs in the over/back section) but people do not
- # read the perldoc.
-
- if ( (m/^=(head[1-4]|pod|item)/) .. (m/^=(cut)/) ) {
- next;
- }
-
- if ( (m/^=(over)/) .. (m/^=(back)/) ) {
- next;
- }
-
- # skip the data section
- if (m/^__(DATA|END)__$/) {
- last;
- }
-
- # Each keyword can appear multiple times. Don't
- # bother with datastructures to store these strings,
- # if we need to print it print it now.
- #
- # Again allow for "our".
- if ( m/^\s*(our\s+)?\$RPM_Requires\s*=\s*["'](.*)['"]/i) {
- foreach $_ (split(/\s+/, $2)) {
- print "$_\n";
- }
- }
-
- if (
-
-# ouch could be in a eval, perhaps we do not want these since we catch
-# an exception they must not be required
-
-# eval { require Term::ReadLine } or die $@;
-# eval "require Term::Rendezvous;" or die $@;
-# eval { require Carp } if defined $^S; # If error/warning during compilation,
-
-
- (m/^(\s*) # we hope the inclusion starts the line
- (require|use)\s+(?!\{) # do not want 'do {' loops
- # quotes around name are always legal
- [\'\"]?([^\;\ \'\"\t]*)[\'\"]?[\t\;\ ]
- # the syntax for 'use' allows version requirements
- \s*([.0-9]*)
- /x)
- ) {
- my ($whitespace, $statement, $module, $version) = ($1, $2, $3,$4);
-
- # we only consider require statements that are flush against
- # the left edge. any other require statements give too many
- # false positives, as they are usually inside of an if statement
- # as a fallback module or a rarely used option
-
- ($whitespace ne "" && $statement eq "require") && next;
-
- # if there is some interpolation of variables just skip this
- # dependency, we do not want
- # do "$ENV{LOGDIR}/$rcfile";
-
- ($module =~ m/\$/) && next;
-
- # skip if the phrase was "use of" -- shows up in gimp-perl, et al.
- next if $module eq 'of';
-
- # if the module ends in a comma we probaly caught some
- # documentation of the form 'check stuff,\n do stuff, clean
- # stuff.' there are several of these in the perl distribution
-
- ($module =~ m/[,>]$/) && next;
-
- # if the module name starts in a dot it is not a module name.
- # Is this necessary? Please give me an example if you turn this
- # back on.
-
- # ($module =~ m/^\./) && next;
-
- # if the module ends with .pm strip it to leave only basename.
- # starts with /, which means its an absolute path to a file
- if ($module =~ m(^/)) {
- print "$module\n";
- next;
- }
-
- # sometimes people do use POSIX qw(foo), or use POSIX(qw(foo)) etc.
- # we can strip qw.*$, as well as (.*$:
- $module =~ s/qw.*$//;
- $module =~ s/\(.*$//;
-
- $module =~ s/\.pm$//;
-
- # some perl programmers write 'require URI/URL;' when
- # they mean 'require URI::URL;'
-
- $module =~ s/\//::/;
-
- # trim off trailing parentheses if any. Sometimes people pass
- # the module an empty list.
-
- $module =~ s/\(\s*\)$//;
-
- if ( $module =~ m/^v?([0-9._]+)$/ ) {
- # if module is a number then both require and use interpret that
- # to mean that a particular version of perl is specified
-
- my $ver=$1;
- if ($ver =~ /5.00/) {
- print "perl>=0:$ver\n";
- next;
- }
- else {
- print "perl>=1:$ver\n";
- next;
- }
-
- };
-
- # ph files do not use the package name inside the file.
- # perlmodlib documentation says:
-
- # the .ph files made by h2ph will probably end up as
- # extension modules made by h2xs.
-
- # so do not expend much effort on these.
-
-
- # there is no easy way to find out if a file named systeminfo.ph
- # will be included with the name sys/systeminfo.ph so only use the
- # basename of *.ph files
-
- ($module =~ m/\.ph$/) && next;
-
- $require{$module}=$version;
- $line{$module}=$_;
- }
-
- }
-
- close(FILE) ||
- die("$0: Could not close file: '$file' : $!\n");
-
- return ;
-}
+++ /dev/null
-#!/bin/sh
-
-PYTHON=$(which python 2>/dev/null)
-
-if [ -z "${PYTHON}" ]; then
- # Python is not present. Fail silently.
- exit 0
-fi
-
-files=""
-for i in $*; do
- if [ -e ${i}c ] && [ -e ${i}o ]; then
- continue # all files we want are already there
- fi
- files="$files $i"
-done
-
-if [ -z "${files}" ]; then
- # No files need to be proceeded.
- exit 0
-fi
-
-$PYTHON -c "
-import sys, os, string, py_compile
-
-files = '''$files'''
-print 'Byte-compiling python modules...'
-for file in string.split(files):
- if not os.path.exists(file) or not (len(file) >= 3 and file[-3:] == '.py'):
- continue
- print file,
- sys.stdout.flush()
- py_compile.compile(file)
-print" || exit $?
-
-# this will fail for python < 1.5, but that doesn't matter ...
-$PYTHON -O -c "
-import sys, os, string, py_compile
-
-files = '''$files'''
-print 'Byte-compiling python modules (optimised versions) ...'
-for file in string.split(files):
- if not os.path.exists(file) or not (len(file) >= 3 and file[-3:] == '.py'):
- continue
- print file,
- sys.stdout.flush()
- py_compile.compile(file)
-print" 2>/dev/null || :
+++ /dev/null
-#!/bin/bash
-
-DIR_QA=${0}.d
-
-failed=0
-for file in ${DIR_QA}/*; do
- [ -x "${file}" ] || continue
-
- ${file} || failed=1
-done
-
-exit ${failed}
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="Include files have to belong to the root user. \
- This script will fix this automatically."
-
-check() {
- if [ ! -d "${BUILDROOT}/usr/include" ]; then
- return 0
- fi
-
- chown -R root:root ${BUILDROOT}/usr/include
-}
-
-run
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="Remove documentation files."
-
-function check() {
- for dir in ${BUILDROOT}/usr/{,share}/{doc,gtk-doc,info}; do
- if [ -d "${dir}" ]; then
- log DEBUG " Removing: ${dir}"
- rm -rf ${dir} || exit $?
- fi
- done
-}
-
-run
-
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="Removing unwanted files: *.a *.la"
-
-function check() {
- for file in $(find ${BUILDROOT} -name "*.a" -or -name "*.la"); do
-
- # Don't remove libc_nonshared.a. It is used by gcc/ld.
- [ "${file##*/}" = "libc_nonshared.a" ] && continue
- [ "${file##*/}" = "libpthread_nonshared.a" ] && continue
- [ "${file##*/}" = "libgcc.a" ] && continue
- [ "${file##*/}" = "libgcc_eh.a" ] && continue
- [ "${file##*/}" = "libfl_pic.a" ] && continue
- [ "${file##*/}" = "libpython2.6.a" ] && continue
-
- log DEBUG " Removing: ${file}"
- rm -f ${file} || exit $?
- done
-}
-
-run
-
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="Searching for world-writeable files..."
-
-function check() {
- local ret=0
-
- local files=$(find ${BUILDROOT} -type f -perm -2 2>/dev/null)
- if [ -n "${files}" ]; then
- log ERROR " QA Security Notice:"
- log ERROR " - The folloing files will be world writable."
- log ERROR " - This may or may not be a security problem, most of the time it is one."
- log ERROR " - Please double check that these files really need a world writeable bit and file bugs accordingly."
- log ERROR
- log ERROR "${files}"
- ret=1
- fi
-
- files=$(find ${BUILDROOT} -type f '(' -perm -2002 -o -perm -4002 ')')
- if [ -n "${files}" ]; then
- log ERROR " QA Notice: Unsafe files detected (set*id and world writable)"
- log ERROR
- log ERROR "${files}"
- ret=1
- fi
-
- return ${ret}
-}
-
-run
-
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-# Check for absolute symlinks.
-# We do not allow them because they may point to any bad location.
-
-log_debug "Search for absolute symlinks"
-
-function check() {
- local failed=0
-
- for link in $(find ${BUILDROOT} -type l); do
- if fgrep -q "/lib/udev/devices" <<<${link}; then
- continue
- fi
-
- destination=$(readlink ${link})
- if [ "${destination:0:1}" = "/" ]; then
- log ERROR " Absolute symlink: ${link}"
- failed=1
- fi
- if [ ! -e "${link%/*}/${destination}" ]; then
- log ERROR " Not existant destination: ${link} -> ${destination}"
- failed=1
- fi
- done
-
- return ${failed}
-}
-
-run
-
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="Checking correct installation of libraries"
-
-function check() {
- local failed=0
- for lib in $(find ${BUILDROOT}/lib -type f -name "lib*.so.*" 2>/dev/null); do
- lib=${lib##*/}
- lib=${lib%%.so*}
-
- if [ ! -e "${BUILDROOT}/usr/lib/${lib}.so" ]; then
- log ERROR " /usr/lib/${lib}.so is missing"
- failed=1
- fi
- done
-
- return ${failed}
-}
-
-run
-
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="Every binary file has to provide a canary."
-
-function check() {
- local failed=0
-
- local file
- for file in $(find_elf_files --prefix=${BUILDROOT} ${BINARY_PATHS}); do
- if filter_startfiles ${file}; then
- continue
- fi
-
- if ! file_has_canary ${file}; then
- log_warning " Has no canary: ${file}"
- failed=1
- fi
- done
-
- # This is currently disabled and will only return a warning !
- failed=0
-
- return ${failed}
-}
-
-run
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="Files with executable stacks will not work properly (or at all!) \
- on some architectures/operating systems."
-
-check() {
- local failed=0
-
- local file
- for file in $(find_elf_files --prefix=${BUILDROOT} ${BINARY_PATHS}); do
- if file_has_execstack ${file}; then
- log_error " File has execstack: ${file}"
- failed=1
- fi
- done
-
- return ${failed}
-}
-
-run
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="Detect invalid interpreters."
-
-check() {
- local failed=0
-
- local file
- local interpreter
- for file in $(find ${BUILDROOT} -type f 2>/dev/null); do
- # If a file is not executeable we don't need to check it
- [ -x "${file}" ] || continue
-
- if file_is_script ${file}; then
- interpreter=$(file_get_interpreter ${file})
-
- if grep -q /usr/local <<<${interpreter}; then
- failed=1
- log_error " Interpreter in /usr/local: ${file}"
- fi
-
- # Search for bad /usr/bin/env
- if [ "$(basename ${interpreter})" = "env" ]; then
- # Autofix that crap
- sed -i ${file} \
- -e "s,/usr/bin/env python.\..,/usr/bin/python," \
- -e "s,/usr/bin/env python,/usr/bin/python," \
- -e "s,/usr/bin/env perl,/usr/bin/perl,"
-
- # If we could not fix it, raise an error
- if [ "${interpreter}" = "$(file_get_interpreter ${file})" ]; then
- failed=1
- log_error " Script uses forbidden \"env\" interpreter: ${file}"
- fi
- fi
- fi
- done
-
- return ${failed}
-}
-
-run
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="Every shared object has to provide the NEEDED entry."
-
-check() {
- local failed=0
-
- local file
- local needed
- for file in $(find_elf_files --prefix=${BUILDROOT} ${LIBARY_PATHS}); do
- if ! file_is_shared_object ${file}; then
- continue
- fi
-
- if ! file_has_interpreter ${file}; then
- continue
- fi
-
- needed=$(file_get_needed ${file})
- if [ -z "${needed}" ]; then
- log_error " File lacks needed attribute: ${file}"
- failed=1
- fi
- done
-
- return ${failed}
-}
-
-run
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="Every shared object has to provide the SONAME entry."
-
-check() {
- local failed=0
-
- local file
- local soname
- for file in $(find_elf_files --prefix=${BUILDROOT} ${LIBARY_PATHS}); do
- if ! grep -q "\.so" <<<${file}; then
- continue
- fi
-
- if ! file_is_shared_object ${file}; then
- continue
- fi
-
- if ! file_has_interpreter ${file}; then
- continue
- fi
-
- soname=$(file_get_soname ${file})
- if [ -z "${soname}" ]; then
- log_error " File lacks soname attribute: ${file}"
- failed=1
- fi
- done
-
- return ${failed}
-}
-
-run
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-function check() {
- local failed=0
-
- FILTER="${QUALITY_AGENT_WHITELIST_NX}"
-
- local file
- for file in $(find_elf_files --prefix=${BUILDROOT} ${BINARY_PATHS}); do
- if filtered ${file}; then
- continue
- fi
-
- if ! file_has_nx ${file}; then
- log_error " No NX: ${file}"
- failed=1
- fi
- done
-
- return ${failed}
-}
-
-run
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="Text relocations force the dynamic linker to perform extra \
- work at startup, waste system resources, and may pose a security \
- risk. On some architectures, the code may not even function \
- properly, if at all."
-
-function check() {
- local failed=0
-
- local file
- for file in $(find_elf_files --prefix=${BUILDROOT} ${BINARY_PATHS}); do
- if filter_startfiles ${file}; then
- continue
- fi
-
- if ! file_is_relro_full ${file}; then
- if [ "${QUALITY_AGENT_PERMIT_NOT_FULL_RELRO}" = "yes" ]; then
- log_warning " Is not full relro: ${file}"
- else
- log_error " Is not relro: ${file}"
- failed=1
- fi
- fi
- done
-
- return ${failed}
-}
-
-run
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="Check for binaries in /bin or /sbin that link to /usr/..."
-
-function check() {
- local ret=0
-
- for file in $(find ${BUILDROOT}/{bin,lib,sbin}/* 2>/dev/null); do
- [ -f "${file}" ] || continue
- log DEBUG " ${file}"
-
- interpreter=$(file_get_interpreter ${file})
- if [ ! -e "${interpreter}" ]; then
- log WARN " SKIPPED because interpreter is not available"
- continue
- fi
-
- libs=$(ldd ${file})
- if grep -q /usr/lib <<<${libs}; then
- log ERROR "${file} links to libs in /usr/lib..."
- log ERROR " ${libs}"
- ret=1
- fi
- done
-
- return ${ret}
-}
-
-run
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="Searching for RPATHs. We don't want paths that point to the tree where \
- the package was built (older, broken libtools would do this). \
- Also check for null paths because the loader will search \$PWD when it \
- finds null paths."
-
-check() {
- local failed=0
-
- local file
- local rpath
- for file in $(find_elf_files --prefix=${BUILDROOT} ${BINARY_PATHS}); do
- if filtered ${file}; then
- continue
- fi
-
- rpath=$(file_get_rpath ${file})
- if [ -n "${rpath}" ]; then
- if [ "${QUALITY_AGENT_RPATH_ALLOW_ORIGIN}" = "yes" ]; then
- [ "${rpath}" = '$ORIGIN' ] && continue
- fi
- if listmatch ${rpath} ${QUALITY_AGENT_WHITELIST_RPATH}; then
- continue
- fi
- log_error " File has unallowed rpath: ${file} - ${rpath}"
- failed=1
- fi
- done
-
- return ${failed}
-}
-
-run
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="Decompressing man-pages..."
-
-function check() {
- for file in $(find ${BUILDROOT}/usr/share/man -type f 2>/dev/null); do
- log DEBUG " Processing: ${file}"
- if [[ ${file} =~ \.gz$ ]]; then
- gzip -d ${file}
- fi
- done
-}
-
-run
-
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="Python byte-code files could be hardlinked if the optimized one is equal"
-DESC="${DESC} to the other one."
-
-function check() {
- for py in $(find ${BUILDROOT} -type f -name "*.py"); do
- if [ -e "${py}c" ] && [ -e "${py}o" ]; then
- if cmp -s "${py}c" "${py}o"; then
- log DEBUG " ${py}c -> ${py}o"
- ln -f "${py}c" "${py}o"
- fi
- fi
- done
-}
-
-run
-
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="Remove unwanted files."
-
-function check() {
- for dir in $(sort_by_length {,/usr}/{{,s}bin,lib{,exec}} /usr/share/man{,/man{0,1,2,3,4,5,6,7,8,9}}); do
- dir="${BUILDROOT}/${dir}"
- if [ -d "${dir}" ] && [ "$(ls -1A ${dir} | wc -l)" = "0" ]; then
- log DEBUG " Removing ${dir}"
- rm -rf ${dir}
- fi
- done
-}
-
-run
-
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="The filelayout should comply to the FHS."
-
-DIRS="/etc/init.d /etc/rc.d /lib/pkgconfig /usr/etc /usr/libexec /usr/local /usr/man /usr/usr /usr/var"
-
-function check() {
- local failed=0
-
- local dir
- for dir in ${DIRS}; do
- if [ -d "${BUILDROOT}${dir}" ]; then
- log_error "Bad directory: ${dir}"
- failed=1
- fi
- done
-
- return ${failed}
-}
-
-run
+++ /dev/null
-#!/bin/bash
-
-. $(dirname ${0})/qa-include
-
-DESC="Stripping reduces size of binary files."
-
-function check() {
- # Strip debugging symbols
- for f in $(find ${BUILDROOT} -type f \( -perm -0100 -or -perm -0010 -or -perm -0001 \)); do
- if (file $f | grep -q ' shared object,'); then
- strip --strip-debug "$f" || :
- fi
- done
-
- for f in $(find ${BUILDROOT} -type f); do
- if (file $f | grep -q ' shared object,'); then
- strip --strip-unneeded "$f" || :
- fi
- done
-}
-
-run
-
+++ /dev/null
-#!/bin/bash
-
-# Include additional functions
-. $(dirname ${0})/../common-functions
-
-function debug() {
- [ "${NAOKI_DEBUG}" = "1" ] || [ "${DEBUG}" = "1" ]
-}
-
-#function log() {
-# local facility=${1}
-# shift
-#
-# printf " %-7s %s\n" "${facility}" "$@"
-#}
-
-function log_debug() {
- debug && log DEBUG "$@"
-}
-
-function log_error() {
- log "ERROR" "$@"
-}
-
-function log_info() {
- log "INFO" "$@"
-}
-
-function log_warning() {
- log "WARNING" "$@"
-}
-
-if [ -z "${BUILDROOT}" ]; then
- echo "${0##*/}: ERROR: BUILDROOT is not set." >&2
- exit 1
-fi
-
-function filtered() {
- [ -z "${FILTER}" ] && return 1
- grep -qE ${FILTER} <<<$@
-}
-
-function print_description() {
- # Remove all whitespaces
- local desc=$(echo ${DESC})
-
- log_info "Check: $(basename ${0})"
- IFS='
-'
- for line in $(fold -s -w 60 <<<${desc}); do
- log_info " ${line}"
- done
- log_info # Empty line
-
- unset IFS
-}
-
-function qa_find() {
- local filetype=${1}
- local command=${2}
-
- log_debug "Running qa_find with command ${command} in ${filetype}"
-
- local file
- for file in $(find_elf_files --prefix=${BUILDROOT} ${!filetype}); do
- ${command} ${file}
- done
-}
-
-function check() {
- log_error "REPLACE THIS FUNCTION BY A CUSTOM CHECK"
- return 1
-}
-
-function run() {
- local error_message
- local ret
-
- error_message=$(check)
- ret=$?
-
- [ -z "${error_message}" ] && \
- [ "${ret}" = "0" ] && return 0
-
- print_description
-
- echo "${error_message}"
- return ${ret}
-}
-
+++ /dev/null
-#!/bin/bash
-
-pychecker --only --limit 1000 \
- --maxlines 500 --maxargs 20 --maxbranches 80 --maxlocals 60 --maxreturns 20 \
- --no-callinit --no-local --no-shadow --no-shadowbuiltin \
- --no-import --no-miximport --no-pkgimport --no-reimport \
- --no-argsused --no-varargsused --no-override \
- naoki/*.py
+++ /dev/null
-#!/bin/bash
-#
-# ssh-keyput -- set up passwordless openssh login.
-#
-# Copyright (C) 2001, 2002, 2006 by SWsoft.
-# Author: Kir Kolyshkin
-#
-# This script is used to put your public ssh keys to another host's
-# authorized_keys[2], so you will be able to ssh login without entering
-# a password. Key pairs are generated if needed, and connectivity
-# is checked after putting the keys.
-
-PROGNAME=`basename $0`
-
-function usage()
-{
- echo "Usage: $PROGNAME [user@]IP [[user@]IP ...]" 1>&2
- exit 0
-}
-
-# Check for correct number of parameters
-test $# -gt 0 || usage;
-
-SSH_KEYGEN=`which ssh-keygen`
-if test $? -ne 0; then
- # Error message is printed by 'which'
- exit 1
-fi
-
-SSH_DIR=~/.ssh
-if ! test -d $SSH_DIR; then
- mkdir $SSH_DIR
-fi
-chmod 700 $SSH_DIR
-
-
-if [ ! -f $SSH_DIR/identity ] || [ ! -f $SSH_DIR/identity.pub ]; then
- echo "Generating ssh1 RSA keys - please wait..."
- rm -f $SSH_DIR/identity $SSH_DIR/identity.pub
- $SSH_KEYGEN -t rsa1 -f $SSH_DIR/identity -P ''
- if [ $? -ne 0 ]; then
- echo "Command \"$SSH_KEYGEN -t rsa1 -f $SSH_DIR/identity" \
- "-P ''\" failed" 1>&2
- exit 1
- fi
-else
- echo "ssh1 RSA key is present"
-fi
-
-if [ ! -f $SSH_DIR/id_dsa ] || [ ! -f $SSH_DIR/id_dsa.pub ]; then
- echo "Generating ssh2 DSA keys - please wait..."
- rm -f $SSH_DIR/id_dsa $SSH_DIR/id_dsa.pub
- $SSH_KEYGEN -t dsa -f $SSH_DIR/id_dsa -P ''
- if test $? -ne 0; then
- echo "Command \"$SSH_KEYGEN -t dsa -f $SSH_DIR/id_dsa" \
- "-P ''\" failed" 1>&2
- exit 1
- fi
-else
- echo "ssh2 DSA key is present"
-fi
-
-SSH1_RSA_KEY=`cat $SSH_DIR/identity.pub`
-SSH2_DSA_KEY=`cat $SSH_DIR/id_dsa.pub`
-
-for IP in $*; do
- echo "You will now be asked for password for $IP"
-# set -x
- ssh -oStrictHostKeyChecking=no $IP "mkdir -p ~/.ssh; chmod 700 ~/.ssh; \
- echo \"$SSH1_RSA_KEY\" >> ~/.ssh/authorized_keys; \
- echo \"$SSH2_DSA_KEY\" >> ~/.ssh/authorized_keys2; \
- chmod 600 ~/.ssh/authorized_keys ~/.ssh/authorized_keys2"
-# set +x
- if test $? -eq 0; then
- echo "Keys were put successfully"
- else
- echo "Error putting keys to $IP" 1>&2
- fi
-done
-
-for IP in $*; do
- for ver in 1 2; do
- echo -n "Checking $IP connectivity by ssh$ver... "
- ssh -q -oProtocol=${ver} -oBatchMode=yes \
- -oStrictHostKeyChecking=no $IP /bin/true
- if [ $? -eq 0 ]; then
- echo "OK"
- else
- echo "failed" 1>&2
- fi
- done
-done
projects / ipfire-3.x.git / commitdiff
