From: Jack Jansen Date: Thu, 3 Jun 2004 13:07:39 +0000 (+0000) Subject: Backport of 1.31-1.34: X-Git-Tag: v2.3.5c1~229 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=a869ef453df053c79712ba61e3b26c01f5c196bf;p=thirdparty%2FPython%2Fcpython.git Backport of 1.31-1.34: - Added a downloader using urllib2 in stead of curl, based on code donated by Kevin Ollivier. This is now the default downloader. - Added a watcher mechanism, whereby downloaders and unpackers (and, later builders) can give status feedback to the user. When running pimp as a command line tool in verbose mode print this output. - Force option should be applied to a single package, not recursively to its dependencies. Fixes #733819. - Don't use "dict" as a variable, it shadows the builtin. Spotted by Bob Ippolito. Two issues spotted by Ronald OUssoren: - there were no accessor functions for the global per-database fields - packages and their dependencies were installed in order in stead of in reverse order. --- diff --git a/Lib/plat-mac/pimp.py b/Lib/plat-mac/pimp.py index 131d5b0dca30..39bc693b08a3 100644 --- a/Lib/plat-mac/pimp.py +++ b/Lib/plat-mac/pimp.py @@ -25,6 +25,7 @@ import md5 import tarfile import tempfile import shutil +import time __all__ = ["PimpPreferences", "PimpDatabase", "PimpPackage", "main", "PIMP_VERSION", "main"] @@ -47,49 +48,49 @@ DEFAULT_INSTALLDIR=distutils.sysconfig.get_python_lib() DEFAULT_PIMPDATABASE_FMT="http://www.python.org/packman/version-%s/%s-%s-%s-%s-%s.plist" def getDefaultDatabase(experimental=False): - if experimental: - status = "exp" - else: - status = "prod" - - major, minor, micro, state, extra = sys.version_info - pyvers = '%d.%d' % (major, minor) - if state != 'final': - pyvers = pyvers + '%s%d' % (state, extra) - - longplatform = distutils.util.get_platform() - osname, release, machine = longplatform.split('-') - # For some platforms we may want to differentiate between - # installation types - if osname == 'darwin': - if sys.prefix.startswith('/System/Library/Frameworks/Python.framework'): - osname = 'darwin_apple' - elif sys.prefix.startswith('/Library/Frameworks/Python.framework'): - osname = 'darwin_macpython' - # Otherwise we don't know... - # Now we try various URLs by playing with the release string. - # We remove numbers off the end until we find a match. - rel = release - while True: - url = DEFAULT_PIMPDATABASE_FMT % (PIMP_VERSION, status, pyvers, osname, rel, machine) - try: - urllib2.urlopen(url) - except urllib2.HTTPError, arg: - pass - else: - break - if not rel: - # We're out of version numbers to try. Use the - # full release number, this will give a reasonable - # error message later - url = DEFAULT_PIMPDATABASE_FMT % (PIMP_VERSION, status, pyvers, osname, release, machine) - break - idx = rel.rfind('.') - if idx < 0: - rel = '' - else: - rel = rel[:idx] - return url + if experimental: + status = "exp" + else: + status = "prod" + + major, minor, micro, state, extra = sys.version_info + pyvers = '%d.%d' % (major, minor) + if state != 'final': + pyvers = pyvers + '%s%d' % (state, extra) + + longplatform = distutils.util.get_platform() + osname, release, machine = longplatform.split('-') + # For some platforms we may want to differentiate between + # installation types + if osname == 'darwin': + if sys.prefix.startswith('/System/Library/Frameworks/Python.framework'): + osname = 'darwin_apple' + elif sys.prefix.startswith('/Library/Frameworks/Python.framework'): + osname = 'darwin_macpython' + # Otherwise we don't know... + # Now we try various URLs by playing with the release string. + # We remove numbers off the end until we find a match. + rel = release + while True: + url = DEFAULT_PIMPDATABASE_FMT % (PIMP_VERSION, status, pyvers, osname, rel, machine) + try: + urllib2.urlopen(url) + except urllib2.HTTPError, arg: + pass + else: + break + if not rel: + # We're out of version numbers to try. Use the + # full release number, this will give a reasonable + # error message later + url = DEFAULT_PIMPDATABASE_FMT % (PIMP_VERSION, status, pyvers, osname, release, machine) + break + idx = rel.rfind('.') + if idx < 0: + rel = '' + else: + rel = rel[:idx] + return url def _cmd(output, dir, *cmditems): """Internal routine to run a shell command in a given directory.""" @@ -109,6 +110,68 @@ def _cmd(output, dir, *cmditems): output.write(line) return child.wait() +class PimpDownloader: + """Abstract base class - Downloader for archives""" + + def __init__(self, argument, + dir="", + watcher=None): + self.argument = argument + self._dir = dir + self._watcher = watcher + + def download(self, url, filename, output=None): + return None + + def update(self, str): + if self._watcher: + return self._watcher.update(str) + return True + +class PimpCurlDownloader(PimpDownloader): + + def download(self, url, filename, output=None): + self.update("Downloading %s..." % url) + exitstatus = _cmd(output, self._dir, + "curl", + "--output", filename, + url) + self.update("Downloading %s: finished" % url) + return (not exitstatus) + +class PimpUrllibDownloader(PimpDownloader): + + def download(self, url, filename, output=None): + output = open(filename, 'wb') + self.update("Downloading %s: opening connection" % url) + keepgoing = True + download = urllib2.urlopen(url) + if download.headers.has_key("content-length"): + length = long(download.headers['content-length']) + else: + length = -1 + + data = download.read(4096) #read 4K at a time + dlsize = 0 + lasttime = 0 + while keepgoing: + dlsize = dlsize + len(data) + if len(data) == 0: + #this is our exit condition + break + output.write(data) + if int(time.time()) != lasttime: + # Update at most once per second + lasttime = int(time.time()) + if length == -1: + keepgoing = self.update("Downloading %s: %d bytes..." % (url, dlsize)) + else: + keepgoing = self.update("Downloading %s: %d%% (%d bytes)..." % (url, int(100.0*dlsize/length), dlsize)) + data = download.read(4096) + if keepgoing: + self.update("Downloading %s: finished" % url) + return keepgoing + class PimpUnpacker: """Abstract base class - Unpacker for archives""" @@ -116,16 +179,23 @@ class PimpUnpacker: def __init__(self, argument, dir="", - renames=[]): + renames=[], + watcher=None): self.argument = argument if renames and not self._can_rename: raise RuntimeError, "This unpacker cannot rename files" self._dir = dir self._renames = renames + self._watcher = watcher def unpack(self, archive, output=None, package=None): return None + def update(self, str): + if self._watcher: + return self._watcher.update(str) + return True + class PimpCommandUnpacker(PimpUnpacker): """Unpack archives by calling a Unix utility""" @@ -173,7 +243,9 @@ class PimpTarUnpacker(PimpUnpacker): #print '????', member.name for member in members: if member in skip: + self.update("Skipping %s" % member.name) continue + self.update("Extracting %s" % member.name) tf.extract(member, self._dir) if skip: names = [member.name for member in skip if member.name[-1] != '/'] @@ -214,6 +286,10 @@ class PimpPreferences: self.downloadDir = downloadDir self.buildDir = buildDir self.pimpDatabase = pimpDatabase + self.watcher = None + + def setWatcher(self, watcher): + self.watcher = watcher def setInstallDir(self, installDir=None): if installDir: @@ -282,11 +358,18 @@ class PimpDatabase: def __init__(self, prefs): self._packages = [] self.preferences = prefs + self._url = "" self._urllist = [] self._version = "" self._maintainer = "" self._description = "" + # Accessor functions + def url(self): return self._url + def version(self): return self._version + def maintainer(self): return self._maintainer + def description(self): return self._description + def close(self): """Clean up""" self._packages = [] @@ -301,24 +384,25 @@ class PimpDatabase: return self._urllist.append(url) fp = urllib2.urlopen(url).fp - dict = plistlib.Plist.fromFile(fp) + plistdata = plistlib.Plist.fromFile(fp) # Test here for Pimp version, etc if included: - version = dict.get('Version') + version = plistdata.get('Version') if version and version > self._version: sys.stderr.write("Warning: included database %s is for pimp version %s\n" % (url, version)) else: - self._version = dict.get('Version') + self._version = plistdata.get('Version') if not self._version: sys.stderr.write("Warning: database has no Version information\n") elif self._version > PIMP_VERSION: sys.stderr.write("Warning: database version %s newer than pimp version %s\n" % (self._version, PIMP_VERSION)) - self._maintainer = dict.get('Maintainer', '') - self._description = dict.get('Description', '').strip() - self._appendPackages(dict['Packages']) - others = dict.get('Include', []) + self._maintainer = plistdata.get('Maintainer', '') + self._description = plistdata.get('Description', '').strip() + self._url = url + self._appendPackages(plistdata['Packages']) + others = plistdata.get('Include', []) for url in others: self.appendURL(url, included=1) @@ -361,13 +445,13 @@ class PimpDatabase: packages = [] for pkg in self._packages: packages.append(pkg.dump()) - dict = { + plistdata = { 'Version': self._version, 'Maintainer': self._maintainer, 'Description': self._description, 'Packages': packages } - plist = plistlib.Plist(**dict) + plist = plistlib.Plist(**plistdata) plist.write(pathOrFile) def find(self, ident): @@ -428,13 +512,13 @@ ALLOWED_KEYS = [ class PimpPackage: """Class representing a single package.""" - def __init__(self, db, dict): + def __init__(self, db, plistdata): self._db = db - name = dict["Name"] - for k in dict.keys(): + name = plistdata["Name"] + for k in plistdata.keys(): if not k in ALLOWED_KEYS: sys.stderr.write("Warning: %s: unknown key %s\n" % (name, k)) - self._dict = dict + self._dict = plistdata def __getitem__(self, key): return self._dict[key] @@ -582,10 +666,10 @@ class PimpPackage: if not self._archiveOK(): if scheme == 'manual': return "Please download package manually and save as %s" % self.archiveFilename - if _cmd(output, self._db.preferences.downloadDir, - "curl", - "--output", self.archiveFilename, - self._dict['Download-URL']): + downloader = PimpUrllibDownloader(None, self._db.preferences.downloadDir, + watcher=self._db.preferences.watcher) + if not downloader.download(self._dict['Download-URL'], + self.archiveFilename, output): return "download command failed" if not os.path.exists(self.archiveFilename) and not NO_EXECUTE: return "archive not found after download" @@ -614,7 +698,8 @@ class PimpPackage: else: return "unknown extension for archive file: %s" % filename self.basename = filename[:-len(ext)] - unpacker = unpackerClass(arg, dir=self._db.preferences.buildDir) + unpacker = unpackerClass(arg, dir=self._db.preferences.buildDir, + watcher=self._db.preferences.watcher) rv = unpacker.unpack(self.archiveFilename, output=output) if rv: return rv @@ -823,7 +908,7 @@ class PimpInstaller: def _addPackages(self, packages): for package in packages: if not package in self._todo: - self._todo.insert(0, package) + self._todo.append(package) def _prepareInstall(self, package, force=0, recursive=1): """Internal routine, recursive engine for prepareInstall. @@ -844,7 +929,7 @@ class PimpInstaller: prereqs = package.prerequisites() for pkg, descr in prereqs: if pkg: - self._prepareInstall(pkg, force, recursive) + self._prepareInstall(pkg, False, recursive) else: self._curmessages.append("Problem with dependency: %s" % descr) @@ -879,10 +964,12 @@ class PimpInstaller: -def _run(mode, verbose, force, args, prefargs): +def _run(mode, verbose, force, args, prefargs, watcher): """Engine for the main program""" prefs = PimpPreferences(**prefargs) + if watcher: + prefs.setWatcher(watcher) rv = prefs.check() if rv: sys.stdout.write(rv) @@ -979,6 +1066,11 @@ def main(): print " -u url URL for database" sys.exit(1) + class _Watcher: + def update(self, msg): + sys.stderr.write(msg + '\r') + return 1 + try: opts, args = getopt.getopt(sys.argv[1:], "slifvdD:Vu:") except getopt.GetoptError: @@ -989,6 +1081,7 @@ def main(): force = 0 verbose = 0 prefargs = {} + watcher = None for o, a in opts: if o == '-s': if mode: @@ -1012,6 +1105,7 @@ def main(): force = 1 if o == '-v': verbose = 1 + watcher = _Watcher() if o == '-D': prefargs['installDir'] = a if o == '-u': @@ -1021,7 +1115,7 @@ def main(): if mode == 'version': print 'Pimp version %s; module name is %s' % (PIMP_VERSION, __name__) else: - _run(mode, verbose, force, args, prefargs) + _run(mode, verbose, force, args, prefargs, watcher) # Finally, try to update ourselves to a newer version. # If the end-user updates pimp through pimp the new version