Changes by Eric Raymond:

1. Generate a correct Content-Length header visible through the info() method
   if a request to open an FTP URL gets a length in the response to RETR.

2. Take a third argument to urlretrieve() that makes it possible to progress-
   meter an urlretrieve call (this is what I needed the above change for).
   See the second patch band below for details.

3. To avoid spurious errors, I commented out the gopher test.  The target
   document no longer exists.

diff --git a/Lib/urllib.py b/Lib/urllib.py
index 2a9087aa53bbb6ee9d9b1bd7c5b8da0040647ad4..62e5c7f5298bc4626e0dfeeeb17c7f75fb95e4ed 100644 (file)
--- a/Lib/urllib.py
+++ b/Lib/urllib.py
@@ -59,14 +59,11 @@ def urlopen(url, data=None):
                return _urlopener.open(url)
        else:
                return _urlopener.open(url, data)
-def urlretrieve(url, filename=None):
+def urlretrieve(url, filename=None, reporthook=None):
        global _urlopener
        if not _urlopener:
                _urlopener = FancyURLopener()
-       if filename:
-               return _urlopener.retrieve(url, filename)
-       else:
-               return _urlopener.retrieve(url)
+        return _urlopener.retrieve(url, filename, reporthook)
 def urlcleanup():
        if _urlopener:
                _urlopener.cleanup()
@@ -171,7 +168,7 @@ class URLopener:
        # External interface
        # retrieve(url) returns (filename, None) for a local object
        # or (tempfilename, headers) for a remote object
-       def retrieve(self, url, filename=None):
+       def retrieve(self, url, filename=None, reporthook=None):
                url = unwrap(url)
                if self.tempcache and self.tempcache.has_key(url):
                        return self.tempcache[url]
@@ -200,10 +197,21 @@ class URLopener:
                        self.tempcache[url] = result
                tfp = open(filename, 'wb')
                bs = 1024*8
+               size = -1
+               blocknum = 1
+               if reporthook:
+                       if headers.has_key("content-length"):
+                            size = int(headers["Content-Length"])
+                       reporthook(0, bs, size)
                block = fp.read(bs)
+               if reporthook:
+                       reporthook(1, bs, size)
                while block:
                        tfp.write(block)
                        block = fp.read(bs)
+                       blocknum = blocknum + 1
+                       if reporthook:
+                               reporthook(blocknum, bs, size)
                fp.close()
                tfp.close()
                del fp
@@ -366,9 +374,14 @@ class URLopener:
                                if string.lower(attr) == 'type' and \
                                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
                                        type = string.upper(value)
-                       return addinfourl(
-                               self.ftpcache[key].retrfile(file, type),
-                               noheaders(), "ftp:" + url)
+                        (fp, retrlen) = self.ftpcache[key].retrfile(file, type)
+                        if retrlen >= 0:
+                            import mimetools, StringIO
+                            headers = mimetools.Message(StringIO.StringIO(
+                                'Content-Length: %d\n' % retrlen))
+                        else:
+                            headers = noheaders()
+                        return addinfourl(fp, headers, "ftp:" + url)
                except ftperrors(), msg:
                        raise IOError, ('ftp error', msg), sys.exc_info()[2]
 
@@ -574,7 +587,7 @@ class ftpwrapper:
                        # Try to retrieve as a file
                        try:
                                cmd = 'RETR ' + file
-                               conn = self.ftp.transfercmd(cmd)
+                               conn = self.ftp.ntransfercmd(cmd)
                        except ftplib.error_perm, reason:
                                if reason[:3] != '550':
                                        raise IOError, ('ftp error', reason), \
@@ -585,9 +598,10 @@ class ftpwrapper:
                        # Try a directory listing
                        if file: cmd = 'LIST ' + file
                        else: cmd = 'LIST'
-                       conn = self.ftp.transfercmd(cmd)
+                       conn = self.ftp.ntransfercmd(cmd)
                self.busy = 1
-               return addclosehook(conn.makefile('rb'), self.endtransfer)
+                # Pass back both a suitably decorated object and a retrieval length
+               return (addclosehook(conn[0].makefile('rb'), self.endtransfer), conn[1])
        def endtransfer(self):
                if not self.busy:
                        return
@@ -977,6 +991,10 @@ def test1():
        print round(t1 - t0, 3), 'sec'
 
 
+def reporthook(blocknum, blocksize, totalsize):
+    # Report during remote transfers
+    print "Block number: %d, Block size: %d, Total size: %d" % (blocknum, blocksize, totalsize)
+
 # Test program
 def test(args=[]):
        if not args:
@@ -985,13 +1003,13 @@ def test(args=[]):
                        'file:/etc/passwd',
                        'file://localhost/etc/passwd',
                        'ftp://ftp.python.org/etc/passwd',
-                       'gopher://gopher.micro.umn.edu/1/',
+##                     'gopher://gopher.micro.umn.edu/1/',
                        'http://www.python.org/index.html',
                        ]
        try:
                for url in args:
                        print '-'*10, url, '-'*10
-                       fn, h = urlretrieve(url)
+                       fn, h = urlretrieve(url, None, reporthook)
                        print fn, h
                        if h:
                                print '======'