[2.7] bpo-27973 - Fix for urllib.urlretrieve() failing on second ftp transfer (#1040)

* bpo-27973: Fix urllib.urlretrieve failing on subsequent ftp transfers from the same host.

* bpo-35411: Skip test_urllibnet FTP tests on Travis CI.

diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py
index 3f2d8dcd43b0c714e971214d934bcc1e717a2df3..df118dc75d1b3f3b6b55f3fc38b6104d777fbf60 100644 (file)
--- a/Lib/test/test_urllibnet.py
+++ b/Lib/test/test_urllibnet.py
@@ -1,5 +1,7 @@
+import tempfile
 import unittest
 from test import test_support
+from test.test_urllib2net import skip_ftp_test_on_travis
 
 import socket
 import urllib
@@ -213,6 +215,40 @@ class urlopen_HttpsTests(unittest.TestCase):
         self.assertIn("Python", response.read())
 
 
+class urlopen_FTPTest(unittest.TestCase):
+    FTP_TEST_FILE = 'ftp://www.pythontest.net/README'
+    NUM_FTP_RETRIEVES = 3
+
+    @skip_ftp_test_on_travis
+    def test_multiple_ftp_retrieves(self):
+
+        with test_support.transient_internet(self.FTP_TEST_FILE):
+            try:
+                for _ in range(self.NUM_FTP_RETRIEVES):
+                    with tempfile.NamedTemporaryFile() as fp:
+                        urllib.FancyURLopener().retrieve(self.FTP_TEST_FILE, fp.name)
+            except IOError as e:
+                self.fail("Failed FTP retrieve while accessing ftp url "
+                          "multiple times.\n Error message was : %s" % e)
+
+    @skip_ftp_test_on_travis
+    def test_multiple_ftp_urlopen_same_host(self):
+        with test_support.transient_internet(self.FTP_TEST_FILE):
+            ftp_fds_to_close = []
+            try:
+                for _ in range(self.NUM_FTP_RETRIEVES):
+                    fd = urllib.urlopen(self.FTP_TEST_FILE)
+                    # test ftp open without closing fd as a supported scenario.
+                    ftp_fds_to_close.append(fd)
+            except IOError as e:
+                self.fail("Failed FTP binary file open. "
+                          "Error message was: %s" % e)
+            finally:
+                # close the open fds
+                for fd in ftp_fds_to_close:
+                    fd.close()
+
+
 def test_main():
     test_support.requires('network')
     with test_support.check_py3k_warnings(
@@ -220,7 +256,8 @@ def test_main():
         test_support.run_unittest(URLTimeoutTest,
                                   urlopenNetworkTests,
                                   urlretrieveNetworkTests,
-                                  urlopen_HttpsTests)
+                                  urlopen_HttpsTests,
+                                  urlopen_FTPTest)
 
 if __name__ == "__main__":
     test_main()

diff --git a/Lib/urllib.py b/Lib/urllib.py
index 156879dd0a140b638f9545eeb098b3bd71d266a8..87bcd94db5f058fa0412f40036cd8938171ffe91 100644 (file)
--- a/Lib/urllib.py
+++ b/Lib/urllib.py
@@ -934,7 +934,13 @@ class ftpwrapper:
         return (ftpobj, retrlen)
 
     def endtransfer(self):
+        if not self.busy:
+            return
         self.busy = 0
+        try:
+            self.ftp.voidresp()
+        except ftperrors():
+            pass
 
     def close(self):
         self.keepalive = False

diff --git a/Misc/NEWS.d/next/Library/2019-12-30-07-59-34.bpo-27973.mgWXH1.rst b/Misc/NEWS.d/next/Library/2019-12-30-07-59-34.bpo-27973.mgWXH1.rst
new file mode 100644 (file)
index 0000000..d50f483
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-12-30-07-59-34.bpo-27973.mgWXH1.rst
@@ -0,0 +1,2 @@
+Fix urllib.urlretrieve failing on subsequent ftp transfers from the same
+host.