From: Tim Peters <tim.peters@gmail.com>
Date: Mon, 20 Mar 2006 06:06:07 +0000 (+0000)
Subject: The new fetch_data_files.py downloads all the input data files
X-Git-Tag: v2.4.3c1~13
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=099e7932cf5aae6713c20bf3cf3b21465aabe171;p=thirdparty%2FPython%2Fcpython.git

The new fetch_data_files.py downloads all the input data files
used by encoding tests.  Fiddled the Windows buildbot helper
scripts to invoke this if needed.  Note that this isn't needed
on the trunk (the encoding tests download input files automatically
in 2.5).
---

diff --git a/Tools/buildbot/README.txt b/Tools/buildbot/README.txt
new file mode 100644
index 000000000000..7c6e05af71e3
--- /dev/null
+++ b/Tools/buildbot/README.txt
@@ -0,0 +1,14 @@
+Helpers used by buildbot-driven core Python testing.
+
+external.bat
+build.bat
+test.bat
+clean.bat
+    On Windows, these scripts are executed by the code sent
+    from the buildbot master to the slaves.
+
+fetch_data_files.py
+    Download all the input files various encoding tests want.  This is
+    used by build.bat on Windows (but could be used on any platform).
+    Note that in Python >= 2.5, the encoding tests download input files
+    automatically.
diff --git a/Tools/buildbot/build.bat b/Tools/buildbot/build.bat
index e3b77be2f12b..ef9b09298310 100644
--- a/Tools/buildbot/build.bat
+++ b/Tools/buildbot/build.bat
@@ -1,4 +1,7 @@
 @rem Used by the buildbot "compile" step.
 cmd /c Tools\buildbot\external.bat
 call "%VS71COMNTOOLS%vsvars32.bat"
-devenv.com /useenv /build Debug PCbuild\pcbuild.sln
+cd PCbuild
+devenv.com /useenv /build Debug pcbuild.sln
+@rem Fetch encoding test files.  Note that python_d needs to be built first.
+if not exist BIG5.TXT python_d.exe ..\Tools\buildbot\fetch_data_files.py
\ No newline at end of file
diff --git a/Tools/buildbot/external.bat b/Tools/buildbot/external.bat
index 5dd11140b761..fff0af23a5e8 100644
--- a/Tools/buildbot/external.bat
+++ b/Tools/buildbot/external.bat
@@ -4,5 +4,4 @@
 cd ..
 
 @rem bzip
-if not exist bzip2-1.0.3 svn export http://svn.python.org/projects/external/bzip2-1.0.3
-
+if not exist bzip2-1.0.3 svn export http://svn.python.org/projects/external/bzip2-1.0.3
\ No newline at end of file
diff --git a/Tools/buildbot/fetch_data_files.py b/Tools/buildbot/fetch_data_files.py
new file mode 100644
index 000000000000..f4b6096ebe3f
--- /dev/null
+++ b/Tools/buildbot/fetch_data_files.py
@@ -0,0 +1,61 @@
+"""A helper to download input files needed by assorted encoding tests.
+
+fetch_data_files.py [directory]
+
+Files are downloaded to directory `directory`.  If a directory isn't given,
+it defaults to the current directory (.).
+"""
+
+DATA_URLS = """
+    http://people.freebsd.org/~perky/i18n/BIG5HKSCS.TXT
+    http://people.freebsd.org/~perky/i18n/EUC-CN.TXT
+    http://people.freebsd.org/~perky/i18n/EUC-JISX0213.TXT
+    http://people.freebsd.org/~perky/i18n/EUC-JP.TXT
+    http://people.freebsd.org/~perky/i18n/EUC-KR.TXT
+    http://people.freebsd.org/~perky/i18n/SHIFT_JISX0213.TXT
+
+    http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP932.TXT
+    http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP936.TXT
+    http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP949.TXT
+    http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP950.TXT
+
+    http://www.unicode.org/Public/3.2-Update/NormalizationTest-3.2.0.txt
+
+    http://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/JIS/SHIFTJIS.TXT
+    http://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/KSC/JOHAB.TXT
+    http://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/OTHER/BIG5.TXT
+"""
+
+# Adapted from test_support.open_urlresource() in Python 2.5.
+# Fetch the file give by `url` off the web, and store it in directory
+# `directory`.  The file name is extracted from the last URL component.
+# If the file already exists, it's not fetched again.
+def fetch_file_from_url(url, directory):
+    import urllib, urlparse
+    import os.path
+
+    filename = urlparse.urlparse(url)[2].split('/')[-1] # '/': it's a URL!
+    target = os.path.join(directory, filename)
+    if os.path.exists(target):
+        print "\tskipping %r -- already exists" % target
+    else:
+        print "\tfetching %s ..." % url
+        urllib.urlretrieve(url, target)
+
+def main(urls, directory):
+    print "Downloading data files to %r" % directory
+    for url in urls.split():
+        fetch_file_from_url(url, directory)
+
+if __name__ == "__main__":
+    import sys
+
+    n = len(sys.argv)
+    if n == 1:
+        directory = "."
+    elif n == 2:
+        directory = sys.argv[1]
+    else:
+        raise ValueError("no more than one argument allowed")
+
+    main(DATA_URLS, directory)