Issue 1267, continued.

Additional patch by Christian Heimes to deal more cleanly with the
FILE* vs file-descriptor issues.
I cleaned up his code a bit, and moved the lseek() call into import.c.

diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst
index e48056c8e15ab241c9aeeaa803dca1fbafcaba4f..1855688a2cc611cbdefe68ba6334ba24d45a7d72 100644 (file)
--- a/Doc/c-api/concrete.rst
+++ b/Doc/c-api/concrete.rst
@@ -2410,31 +2410,23 @@ change in future releases of Python.
    :ctype:`PyFileObject`.
 
 
-.. cfunction:: PyObject* PyFile_FromString(char *filename, char *mode)
+.. cfunction:: PyFile_FromFd(int fd, char *name, char *mode, int buffering, char *encoding, char *newline)
 
-   .. index:: single: fopen()
-
-   On success, return a new file object that is opened on the file given by
-   *filename*, with a file mode given by *mode*, where *mode* has the same
-   semantics as the standard C routine :cfunc:`fopen`.  On failure, return *NULL*.
-
-
-.. cfunction:: PyObject* PyFile_FromFile(FILE *fp, char *name, char *mode, int (*close)(FILE*))
+   Create a new :ctype:`PyFileObject` from the file descriptor of an already
+   opened file *fd*. The arguments *name*, *encoding* and *newline* can be
+   *NULL* as well as buffering can be *-1* to use the defaults. Return *NULL* on
+   failure.
 
-   Create a new :ctype:`PyFileObject` from the already-open standard C file
-   pointer, *fp*.  The function *close* will be called when the file should be
-   closed.  Return *NULL* on failure.
+   .. warning::
 
-.. cfunction:: PyFile_FromFileEx(FILE *fp, char *name, char *mode, int (*close)(FILE *), int buffering, char *encoding, char *newline)
+     Take care when you are mixing streams and descriptors! For more 
+     information, see `GNU C Library
+     <http://www.gnu.org/software/libc/manual/html_node/Stream_002fDescriptor-Precautions.html#Stream_002fDescriptor-Precautions>`_.
 
-   Create a new :ctype:`PyFileObject` from the already-open standard C file
-   pointer, *fp*. The functions works similar to *PyFile_FromFile* but takes
-   optional arguments for *buffering*, *encoding* and *newline*. Use -1 resp.
-   *NULL* for default values.
 
-.. cfunction:: FILE* PyFile_AsFile(PyObject *p)
+.. cfunction:: int PyObject_AsFileDescriptor(PyObject *p)
 
-   Return the file object associated with *p* as a :ctype:`FILE\*`.
+   Return the file descriptor associated with *p* as an :ctype:`int`.
 
 
 .. cfunction:: PyObject* PyFile_GetLine(PyObject *p, int n)

diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst
index 4da1606ea18c4b6120e9c5bc85f6eb4a71b897ab..ceb2cf373627c44bd4b012802c6288ffc4dca229 100644 (file)
--- a/Doc/reference/introduction.rst
+++ b/Doc/reference/introduction.rst
@@ -60,7 +60,7 @@ Python for .NET
    This implementation actually uses the CPython implementation, but is a managed
    .NET application and makes .NET libraries available.  This was created by Brian
    Lloyd.  For more information, see the `Python for .NET home page
-   <http://www.zope.org/Members/Brian/PythonNet>`_.
+   <http://pythonnet.sourceforge.net>`_.
 
 IronPython
    An alternate Python for .NET.  Unlike Python.NET, this is a complete Python

diff --git a/Include/fileobject.h b/Include/fileobject.h
index b65d9847d45cb9e7dc9b83925b448c0cf16a19f7..acb8c6ddc19c056947e531b423e6c887a309c868 100644 (file)
--- a/Include/fileobject.h
+++ b/Include/fileobject.h
@@ -8,10 +8,7 @@ extern "C" {
 
 #define PY_STDIOTEXTMODE "b"
 
-PyAPI_FUNC(PyObject *) PyFile_FromFile(FILE *, char *, char *, int (*)(FILE*));
-PyAPI_FUNC(PyObject *) PyFile_FromFileEx(FILE *, char *, char *,
-                                        int (*)(FILE *), int, char *,
-                                        char *);
+PyAPI_FUNC(PyObject *) PyFile_FromFd(int, char *, char *, int, char *, char *);
 PyAPI_FUNC(PyObject *) PyFile_GetLine(PyObject *, int);
 PyAPI_FUNC(int) PyFile_WriteObject(PyObject *, PyObject *, int);
 PyAPI_FUNC(int) PyFile_WriteString(const char *, PyObject *);

diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py
index 87efc33415edcc7acf28d2f5e59e468c7cca3747..268a4b77e6e4ce7d67286640efb57c3e6ecdb272 100644 (file)
--- a/Lib/test/test_imp.py
+++ b/Lib/test/test_imp.py
@@ -44,6 +44,23 @@ class ImportTests(unittest.TestCase):
         fd = imp.find_module("heapq")[0]
         self.assertEqual(fd.encoding, "iso-8859-1")
 
+    def test_issue1267(self):
+        fp, filename, info  = imp.find_module("pydoc")
+        self.assertNotEqual(fp, None)
+        self.assertEqual(fp.encoding, "iso-8859-1")
+        self.assertEqual(fp.tell(), 0)
+        self.assertEqual(fp.readline(), '#!/usr/bin/env python\n')
+        fp.close()
+
+        fp, filename, info = imp.find_module("tokenize")
+        self.assertNotEqual(fp, None)
+        self.assertEqual(fp.encoding, "utf-8")
+        self.assertEqual(fp.tell(), 0)
+        self.assertEqual(fp.readline(),
+                         '"""Tokenization help for Python programs.\n')
+        fp.close()
+
+
 def test_main():
     test_support.run_unittest(
                 LockTests,

diff --git a/Misc/NEWS b/Misc/NEWS
index 749b94c6aeb306358da90688d70917e341b644af..86c5a59fe62852428b13a69b5d7c53200593f23e 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -8,6 +8,18 @@ What's New in Python 3.0a2?
 
 *Unreleased*
 
+Core and Builtins
+-----------------
+
+- Replaced `PyFile_FromFile()` with `PyFile_FromFd(fd, name. mode, buffer, 
+  encoding, newline)`
+
+- Fixed `imp.find_module()` to obey the -*- coding: -*- header.
+
+- Changed `__file__` and `co_filename` to unicode. The path names are decoded
+  with `Py_FileSystemDefaultEncoding` and a new API method 
+  `PyUnicode_DecodeFSDefault(char*)` was added.
+
 Extension Modules
 -----------------
 

diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 647ea3ef888a1a79c648e2a7db05bebea6ebd95a..e0de9612f24ca07c93826c64a77bf7fa86a0a0a9 100644 (file)
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -5386,11 +5386,18 @@ static PyObject *
 posix_tmpfile(PyObject *self, PyObject *noargs)
 {
     FILE *fp;
+    int fd;
 
     fp = tmpfile();
     if (fp == NULL)
         return posix_error();
-    return PyFile_FromFile(fp, "<tmpfile>", "w+b", fclose);
+    fd = fileno(fp);
+    if (fd != -1)
+       fd = dup(fd);
+    fclose(fp);
+    if (fd == -1)
+        return posix_error();
+    return PyFile_FromFd(fd, "<tmpfile>", "w+b", -1, NULL, NULL);
 }
 #endif
 

diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
index 18d0f57f8809872967d17cfe0592bf52ca2d2692..db475cd3d37ea4da0967643ed9da8f7bb1bb32bf 100644 (file)
--- a/Objects/bytesobject.c
+++ b/Objects/bytesobject.c
@@ -1214,7 +1214,7 @@ _bytes_tailmatch(PyBytesObject *self, PyObject *substr, Py_ssize_t start,
     Py_ssize_t len = PyBytes_GET_SIZE(self);
     const char* str;
     Py_buffer vsubstr;
-    int rv;
+    int rv = 0;
 
     str = PyBytes_AS_STRING(self);
 
@@ -1226,13 +1226,11 @@ _bytes_tailmatch(PyBytesObject *self, PyObject *substr, Py_ssize_t start,
     if (direction < 0) {
         /* startswith */
         if (start+vsubstr.len > len) {
-            rv = 0;
             goto done;
         }
     } else {
         /* endswith */
         if (end-start < vsubstr.len || start > len) {
-            rv = 0;
             goto done;
         }
 

diff --git a/Objects/fileobject.c b/Objects/fileobject.c
index b6d200d0c21068e6110c23250c3e1cc29191c4eb..b4abac582ac8bbf2e5e0ad99055c16b0a1e1984e 100644 (file)
--- a/Objects/fileobject.c
+++ b/Objects/fileobject.c
@@ -26,22 +26,16 @@ extern "C" {
 /* External C interface */
 
 PyObject *
-PyFile_FromFile(FILE *fp, char *name, char *mode, int (*close)(FILE *))
+PyFile_FromFd(int fd, char *name, char *mode, int buffering, char *encoding,
+             char *newline)
 {
-       return PyFile_FromFileEx(fp, name, mode, close, -1, NULL, NULL);
-}
-
-PyObject *
-PyFile_FromFileEx(FILE *fp, char *name, char *mode, int (*close)(FILE *),
-              int buffering, char *encoding, char *newline)
-{
-       PyObject *io, *stream, *nameobj=NULL;
+       PyObject *io, *stream, *nameobj = NULL;
 
        io = PyImport_ImportModule("io");
        if (io == NULL)
                return NULL;
-       stream = PyObject_CallMethod(io, "open", "isiss", fileno(fp), mode,
-                                   buffering, encoding, newline);
+       stream = PyObject_CallMethod(io, "open", "isiss", fd, mode,
+                                    buffering, encoding, newline);
        Py_DECREF(io);
        if (stream == NULL)
                return NULL;

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 8c24cf28decee8ba045798d14fa5dd75414cf9ce..5b3fd9e61d3e62bf57e6df4e7bd63c42f72b3e4e 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1602,40 +1602,44 @@ PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset)
 }
 #endif
 
-/* Get -*- encoding -*- from a Python file
+/* Get -*- encoding -*- from a Python file.
 
    PyTokenizer_FindEncoding returns NULL when it can't find the encoding in
    the first or second line of the file (in which case the encoding 
    should be assumed to be PyUnicode_GetDefaultEncoding()).
 
-   The char * returned was malloc'ed from PyMem_MALLOC() and thus must be freed
-   when no longer needed.
+   The char * returned is malloc'ed via PyMem_MALLOC() and thus must be freed
+   by the caller.
 */
 char *
-PyTokenizer_FindEncoding(FILE *fp) {
+PyTokenizer_FindEncoding(int fd)
+{
        struct tok_state *tok;
-       char *p_start=NULL, *p_end=NULL, *encoding=NULL;
+       FILE *fp;
+       char *p_start =NULL , *p_end =NULL , *encoding = NULL;
 
-       if ((tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL)) == NULL) {
-               /* lseek() usage is on purpose; see note later in code. */
-               lseek(fileno(fp), 0, 0);
+       fd = dup(fd);
+       if (fd < 0) {
+               return NULL;
+       }
+       fp = fdopen(fd, "r");
+       if (fp == NULL) {
+               return NULL;
+       }
+       tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL);
+       if (tok == NULL) {
+               fclose(fp);
                return NULL;
        }
-       while(((tok->lineno < 2) && (tok->done == E_OK))) {
+       while (tok->lineno < 2 && tok->done == E_OK) {
                PyTokenizer_Get(tok, &p_start, &p_end);
        }
-
-       /* lseek() must be used instead of fseek()/rewind() as those fail on
-          OS X 10.4 to properly seek back to the beginning when reading from
-          the file descriptor instead of the file pointer.  */
-       lseek(fileno(fp), 0, 0);
-
+       fclose(fp);
        if (tok->encoding) {
             encoding = (char *)PyMem_MALLOC(strlen(tok->encoding) + 1);
             strcpy(encoding, tok->encoding);
         }
        PyTokenizer_Free(tok);
-
        return encoding;
 }
 

diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index a66d78e7d97b44a2f1918fefed174bc34e414be3..c45dea1a8b0b87b1025da238629198cc0a6e6df4 100644 (file)
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -67,7 +67,7 @@ extern void PyTokenizer_Free(struct tok_state *);
 extern int PyTokenizer_Get(struct tok_state *, char **, char **);
 extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok, 
                                          int len, int *offset);
-extern char * PyTokenizer_FindEncoding(FILE *fp);
+extern char * PyTokenizer_FindEncoding(int);
 
 #ifdef __cplusplus
 }

diff --git a/Python/import.c b/Python/import.c
index 2a316caec9eed3c20475a22bd2d13d0f1923b3da..24935547e6d5737570454e32a99e460af803b42c 100644 (file)
--- a/Python/import.c
+++ b/Python/import.c
@@ -92,7 +92,7 @@ static PyObject *extensions = NULL;
 extern struct _inittab _PyImport_Inittab[];
 
 /* Method from Parser/tokenizer.c */
-extern char * PyTokenizer_FindEncoding(FILE *fp);
+extern char * PyTokenizer_FindEncoding(int);
 
 struct _inittab *PyImport_Inittab = _PyImport_Inittab;
 
@@ -2561,6 +2561,7 @@ call_find_module(char *name, PyObject *path)
        struct filedescr *fdp;
        char pathname[MAXPATHLEN+1];
        FILE *fp = NULL;
+       int fd = -1;
        char *found_encoding = NULL;
        char *encoding = NULL;
 
@@ -2571,17 +2572,25 @@ call_find_module(char *name, PyObject *path)
        if (fdp == NULL)
                return NULL;
        if (fp != NULL) {
+               fd = fileno(fp);
+               if (fd != -1)
+                       fd = dup(fd);
+               fclose(fp);
+               fp = NULL;
+       }
+       if (fd != -1) {
                if (strchr(fdp->mode, 'b') == NULL) {
                        /* PyTokenizer_FindEncoding() returns PyMem_MALLOC'ed
                           memory. */
-                       found_encoding = PyTokenizer_FindEncoding(fp);
+                       found_encoding = PyTokenizer_FindEncoding(fd);
+                       lseek(fd, 0, 0); /* Reset position */
                        encoding = (found_encoding != NULL) ? found_encoding :
                                   (char*)PyUnicode_GetDefaultEncoding();
                }
-               fob = PyFile_FromFileEx(fp, pathname, fdp->mode, fclose, -1,
+               fob = PyFile_FromFd(fd, pathname, fdp->mode, -1,
                                        (char*)encoding, NULL);
                if (fob == NULL) {
-                       fclose(fp);
+                       close(fd);
                        PyMem_FREE(found_encoding);
                        return NULL;
                }

diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index f641547afd5b8c337e85a0735fcc04e2fd5ef39e..330667a219b7710782683341c0789ca2aea53f40 100644 (file)
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -719,7 +719,7 @@ initstdio(void)
        }
 
        /* Set sys.stdin */
-       if (!(std = PyFile_FromFileEx(stdin, "<stdin>", "r", fclose, -1,
+       if (!(std = PyFile_FromFd(fileno(stdin), "<stdin>", "r", -1,
                                      NULL, "\n"))) {
                goto error;
        }
@@ -728,7 +728,7 @@ initstdio(void)
        Py_DECREF(std);
 
        /* Set sys.stdout */
-       if (!(std = PyFile_FromFileEx(stdout, "<stdout>", "w", fclose, -1,
+       if (!(std = PyFile_FromFd(fileno(stdout), "<stdout>", "w", -1,
                                      NULL, "\n"))) {
             goto error;
         }
@@ -737,7 +737,7 @@ initstdio(void)
        Py_DECREF(std);
 
        /* Set sys.stderr */
-       if (!(std = PyFile_FromFileEx(stderr, "<stderr>", "w", fclose, -1,
+       if (!(std = PyFile_FromFd(fileno(stderr), "<stderr>", "w", -1,
                                      NULL, "\n"))) {
             goto error;
         }