Fix #define ordering.

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 7fcff201de7b7fb4f579d09f284be9712a713ba8..ee353aaebf5cec2e639bf6e4964ef763a9725583 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1533,7 +1533,6 @@ PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)
        return result;
 }
 
-#ifdef Py_USING_UNICODE
 /* This function is only called from parsetok. However, it cannot live
    there, as it must be empty for PGEN, and we can check for PGEN only
    in this file. */
@@ -1545,6 +1544,7 @@ PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int* offset)
        return NULL;
 }
 #else
+#ifdef Py_USING_UNICODE
 static PyObject *
 dec_utf8(const char *enc, const char *text, size_t len) {
        PyObject *ret = NULL;   
@@ -1592,10 +1592,9 @@ PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset)
        return text;
 
 }
+#endif /* defined(Py_USING_UNICODE) */
 #endif
 
-#endif /* defined(Py_USING_UNICODE) */
-                          
 
 #ifdef Py_DEBUG
 

diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index feccd4e2ba0de32c9ab343b888cdbd7ab8d27543..e10972cc99d05e59a662b59d0853490bd9eda015 100644 (file)
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -58,7 +58,7 @@ extern struct tok_state *PyTokenizer_FromString(const char *);
 extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *);
 extern void PyTokenizer_Free(struct tok_state *);
 extern int PyTokenizer_Get(struct tok_state *, char **, char **);
-#ifdef Py_USING_UNICODE
+#if defined(PGEN) || defined(Py_USING_UNICODE)
 extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok, 
                                          int len, int *offset);
 #endif