[3.8] bpo-39562: Prevent collision of future and compiler flags (GH-19230) (GH-19835)

The constant values of future flags in the __future__ module
is updated in order to prevent collision with compiler flags.
Previously PyCF_ALLOW_TOP_LEVEL_AWAIT was clashing
with CO_FUTURE_DIVISION..
(cherry picked from commit 4454057269b995341b04d13f0bf97f96080f27d0)

Co-authored-by: Batuhan Taşkaya <batuhanosmantaskaya@gmail.com>

diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index ad7d5d4c670b3d28c2a16c00168bdf2c42a7f905..dca7f48979e16cfb8b506a86ef5507ed5c377af8 100644 (file)
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -2234,3 +2234,8 @@ Notable changes in Python 3.8.2
 Fixed a regression with the ``ignore`` callback of :func:`shutil.copytree`.
 The argument types are now str and List[str] again.
 (Contributed by Manuel Barkhau and Giampaolo Rodola in :issue:`39390`.)
+
+The constant values of future flags in the :mod:`__future__` module
+are updated in order to prevent collision with compiler flags. Previously
+``PyCF_ALLOW_TOP_LEVEL_AWAIT`` was clashing with ``CO_FUTURE_DIVISION``.
+(Contributed by Batuhan Taskaya in :issue:`39562`)

diff --git a/Include/code.h b/Include/code.h
index 3afddd20c80d74b84b3d48a5d587d050c2dd99a4..a1cd58f44a0e892b6ac1b6a9e63e572e75c11765 100644 (file)
--- a/Include/code.h
+++ b/Include/code.h
@@ -88,19 +88,19 @@ typedef struct {
 #define CO_ITERABLE_COROUTINE   0x0100
 #define CO_ASYNC_GENERATOR      0x0200
 
-/* These are no longer used. */
-#if 0
-#define CO_GENERATOR_ALLOWED    0x1000
-#endif
-#define CO_FUTURE_DIVISION      0x2000
-#define CO_FUTURE_ABSOLUTE_IMPORT 0x4000 /* do absolute imports by default */
-#define CO_FUTURE_WITH_STATEMENT  0x8000
-#define CO_FUTURE_PRINT_FUNCTION  0x10000
-#define CO_FUTURE_UNICODE_LITERALS 0x20000
-
-#define CO_FUTURE_BARRY_AS_BDFL  0x40000
-#define CO_FUTURE_GENERATOR_STOP  0x80000
-#define CO_FUTURE_ANNOTATIONS    0x100000
+/* bpo-39562: These constant values are changed in Python 3.9
+   to prevent collision with compiler flags. CO_FUTURE_ and PyCF_
+   constants must be kept unique. PyCF_ constants can use bits from
+   0x0100 to 0x10000. CO_FUTURE_ constants use bits starting at 0x20000. */
+#define CO_FUTURE_DIVISION      0x20000
+#define CO_FUTURE_ABSOLUTE_IMPORT 0x40000 /* do absolute imports by default */
+#define CO_FUTURE_WITH_STATEMENT  0x80000
+#define CO_FUTURE_PRINT_FUNCTION  0x100000
+#define CO_FUTURE_UNICODE_LITERALS 0x200000
+
+#define CO_FUTURE_BARRY_AS_BDFL  0x400000
+#define CO_FUTURE_GENERATOR_STOP  0x800000
+#define CO_FUTURE_ANNOTATIONS    0x1000000
 
 /* This value is found in the co_cell2arg array when the associated cell
    variable does not correspond to an argument. */

diff --git a/Include/compile.h b/Include/compile.h
index 1cda955c1425512a429fb2c1cefe79197e60bf42..015584d03b0233aba038f8e1e2c0032a53739b57 100644 (file)
--- a/Include/compile.h
+++ b/Include/compile.h
@@ -18,12 +18,18 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
                    CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \
                    CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS)
 #define PyCF_MASK_OBSOLETE (CO_NESTED)
+
+/* bpo-39562: CO_FUTURE_ and PyCF_ constants must be kept unique.
+   PyCF_ constants can use bits from 0x0100 to 0x10000.
+   CO_FUTURE_ constants use bits starting at 0x20000. */
 #define PyCF_SOURCE_IS_UTF8  0x0100
 #define PyCF_DONT_IMPLY_DEDENT 0x0200
 #define PyCF_ONLY_AST 0x0400
 #define PyCF_IGNORE_COOKIE 0x0800
 #define PyCF_TYPE_COMMENTS 0x1000
 #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000
+#define PyCF_COMPILE_MASK (PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | \
+                           PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT)
 
 #ifndef Py_LIMITED_API
 typedef struct {

diff --git a/Lib/__future__.py b/Lib/__future__.py
index e1135685d846ce7064559aa4aa4c24839f7145aa..d7cb8ac5f497456f1dfac50b584302e99648cf06 100644 (file)
--- a/Lib/__future__.py
+++ b/Lib/__future__.py
@@ -68,14 +68,14 @@ __all__ = ["all_feature_names"] + all_feature_names
 # this module.
 CO_NESTED            = 0x0010   # nested_scopes
 CO_GENERATOR_ALLOWED = 0        # generators (obsolete, was 0x1000)
-CO_FUTURE_DIVISION   = 0x2000   # division
-CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default
-CO_FUTURE_WITH_STATEMENT  = 0x8000   # with statement
-CO_FUTURE_PRINT_FUNCTION  = 0x10000   # print function
-CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals
-CO_FUTURE_BARRY_AS_BDFL = 0x40000
-CO_FUTURE_GENERATOR_STOP  = 0x80000 # StopIteration becomes RuntimeError in generators
-CO_FUTURE_ANNOTATIONS     = 0x100000  # annotations become strings at runtime
+CO_FUTURE_DIVISION   = 0x20000   # division
+CO_FUTURE_ABSOLUTE_IMPORT = 0x40000 # perform absolute imports by default
+CO_FUTURE_WITH_STATEMENT  = 0x80000   # with statement
+CO_FUTURE_PRINT_FUNCTION  = 0x100000   # print function
+CO_FUTURE_UNICODE_LITERALS = 0x200000 # unicode string literals
+CO_FUTURE_BARRY_AS_BDFL = 0x400000
+CO_FUTURE_GENERATOR_STOP  = 0x800000 # StopIteration becomes RuntimeError in generators
+CO_FUTURE_ANNOTATIONS     = 0x1000000  # annotations become strings at runtime
 
 class _Feature:
     def __init__(self, optionalRelease, mandatoryRelease, compiler_flag):

diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py
index d83c47ef155910456abe6f353dcf923c3345f479..ea13533b5143dedebe6a31d259d4c8753fa4560b 100644 (file)
--- a/Lib/test/test_future.py
+++ b/Lib/test/test_future.py
@@ -1,5 +1,7 @@
 # Test various flavors of legal and illegal future statements
 
+import __future__
+import ast
 import unittest
 from test import support
 from textwrap import dedent
@@ -74,6 +76,21 @@ class FutureTest(unittest.TestCase):
             from test import badsyntax_future10
         self.check_syntax_error(cm.exception, "badsyntax_future10", 3)
 
+    def test_ensure_flags_dont_clash(self):
+        # bpo-39562: test that future flags and compiler flags doesn't clash
+
+        # obtain future flags (CO_FUTURE_***) from the __future__ module
+        flags = {
+            f"CO_FUTURE_{future.upper()}": getattr(__future__, future).compiler_flag
+            for future in __future__.all_feature_names
+        }
+        # obtain some of the exported compiler flags (PyCF_***) from the ast module
+        flags.update({
+            flag: getattr(ast, flag)
+            for flag in dir(ast) if flag.startswith("PyCF_")
+        })
+        self.assertCountEqual(set(flags.values()), flags.values())
+
     def test_parserhack(self):
         # test that the parser.c::future_hack function works as expected
         # Note: although this test must pass, it's not testing the original

diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-05-01-14-58-16.bpo-39562.KCsX8n.rst b/Misc/NEWS.d/next/Core and Builtins/2020-05-01-14-58-16.bpo-39562.KCsX8n.rst
new file mode 100644 (file)
index 0000000..5d7ef96
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2020-05-01-14-58-16.bpo-39562.KCsX8n.rst
@@ -0,0 +1,3 @@
+The constant values of future flags in the :mod:`__future__` module are
+updated in order to prevent collision with compiler flags. Previously
+``PyCF_ALLOW_TOP_LEVEL_AWAIT`` was clashing with ``CO_FUTURE_DIVISION``.

diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 33f969094e7d15994583abbae2a9cb8489e19737..fe22bbdde4e91fb297674a30b72ff3b4e3888d51 100644 (file)
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -738,7 +738,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename,
     }
 
     if (flags &
-        ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT | PyCF_ONLY_AST | PyCF_TYPE_COMMENTS))
+        ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_COMPILE_MASK))
     {
         PyErr_SetString(PyExc_ValueError,
                         "compile(): unrecognised flags");