From: Mike Bayer Date: Thu, 7 Sep 2017 21:43:00 +0000 (-0400) Subject: Support pep3149, latest import mechanics, fully X-Git-Tag: rel_0_9_6~11 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9b5f12a0a3f6b83bccb19c23e0c95b17d7ec8ae3;p=thirdparty%2Fsqlalchemy%2Falembic.git Support pep3149, latest import mechanics, fully Reworked "sourceless" system to be fully capable of handling any combination of: Python2/3x, pep3149 or not, PYTHONOPTIMIZE or not, for locating and loading both env.py files as well as versioning files. This includes: locating files inside of ``__pycache__`` as well as listing out version files that might be only in ``versions/__pycache__``, deduplicating version files that may be in ``versions/__pycache__`` and ``versions/`` at the same time, correctly looking for .pyc or .pyo files based on if pep488 is present or not. The latest Python3x deprecation warnings involving importlib are also corrected. Change-Id: I2495e793c81846d3f05620dbececb18973dd8a8f Fixes: #449 --- diff --git a/alembic/script/base.py b/alembic/script/base.py index d1760e58..2e6e0fb0 100644 --- a/alembic/script/base.py +++ b/alembic/script/base.py @@ -87,7 +87,7 @@ class ScriptDirectory(object): dupes = set() for vers in paths: - for file_ in os.listdir(vers): + for file_ in Script._list_py_dir(self, vers): path = os.path.realpath(os.path.join(vers, file_)) if path in dupes: util.warn( @@ -750,6 +750,29 @@ class Script(revision.Revision): dir_, filename = os.path.split(path) return cls._from_filename(scriptdir, dir_, filename) + @classmethod + def _list_py_dir(cls, scriptdir, path): + if scriptdir.sourceless: + # read files in version path, e.g. pyc or pyo files + # in the immediate path + paths = os.listdir(path) + + names = set(fname.split(".")[0] for fname in paths) + + # look for __pycache__ + if os.path.exists(os.path.join(path, '__pycache__')): + # add all files from __pycache__ whose filename is not + # already in the names we got from the version directory. + # add as relative paths including __pycache__ token + paths.extend( + os.path.join('__pycache__', pyc) + for pyc in os.listdir(os.path.join(path, '__pycache__')) + if pyc.split(".")[0] not in names + ) + return paths + else: + return os.listdir(path) + @classmethod def _from_filename(cls, scriptdir, dir_, filename): if scriptdir.sourceless: diff --git a/alembic/script/revision.py b/alembic/script/revision.py index 9145f551..7e25a865 100644 --- a/alembic/script/revision.py +++ b/alembic/script/revision.py @@ -348,6 +348,9 @@ class RevisionMap(object): try: revision = self._revision_map[resolved_id] except KeyError: + # break out to avoid misleading py3k stack traces + revision = False + if revision is False: # do a partial lookup revs = [x for x in self._revision_map if x and x.startswith(resolved_id)] diff --git a/alembic/testing/env.py b/alembic/testing/env.py index 5ad6c694..7e328fda 100644 --- a/alembic/testing/env.py +++ b/alembic/testing/env.py @@ -4,7 +4,7 @@ import os import shutil import textwrap -from ..util.compat import u +from ..util.compat import u, has_pep3147, get_current_bytecode_suffixes from ..script import Script, ScriptDirectory from .. import util from . import engines @@ -37,7 +37,12 @@ def staging_env(create=True, template="generic", sourceless=False): # generate .pyc/.pyo without importing but not really # worth it. pass - make_sourceless(os.path.join(path, "env.py")) + assert sourceless in ( + "pep3147_envonly", "simple", "pep3147_everything"), sourceless + make_sourceless( + os.path.join(path, "env.py"), + "pep3147" if "pep3147" in sourceless else "simple" + ) sc = script.ScriptDirectory.from_config(cfg) return sc @@ -64,7 +69,7 @@ config = context.config path = os.path.join(dir_, "env.py") pyc_path = util.pyc_file_from_path(path) - if os.access(pyc_path, os.F_OK): + if pyc_path: os.unlink(pyc_path) with open(path, 'w') as f: @@ -113,8 +118,6 @@ datefmt = %%H:%%M:%%S """ % (dir_, url, "true" if sourceless else "false")) - - def _multi_dir_testing_config(sourceless=False, extra_version_location=''): dir_ = os.path.join(_get_staging_directory(), 'scripts') url = "sqlite:///%s/foo.db" % dir_ @@ -215,7 +218,7 @@ def write_script( with open(path, 'wb') as fp: fp.write(content) pyc_path = util.pyc_file_from_path(path) - if os.access(pyc_path, os.F_OK): + if pyc_path: os.unlink(pyc_path) script = Script._from_path(scriptdir, path) old = scriptdir.revision_map.get_revision(script.revision) @@ -225,21 +228,32 @@ def write_script( scriptdir.revision_map.add_revision(script, _replace=True) if sourceless: - make_sourceless(path) - + make_sourceless( + path, + "pep3147" if sourceless == "pep3147_everything" else "simple" + ) + + +def make_sourceless(path, style): + + import py_compile + py_compile.compile(path) + + if style == "simple" and has_pep3147(): + pyc_path = util.pyc_file_from_path(path) + suffix = get_current_bytecode_suffixes()[0] + filepath, ext = os.path.splitext(path) + simple_pyc_path = filepath + suffix + shutil.move(pyc_path, simple_pyc_path) + pyc_path = simple_pyc_path + elif style == "pep3147" and not has_pep3147(): + raise NotImplementedError() + else: + assert style in ("pep3147", "simple") + pyc_path = util.pyc_file_from_path(path) -def make_sourceless(path): - # note that if -O is set, you'd see pyo files here, - # the pyc util function looks at sys.flags.optimize to handle this - pyc_path = util.pyc_file_from_path(path) assert os.access(pyc_path, os.F_OK) - # look for a non-pep3147 path here. - # if not present, need to copy from __pycache__ - simple_pyc_path = util.simple_pyc_file_from_path(path) - - if not os.access(simple_pyc_path, os.F_OK): - shutil.copyfile(pyc_path, simple_pyc_path) os.unlink(path) diff --git a/alembic/testing/requirements.py b/alembic/testing/requirements.py index 51d06fff..6f8014cd 100644 --- a/alembic/testing/requirements.py +++ b/alembic/testing/requirements.py @@ -171,3 +171,11 @@ class SuiteRequirements(Requirements): lambda config: not util.sqla_110, "SQLAlchemy 1.1.0 or greater required" ) + + @property + def pep3147(self): + + return exclusions.only_if( + lambda config: util.compat.has_pep3147() + ) + diff --git a/alembic/util/__init__.py b/alembic/util/__init__.py index 8a857366..69b652fe 100644 --- a/alembic/util/__init__.py +++ b/alembic/util/__init__.py @@ -4,7 +4,7 @@ from .langhelpers import ( # noqa from .messaging import ( # noqa write_outstream, status, err, obfuscate_url_pw, warn, msg, format_as_comma) from .pyfiles import ( # noqa - template_to_file, coerce_resource_to_filename, simple_pyc_file_from_path, + template_to_file, coerce_resource_to_filename, pyc_file_from_path, load_python_file, edit) from .sqla_compat import ( # noqa sqla_07, sqla_079, sqla_08, sqla_083, sqla_084, sqla_09, sqla_092, diff --git a/alembic/util/compat.py b/alembic/util/compat.py index 45287ee3..4d7e6fad 100644 --- a/alembic/util/compat.py +++ b/alembic/util/compat.py @@ -8,12 +8,14 @@ py27 = sys.version_info >= (2, 7) py2k = sys.version_info < (3, 0) py3k = sys.version_info >= (3, 0) py33 = sys.version_info >= (3, 3) +py35 = sys.version_info >= (3, 5) +py36 = sys.version_info >= (3, 6) if py3k: from io import StringIO else: # accepts strings - from StringIO import StringIO + from StringIO import StringIO # noqa if py3k: import builtins as compat_builtins @@ -50,41 +52,101 @@ if py3k: from configparser import ConfigParser as SafeConfigParser import configparser else: - from ConfigParser import SafeConfigParser - import ConfigParser as configparser + from ConfigParser import SafeConfigParser # noqa + import ConfigParser as configparser # noqa if py2k: from mako.util import parse_encoding -if py33: - from importlib import machinery +if py35: + import importlib.util + import importlib.machinery + + def load_module_py(module_id, path): + spec = importlib.util.spec_from_file_location(module_id, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def load_module_pyc(module_id, path): + spec = importlib.util.spec_from_file_location(module_id, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + +elif py33: + import importlib.machinery def load_module_py(module_id, path): - return machinery.SourceFileLoader( + module = importlib.machinery.SourceFileLoader( module_id, path).load_module(module_id) + del sys.modules[module_id] + return module def load_module_pyc(module_id, path): - return machinery.SourcelessFileLoader( + module = importlib.machinery.SourcelessFileLoader( module_id, path).load_module(module_id) + del sys.modules[module_id] + return module + +if py33: + def get_bytecode_suffixes(): + try: + return importlib.machinery.BYTECODE_SUFFIXES + except AttributeError: + return importlib.machinery.DEBUG_BYTECODE_SUFFIXES + + def get_current_bytecode_suffixes(): + if py35: + suffixes = importlib.machinery.BYTECODE_SUFFIXES + elif py33: + if sys.flags.optimize: + suffixes = importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES + else: + suffixes = importlib.machinery.BYTECODE_SUFFIXES + else: + if sys.flags.optimize: + suffixes = [".pyo"] + else: + suffixes = [".pyc"] + + return suffixes + + def has_pep3147(): + # http://www.python.org/dev/peps/pep-3147/#detecting-pep-3147-availability + + import imp + return hasattr(imp, 'get_tag') else: import imp - def load_module_py(module_id, path): + def load_module_py(module_id, path): # noqa with open(path, 'rb') as fp: mod = imp.load_source(module_id, path, fp) if py2k: source_encoding = parse_encoding(fp) if source_encoding: mod._alembic_source_encoding = source_encoding + del sys.modules[module_id] return mod - def load_module_pyc(module_id, path): + def load_module_pyc(module_id, path): # noqa with open(path, 'rb') as fp: mod = imp.load_compiled(module_id, path, fp) # no source encoding here + del sys.modules[module_id] return mod + def get_current_bytecode_suffixes(): + if sys.flags.optimize: + return [".pyo"] # e.g. .pyo + else: + return [".pyc"] # e.g. .pyc + + def has_pep3147(): + return False + try: exec_ = getattr(compat_builtins, 'exec') except AttributeError: diff --git a/alembic/util/pyfiles.py b/alembic/util/pyfiles.py index e6514035..0e521335 100644 --- a/alembic/util/pyfiles.py +++ b/alembic/util/pyfiles.py @@ -1,7 +1,8 @@ import sys import os import re -from .compat import load_module_py, load_module_pyc +from .compat import load_module_py, load_module_pyc, \ + get_current_bytecode_suffixes, has_pep3147 from mako.template import Template from mako import exceptions import tempfile @@ -40,36 +41,23 @@ def coerce_resource_to_filename(fname): return fname -def simple_pyc_file_from_path(path): - """Given a python source path, return the so-called - "sourceless" .pyc or .pyo path. - - This just a .pyc or .pyo file where the .py file would be. - - Even with PEP-3147, which normally puts .pyc/.pyo files in __pycache__, - this use case remains supported as a so-called "sourceless module import". - - """ - if sys.flags.optimize: - return path + "o" # e.g. .pyo - else: - return path + "c" # e.g. .pyc - - def pyc_file_from_path(path): """Given a python source path, locate the .pyc. - See http://www.python.org/dev/peps/pep-3147/ - #detecting-pep-3147-availability - http://www.python.org/dev/peps/pep-3147/#file-extension-checks - """ - import imp - has3147 = hasattr(imp, 'get_tag') - if has3147: - return imp.cache_from_source(path) + + if has_pep3147(): + import imp + candidate = imp.cache_from_source(path) + if os.path.exists(candidate): + return candidate + + filepath, ext = os.path.splitext(path) + for ext in get_current_bytecode_suffixes(): + if os.path.exists(filepath + ext): + return filepath + ext else: - return simple_pyc_file_from_path(path) + return None def edit(path): @@ -91,13 +79,12 @@ def load_python_file(dir_, filename): if ext == ".py": if os.path.exists(path): module = load_module_py(module_id, path) - elif os.path.exists(simple_pyc_file_from_path(path)): - # look for sourceless load - module = load_module_pyc( - module_id, simple_pyc_file_from_path(path)) else: - raise ImportError("Can't find Python file %s" % path) + pyc_path = pyc_file_from_path(path) + if pyc_path is None: + raise ImportError("Can't find Python file %s" % path) + else: + module = load_module_pyc(module_id, pyc_path) elif ext in (".pyc", ".pyo"): module = load_module_pyc(module_id, path) - del sys.modules[module_id] return module diff --git a/docs/build/unreleased/449.rst b/docs/build/unreleased/449.rst new file mode 100644 index 00000000..09661b1d --- /dev/null +++ b/docs/build/unreleased/449.rst @@ -0,0 +1,13 @@ +.. change:: + :tags: bug, runtime, py3k + :tickets: 449 + + Reworked "sourceless" system to be fully capable of handling any + combination of: Python2/3x, pep3149 or not, PYTHONOPTIMIZE or not, + for locating and loading both env.py files as well as versioning files. + This includes: locating files inside of ``__pycache__`` as well as listing + out version files that might be only in ``versions/__pycache__``, deduplicating + version files that may be in ``versions/__pycache__`` and ``versions/`` + at the same time, correctly looking for .pyc or .pyo files based on + if pep488 is present or not. The latest Python3x deprecation warnings + involving importlib are also corrected. \ No newline at end of file diff --git a/tests/test_script_consumption.py b/tests/test_script_consumption.py index 14dd7833..7a8f0dc3 100644 --- a/tests/test_script_consumption.py +++ b/tests/test_script_consumption.py @@ -16,6 +16,7 @@ from alembic.environment import EnvironmentContext from contextlib import contextmanager from alembic.testing import mock + class ApplyVersionsFunctionalTest(TestBase): __only_on__ = 'sqlite' @@ -138,8 +139,22 @@ class ApplyVersionsFunctionalTest(TestBase): assert not db.dialect.has_table(db.connect(), 'bat') -class SourcelessApplyVersionsTest(ApplyVersionsFunctionalTest): - sourceless = True +class SimpleSourcelessApplyVersionsTest(ApplyVersionsFunctionalTest): + sourceless = "simple" + + +class NewFangledSourcelessEnvOnlyApplyVersionsTest( + ApplyVersionsFunctionalTest): + sourceless = "pep3147_envonly" + + __requires__ = "pep3147", + + +class NewFangledSourcelessEverythingApplyVersionsTest( + ApplyVersionsFunctionalTest): + sourceless = "pep3147_everything" + + __requires__ = "pep3147", class CallbackEnvironmentTest(ApplyVersionsFunctionalTest): @@ -591,8 +606,20 @@ class IgnoreFilesTest(TestBase): self._test_ignore_dot_hash_py("pyo") -class SourcelessIgnoreFilesTest(IgnoreFilesTest): - sourceless = True +class SimpleSourcelessIgnoreFilesTest(IgnoreFilesTest): + sourceless = "simple" + + +class NewFangledEnvOnlySourcelessIgnoreFilesTest(IgnoreFilesTest): + sourceless = "pep3147_envonly" + + __requires__ = "pep3147", + + +class NewFangledEverythingSourcelessIgnoreFilesTest(IgnoreFilesTest): + sourceless = "pep3147_everything" + + __requires__ = "pep3147", class SourcelessNeedsFlagTest(TestBase): diff --git a/tox.ini b/tox.ini index 5424402c..e527d68f 100644 --- a/tox.ini +++ b/tox.ini @@ -41,6 +41,8 @@ setenv= mysql: MYSQL={env:TOX_MYSQL:--db mysql} oracle: ORACLE={env:TOX_ORACLE:--db oracle} --low-connections --write-idents oracle_idents.txt mssql: MSSQL={env:TOX_MSSQL:--db pymssql} + pyoptimize: PYTHONOPTIMIZE=1 + pyoptimize: LIMITTESTS="tests/test_script_consumption.py" # tox as of 2.0 blocks all environment variables from the # outside, unless they are here (or in TOX_TESTENV_PASSENV, @@ -48,7 +50,7 @@ setenv= passenv=ORACLE_HOME NLS_LANG TOX_SQLITE TOX_POSTGRESQL TOX_MYSQL TOX_ORACLE TOX_MSSQL commands= - {env:BASECOMMAND} {env:WORKERS} {env:SQLITE:} {env:POSTGRESQL:} {env:MYSQL:} {env:ORACLE:} {env:MSSQL:} {env:BACKENDONLY:} {env:COVERAGE:} {posargs} + {env:BASECOMMAND} {env:WORKERS} {env:SQLITE:} {env:POSTGRESQL:} {env:MYSQL:} {env:ORACLE:} {env:MSSQL:} {env:BACKENDONLY:} {env:COVERAGE:} {env:LIMITTESTS:} {posargs} {oracle}: python reap_oracle_dbs.py oracle_idents.txt