dupes = set()
for vers in paths:
- for file_ in os.listdir(vers):
+ for file_ in Script._list_py_dir(self, vers):
path = os.path.realpath(os.path.join(vers, file_))
if path in dupes:
util.warn(
dir_, filename = os.path.split(path)
return cls._from_filename(scriptdir, dir_, filename)
+ @classmethod
+ def _list_py_dir(cls, scriptdir, path):
+ if scriptdir.sourceless:
+ # read files in version path, e.g. pyc or pyo files
+ # in the immediate path
+ paths = os.listdir(path)
+
+ names = set(fname.split(".")[0] for fname in paths)
+
+ # look for __pycache__
+ if os.path.exists(os.path.join(path, '__pycache__')):
+ # add all files from __pycache__ whose filename is not
+ # already in the names we got from the version directory.
+ # add as relative paths including __pycache__ token
+ paths.extend(
+ os.path.join('__pycache__', pyc)
+ for pyc in os.listdir(os.path.join(path, '__pycache__'))
+ if pyc.split(".")[0] not in names
+ )
+ return paths
+ else:
+ return os.listdir(path)
+
@classmethod
def _from_filename(cls, scriptdir, dir_, filename):
if scriptdir.sourceless:
try:
revision = self._revision_map[resolved_id]
except KeyError:
+ # break out to avoid misleading py3k stack traces
+ revision = False
+ if revision is False:
# do a partial lookup
revs = [x for x in self._revision_map
if x and x.startswith(resolved_id)]
import shutil
import textwrap
-from ..util.compat import u
+from ..util.compat import u, has_pep3147, get_current_bytecode_suffixes
from ..script import Script, ScriptDirectory
from .. import util
from . import engines
# generate .pyc/.pyo without importing but not really
# worth it.
pass
- make_sourceless(os.path.join(path, "env.py"))
+ assert sourceless in (
+ "pep3147_envonly", "simple", "pep3147_everything"), sourceless
+ make_sourceless(
+ os.path.join(path, "env.py"),
+ "pep3147" if "pep3147" in sourceless else "simple"
+ )
sc = script.ScriptDirectory.from_config(cfg)
return sc
path = os.path.join(dir_, "env.py")
pyc_path = util.pyc_file_from_path(path)
- if os.access(pyc_path, os.F_OK):
+ if pyc_path:
os.unlink(pyc_path)
with open(path, 'w') as f:
""" % (dir_, url, "true" if sourceless else "false"))
-
-
def _multi_dir_testing_config(sourceless=False, extra_version_location=''):
dir_ = os.path.join(_get_staging_directory(), 'scripts')
url = "sqlite:///%s/foo.db" % dir_
with open(path, 'wb') as fp:
fp.write(content)
pyc_path = util.pyc_file_from_path(path)
- if os.access(pyc_path, os.F_OK):
+ if pyc_path:
os.unlink(pyc_path)
script = Script._from_path(scriptdir, path)
old = scriptdir.revision_map.get_revision(script.revision)
scriptdir.revision_map.add_revision(script, _replace=True)
if sourceless:
- make_sourceless(path)
-
+ make_sourceless(
+ path,
+ "pep3147" if sourceless == "pep3147_everything" else "simple"
+ )
+
+
+def make_sourceless(path, style):
+
+ import py_compile
+ py_compile.compile(path)
+
+ if style == "simple" and has_pep3147():
+ pyc_path = util.pyc_file_from_path(path)
+ suffix = get_current_bytecode_suffixes()[0]
+ filepath, ext = os.path.splitext(path)
+ simple_pyc_path = filepath + suffix
+ shutil.move(pyc_path, simple_pyc_path)
+ pyc_path = simple_pyc_path
+ elif style == "pep3147" and not has_pep3147():
+ raise NotImplementedError()
+ else:
+ assert style in ("pep3147", "simple")
+ pyc_path = util.pyc_file_from_path(path)
-def make_sourceless(path):
- # note that if -O is set, you'd see pyo files here,
- # the pyc util function looks at sys.flags.optimize to handle this
- pyc_path = util.pyc_file_from_path(path)
assert os.access(pyc_path, os.F_OK)
- # look for a non-pep3147 path here.
- # if not present, need to copy from __pycache__
- simple_pyc_path = util.simple_pyc_file_from_path(path)
-
- if not os.access(simple_pyc_path, os.F_OK):
- shutil.copyfile(pyc_path, simple_pyc_path)
os.unlink(path)
lambda config: not util.sqla_110,
"SQLAlchemy 1.1.0 or greater required"
)
+
+ @property
+ def pep3147(self):
+
+ return exclusions.only_if(
+ lambda config: util.compat.has_pep3147()
+ )
+
from .messaging import ( # noqa
write_outstream, status, err, obfuscate_url_pw, warn, msg, format_as_comma)
from .pyfiles import ( # noqa
- template_to_file, coerce_resource_to_filename, simple_pyc_file_from_path,
+ template_to_file, coerce_resource_to_filename,
pyc_file_from_path, load_python_file, edit)
from .sqla_compat import ( # noqa
sqla_07, sqla_079, sqla_08, sqla_083, sqla_084, sqla_09, sqla_092,
py2k = sys.version_info < (3, 0)
py3k = sys.version_info >= (3, 0)
py33 = sys.version_info >= (3, 3)
+py35 = sys.version_info >= (3, 5)
+py36 = sys.version_info >= (3, 6)
if py3k:
from io import StringIO
else:
# accepts strings
- from StringIO import StringIO
+ from StringIO import StringIO # noqa
if py3k:
import builtins as compat_builtins
from configparser import ConfigParser as SafeConfigParser
import configparser
else:
- from ConfigParser import SafeConfigParser
- import ConfigParser as configparser
+ from ConfigParser import SafeConfigParser # noqa
+ import ConfigParser as configparser # noqa
if py2k:
from mako.util import parse_encoding
-if py33:
- from importlib import machinery
+if py35:
+ import importlib.util
+ import importlib.machinery
+
+ def load_module_py(module_id, path):
+ spec = importlib.util.spec_from_file_location(module_id, path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+ def load_module_pyc(module_id, path):
+ spec = importlib.util.spec_from_file_location(module_id, path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+elif py33:
+ import importlib.machinery
def load_module_py(module_id, path):
- return machinery.SourceFileLoader(
+ module = importlib.machinery.SourceFileLoader(
module_id, path).load_module(module_id)
+ del sys.modules[module_id]
+ return module
def load_module_pyc(module_id, path):
- return machinery.SourcelessFileLoader(
+ module = importlib.machinery.SourcelessFileLoader(
module_id, path).load_module(module_id)
+ del sys.modules[module_id]
+ return module
+
+if py33:
+ def get_bytecode_suffixes():
+ try:
+ return importlib.machinery.BYTECODE_SUFFIXES
+ except AttributeError:
+ return importlib.machinery.DEBUG_BYTECODE_SUFFIXES
+
+ def get_current_bytecode_suffixes():
+ if py35:
+ suffixes = importlib.machinery.BYTECODE_SUFFIXES
+ elif py33:
+ if sys.flags.optimize:
+ suffixes = importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES
+ else:
+ suffixes = importlib.machinery.BYTECODE_SUFFIXES
+ else:
+ if sys.flags.optimize:
+ suffixes = [".pyo"]
+ else:
+ suffixes = [".pyc"]
+
+ return suffixes
+
+ def has_pep3147():
+ # http://www.python.org/dev/peps/pep-3147/#detecting-pep-3147-availability
+
+ import imp
+ return hasattr(imp, 'get_tag')
else:
import imp
- def load_module_py(module_id, path):
+ def load_module_py(module_id, path): # noqa
with open(path, 'rb') as fp:
mod = imp.load_source(module_id, path, fp)
if py2k:
source_encoding = parse_encoding(fp)
if source_encoding:
mod._alembic_source_encoding = source_encoding
+ del sys.modules[module_id]
return mod
- def load_module_pyc(module_id, path):
+ def load_module_pyc(module_id, path): # noqa
with open(path, 'rb') as fp:
mod = imp.load_compiled(module_id, path, fp)
# no source encoding here
+ del sys.modules[module_id]
return mod
+ def get_current_bytecode_suffixes():
+ if sys.flags.optimize:
+ return [".pyo"] # e.g. .pyo
+ else:
+ return [".pyc"] # e.g. .pyc
+
+ def has_pep3147():
+ return False
+
try:
exec_ = getattr(compat_builtins, 'exec')
except AttributeError:
import sys
import os
import re
-from .compat import load_module_py, load_module_pyc
+from .compat import load_module_py, load_module_pyc, \
+ get_current_bytecode_suffixes, has_pep3147
from mako.template import Template
from mako import exceptions
import tempfile
return fname
-def simple_pyc_file_from_path(path):
- """Given a python source path, return the so-called
- "sourceless" .pyc or .pyo path.
-
- This just a .pyc or .pyo file where the .py file would be.
-
- Even with PEP-3147, which normally puts .pyc/.pyo files in __pycache__,
- this use case remains supported as a so-called "sourceless module import".
-
- """
- if sys.flags.optimize:
- return path + "o" # e.g. .pyo
- else:
- return path + "c" # e.g. .pyc
-
-
def pyc_file_from_path(path):
"""Given a python source path, locate the .pyc.
- See http://www.python.org/dev/peps/pep-3147/
- #detecting-pep-3147-availability
- http://www.python.org/dev/peps/pep-3147/#file-extension-checks
-
"""
- import imp
- has3147 = hasattr(imp, 'get_tag')
- if has3147:
- return imp.cache_from_source(path)
+
+ if has_pep3147():
+ import imp
+ candidate = imp.cache_from_source(path)
+ if os.path.exists(candidate):
+ return candidate
+
+ filepath, ext = os.path.splitext(path)
+ for ext in get_current_bytecode_suffixes():
+ if os.path.exists(filepath + ext):
+ return filepath + ext
else:
- return simple_pyc_file_from_path(path)
+ return None
def edit(path):
if ext == ".py":
if os.path.exists(path):
module = load_module_py(module_id, path)
- elif os.path.exists(simple_pyc_file_from_path(path)):
- # look for sourceless load
- module = load_module_pyc(
- module_id, simple_pyc_file_from_path(path))
else:
- raise ImportError("Can't find Python file %s" % path)
+ pyc_path = pyc_file_from_path(path)
+ if pyc_path is None:
+ raise ImportError("Can't find Python file %s" % path)
+ else:
+ module = load_module_pyc(module_id, pyc_path)
elif ext in (".pyc", ".pyo"):
module = load_module_pyc(module_id, path)
- del sys.modules[module_id]
return module
--- /dev/null
+.. change::
+ :tags: bug, runtime, py3k
+ :tickets: 449
+
+ Reworked "sourceless" system to be fully capable of handling any
+ combination of: Python2/3x, pep3149 or not, PYTHONOPTIMIZE or not,
+ for locating and loading both env.py files as well as versioning files.
+ This includes: locating files inside of ``__pycache__`` as well as listing
+ out version files that might be only in ``versions/__pycache__``, deduplicating
+ version files that may be in ``versions/__pycache__`` and ``versions/``
+ at the same time, correctly looking for .pyc or .pyo files based on
+ if pep488 is present or not. The latest Python3x deprecation warnings
+ involving importlib are also corrected.
\ No newline at end of file
from contextlib import contextmanager
from alembic.testing import mock
+
class ApplyVersionsFunctionalTest(TestBase):
__only_on__ = 'sqlite'
assert not db.dialect.has_table(db.connect(), 'bat')
-class SourcelessApplyVersionsTest(ApplyVersionsFunctionalTest):
- sourceless = True
+class SimpleSourcelessApplyVersionsTest(ApplyVersionsFunctionalTest):
+ sourceless = "simple"
+
+
+class NewFangledSourcelessEnvOnlyApplyVersionsTest(
+ ApplyVersionsFunctionalTest):
+ sourceless = "pep3147_envonly"
+
+ __requires__ = "pep3147",
+
+
+class NewFangledSourcelessEverythingApplyVersionsTest(
+ ApplyVersionsFunctionalTest):
+ sourceless = "pep3147_everything"
+
+ __requires__ = "pep3147",
class CallbackEnvironmentTest(ApplyVersionsFunctionalTest):
self._test_ignore_dot_hash_py("pyo")
-class SourcelessIgnoreFilesTest(IgnoreFilesTest):
- sourceless = True
+class SimpleSourcelessIgnoreFilesTest(IgnoreFilesTest):
+ sourceless = "simple"
+
+
+class NewFangledEnvOnlySourcelessIgnoreFilesTest(IgnoreFilesTest):
+ sourceless = "pep3147_envonly"
+
+ __requires__ = "pep3147",
+
+
+class NewFangledEverythingSourcelessIgnoreFilesTest(IgnoreFilesTest):
+ sourceless = "pep3147_everything"
+
+ __requires__ = "pep3147",
class SourcelessNeedsFlagTest(TestBase):
mysql: MYSQL={env:TOX_MYSQL:--db mysql}
oracle: ORACLE={env:TOX_ORACLE:--db oracle} --low-connections --write-idents oracle_idents.txt
mssql: MSSQL={env:TOX_MSSQL:--db pymssql}
+ pyoptimize: PYTHONOPTIMIZE=1
+ pyoptimize: LIMITTESTS="tests/test_script_consumption.py"
# tox as of 2.0 blocks all environment variables from the
# outside, unless they are here (or in TOX_TESTENV_PASSENV,
passenv=ORACLE_HOME NLS_LANG TOX_SQLITE TOX_POSTGRESQL TOX_MYSQL TOX_ORACLE TOX_MSSQL
commands=
- {env:BASECOMMAND} {env:WORKERS} {env:SQLITE:} {env:POSTGRESQL:} {env:MYSQL:} {env:ORACLE:} {env:MSSQL:} {env:BACKENDONLY:} {env:COVERAGE:} {posargs}
+ {env:BASECOMMAND} {env:WORKERS} {env:SQLITE:} {env:POSTGRESQL:} {env:MYSQL:} {env:ORACLE:} {env:MSSQL:} {env:BACKENDONLY:} {env:COVERAGE:} {env:LIMITTESTS:} {posargs}
{oracle}: python reap_oracle_dbs.py oracle_idents.txt
projects / thirdparty / sqlalchemy / alembic.git / commitdiff
