From 9907011222e3fb28700f1a8c505aed5c90de9def Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Fri, 26 Sep 2025 17:32:45 -0400 Subject: [PATCH] use nox this includes an adapted form of tox generative environments ported to nox. Change-Id: Ifada3485adb50ab6ed5a80f78986eb657abc5f08 --- .github/workflows/run-on-pr.yaml | 8 +- .github/workflows/run-test.yaml | 10 +- .gitignore | 1 + MANIFEST.in | 2 +- docs/build/unreleased/nox.rst | 8 ++ noxfile.py | 236 +++++++++++++++++++++++++++++++ pyproject.toml | 34 +++++ setup.cfg | 6 +- tools/toxnox.py | 211 +++++++++++++++++++++++++++ tools/warn_tox.py | 12 ++ tox.ini | 6 + 11 files changed, 521 insertions(+), 13 deletions(-) create mode 100644 docs/build/unreleased/nox.rst create mode 100644 noxfile.py create mode 100644 tools/toxnox.py create mode 100644 tools/warn_tox.py diff --git a/.github/workflows/run-on-pr.yaml b/.github/workflows/run-on-pr.yaml index 0795e9b7..098d0e5d 100644 --- a/.github/workflows/run-on-pr.yaml +++ b/.github/workflows/run-on-pr.yaml @@ -47,11 +47,11 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install --upgrade tox setuptools + pip install --upgrade nox setuptools pip list - name: Run tests - run: tox -e py-${{ matrix.sqlalchemy }} + run: nox -t py-${{ matrix.sqlalchemy }} run-pep484: name: pep484-${{ matrix.python-version }}-${{ matrix.os }} @@ -79,8 +79,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install --upgrade tox setuptools + pip install --upgrade nox setuptools pip list - name: Run pep484 - run: tox -e pep484 + run: nox -s pep484 diff --git a/.github/workflows/run-test.yaml b/.github/workflows/run-test.yaml index 64cab8fa..33708edf 100644 --- a/.github/workflows/run-test.yaml +++ b/.github/workflows/run-test.yaml @@ -64,11 +64,11 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install --upgrade tox setuptools + pip install --upgrade nox setuptools pip list - name: Run tests - run: tox -e py-${{ matrix.sqlalchemy }} + run: nox -t py-${{ matrix.sqlalchemy }} run-pep484: name: pep484-${{ matrix.python-version }}-${{ matrix.os }} @@ -96,8 +96,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install --upgrade tox setuptools + pip install --upgrade nox setuptools pip list - - name: Run tox pep484 - run: tox -e pep484 + - name: Run nox pep484 + run: nox -s pep484 diff --git a/.gitignore b/.gitignore index 7d69c127..fd758818 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ alembic.ini .coverage coverage.xml .tox +.nox *.patch /scratch /scratch_test_* diff --git a/MANIFEST.in b/MANIFEST.in index dc8ec40b..ace73359 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,7 @@ recursive-include alembic/templates *.mako README *.py *.pyi recursive-include alembic *.py *.pyi py.typed recursive-include tools *.py -include README* LICENSE CHANGES* tox.ini +include README* LICENSE CHANGES* tox.ini noxfile.py prune docs/build/output diff --git a/docs/build/unreleased/nox.rst b/docs/build/unreleased/nox.rst new file mode 100644 index 00000000..dbc4d5ef --- /dev/null +++ b/docs/build/unreleased/nox.rst @@ -0,0 +1,8 @@ +.. change:: + :tags: change, tests + + The top-level test runner has been changed to use ``nox``, adding a + ``noxfile.py`` as well as some included modules. The ``tox.ini`` file + remains in place so that ``tox`` runs will continue to function in the near + term, however it will be eventually removed and improvements and + maintenance going forward will be only towards ``noxfile.py``. diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 00000000..d1dc2144 --- /dev/null +++ b/noxfile.py @@ -0,0 +1,236 @@ +"""Nox configuration for Alembic.""" + +from __future__ import annotations + +import os +import sys +from typing import Optional +from typing import Sequence + +import nox +from packaging.version import parse as parse_version + +if True: + sys.path.insert(0, ".") + from tools.toxnox import tox_parameters + from tools.toxnox import extract_opts + + +SQLA_REPO = os.environ.get( + "SQLA_REPO", "git+https://github.com/sqlalchemy/sqlalchemy.git" +) + +PYTHON_VERSIONS = [ + "3.8", + "3.9", + "3.10", + "3.11", + "3.12", + "3.13", + "3.13t", + "3.14", + "3.14t", +] +DATABASES = ["sqlite", "postgresql", "mysql", "oracle", "mssql"] +SQLALCHEMY_VERSIONS = ["default", "sqla14", "sqla20", "sqlamain"] + +pyproject = nox.project.load_toml("pyproject.toml") + +nox.options.sessions = ["tests"] +nox.options.tags = ["py"] + + +def filter_sqla( + python: str, sqlalchemy: str, database: Optional[str] = None +) -> bool: + python_version = parse_version(python.rstrip("t")) + if sqlalchemy == "sqla14": + return python_version < parse_version("3.14") + elif sqlalchemy == "sqlamain": + return python_version > parse_version("3.9") + else: + return True + + +@nox.session() +@tox_parameters( + ["python", "sqlalchemy", "database"], + [PYTHON_VERSIONS, SQLALCHEMY_VERSIONS, DATABASES], + filter_=filter_sqla, +) +def tests(session: nox.Session, sqlalchemy: str, database: str) -> None: + """Run the main test suite against one database at a time""" + + _tests(session, sqlalchemy, [database]) + + +@nox.session() +@tox_parameters( + ["python", "sqlalchemy"], + [PYTHON_VERSIONS, SQLALCHEMY_VERSIONS], + filter_=filter_sqla, + base_tag="all", +) +def tests_alldb(session: nox.Session, sqlalchemy: str) -> None: + """Run the main test suite against all backends at once""" + + _tests(session, sqlalchemy, DATABASES) + + +@nox.session(name="coverage") +@tox_parameters( + ["database"], + [DATABASES], + base_tag="coverage", +) +def coverage(session: nox.Session, database: str) -> None: + """Run tests with coverage.""" + + _tests(session, "default", [database], coverage=True) + + +def _tests( + session: nox.Session, + sqlalchemy: str, + databases: Sequence[str], + coverage: bool = False, +) -> None: + if sqlalchemy == "sqla14": + session.install(f"{SQLA_REPO}@rel_1_4#egg=sqlalchemy") + elif sqlalchemy == "sqla20": + session.install(f"{SQLA_REPO}@rel_2_0#egg=sqlalchemy") + elif sqlalchemy == "sqlamain": + session.install(f"{SQLA_REPO}#egg=sqlalchemy") + + # for sqlalchemy == "default", the alembic install will install + # current released SQLAlchemy version as a dependency + if coverage: + session.install("-e", ".") + else: + session.install(".") + + session.install(*nox.project.dependency_groups(pyproject, "tests")) + + if coverage: + session.install(*nox.project.dependency_groups(pyproject, "coverage")) + + session.env["SQLALCHEMY_WARN_20"] = "1" + + cmd = ["python", "-m", "pytest"] + + if coverage: + cmd.extend( + [ + "--cov=alembic", + "--cov-append", + "--cov-report", + "term", + "--cov-report", + "xml", + ], + ) + + cmd.extend(os.environ.get("TOX_WORKERS", "-n4").split()) + + for database in databases: + if database == "sqlite": + cmd.extend(os.environ.get("TOX_SQLITE", "--db sqlite").split()) + elif database == "postgresql": + session.install( + *nox.project.dependency_groups(pyproject, "tests_postgresql") + ) + cmd.extend( + os.environ.get("TOX_POSTGRESQL", "--db postgresql").split() + ) + elif database == "mysql": + session.install( + *nox.project.dependency_groups(pyproject, "tests_mysql") + ) + cmd.extend(os.environ.get("TOX_MYSQL", "--db mysql").split()) + elif databases == "oracle": + # we'd like to use oracledb but SQLAlchemy 1.4 does not have + # oracledb support... + session.install( + *nox.project.dependency_groups(pyproject, "tests_oracle") + ) + if "ORACLE_HOME" in os.environ: + session.env["ORACLE_HOME"] = os.environ.get("ORACLE_HOME") + if "NLS_LANG" in os.environ: + session.env["NLS_LANG"] = os.environ.get("NLS_LANG") + cmd.extend(os.environ.get("TOX_ORACLE", "--db oracle").split()) + cmd.extend("--write-idents db_idents.txt".split()) + + elif database == "mssql": + session.install( + *nox.project.dependency_groups(pyproject, "tests_mssql") + ) + cmd.extend(os.environ.get("TOX_MSSQL", "--db mssql").split()) + cmd.extend("--write-idents db_idents.txt".split()) + + posargs, opts = extract_opts(session.posargs, "generate-junit") + if opts.generate_junit: + if len(databases) == 1: + cmd.extend(["--junitxml", f"junit-{databases[0]}.xml"]) + else: + cmd.extend(["--junitxml", "junit-general.xml"]) + + cmd.extend(posargs) + + session.run(*cmd) + + # Run cleanup for oracle/mssql + for database in databases: + if database in ["oracle", "mssql"]: + session.run("python", "reap_dbs.py", "db_idents.txt") + + +@nox.session(name="pep484") +def mypy_check(session: nox.Session) -> None: + """Run mypy type checking.""" + + session.install(*nox.project.dependency_groups(pyproject, "mypy")) + + session.install("-e", ".") + + session.run( + "mypy", "noxfile.py", "./alembic/", "--exclude", "alembic/templates" + ) + + +@nox.session(name="pep8") +def lint(session: nox.Session) -> None: + """Run linting and formatting checks.""" + + session.install(*nox.project.dependency_groups(pyproject, "lint")) + + file_paths = [ + "./alembic/", + "./tests/", + "./tools/", + "noxfile.py", + "docs/build/conf.py", + ] + session.run("flake8", *file_paths) + session.run("black", "--check", *file_paths) + + +@nox.session(name="pyoptimize") +@tox_parameters(["python"], [PYTHON_VERSIONS], base_tag="pyoptimize") +def test_pyoptimize(session: nox.Session) -> None: + """Run the script consumption suite against .pyo files rather than .pyc""" + + session.install(*nox.project.dependency_groups(pyproject, "test")) + session.install(".") + + session.env["PYTHONOPTIMIZE"] = "1" + session.env["SQLALCHEMY_WARN_20"] = "1" + + cmd = [ + "python", + "-m", + "pytest", + ] + cmd.extend(os.environ.get("TOX_WORKERS", "-n4").split()) + cmd.append("tests/test_script_consumption.py") + cmd.extend(session.posargs) + session.run(*cmd) diff --git a/pyproject.toml b/pyproject.toml index 7c648165..2ddb1ae6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,40 @@ tz = ["tzdata"] [project.scripts] alembic = "alembic.config:main" +[dependency-groups] +tests = [ + "pytest>8,<8.4", + "pytest-xdist", + "black==25.1.0" # for test_post_write.py +] + +coverage = [ + "pytest-cov" +] + +tests_postgresql = ["psycopg2>=2.7"] +tests_mysql = ["mysqlclient>=1.4.0", "pymysql"] +tests_oracle = ["cx_oracle", "oracledb"] +tests_mssql = ["pyodbc"] + +lint = [ + "flake8", + "flake8-import-order>=0.19.2", + "flake8-import-single==0.1.5", + "flake8-builtins", + "flake8-docstrings", + "flake8-rst-docstrings", + "pydocstyle<4.0.0", + "pygments", + "black==25.1.0" +] + +mypy = [ + "mypy>=1.16.0", + "nox", # because we check noxfile.py + "pytest>8,<8.4", # alembic/testing imports pytest +] + [tool.setuptools] include-package-data = true zip-safe = false diff --git a/setup.cfg b/setup.cfg index 3191a664..282ce17b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,9 +31,9 @@ postgresql=postgresql://scott:tiger@127.0.0.1:5432/test psycopg=postgresql+psycopg://scott:tiger@127.0.0.1:5432/test mysql=mysql://scott:tiger@127.0.0.1:3306/test?charset=utf8mb4 mariadb=mariadb://scott:tiger@127.0.0.1:3306/test?charset=utf8mb4 -mssql=mssql+pyodbc://scott:tiger^5HHH@mssql2017:1433/test?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes -oracle=oracle://scott:tiger@127.0.0.1:1521 -oracle8=oracle://scott:tiger@127.0.0.1:1521/?use_ansi=0 +mssql=mssql+pyodbc://scott:tiger^5HHH@mssql2022:1433/test?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes&Encrypt=Optional +oracle=oracle://scott:tiger@oracle18c/xe + diff --git a/tools/toxnox.py b/tools/toxnox.py new file mode 100644 index 00000000..09ba5260 --- /dev/null +++ b/tools/toxnox.py @@ -0,0 +1,211 @@ +"""Provides the tox_parameters() utility, which generates parameterized +sections for nox tests, which include tags that indicate various combinations +of those parameters in such a way that it's somewhat similar to how +we were using the tox project; where individual dash-separated tags could +be added to add more specificity to the suite configuation, or omitting them +would fall back to defaults. + + +""" + +from __future__ import annotations + +import collections +import re +import sys +from typing import Any +from typing import Callable +from typing import List +from typing import Optional +from typing import Sequence +from typing import Tuple + +import nox + +OUR_PYTHON = f"{sys.version_info.major}.{sys.version_info.minor}" + + +def tox_parameters( + names: Sequence[str], + token_lists: Sequence[Sequence[str]], + *, + base_tag: Optional[str] = None, + filter_: Optional[Callable[..., bool]] = None, + always_include_in_tag: Optional[Sequence[str]] = None, +) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + r"""Decorator to create a parameter/tagging structure for a nox session + function that acts to a large degree like tox's generative environments. + + The output is a ``nox.parametrize()`` decorator that's built up from + individual ``nox.param()`` instances. + + :param names: names of the parameters sent to the session function. + These names go straight to the first argument of ``nox.parametrize()`` + and should all match argument names accepted by the decorated function + (except for ``python``, which is optional). + :param token_lists: a sequence of lists of values for each parameter. a + ``nox.param()`` will be created for the full product of these values, + minus those filtered out using the ``filter_`` callable. These tokens + are used to create the args, tags, and ids of each ``nox.param()``. The + list of tags will be generated out including all values for a parameter + joined by ``-``, as well as combinations that include a subset of those + values, where the omitted elements of the tag are implicitly considered to + match the "default" value, indicated by them being first in their + collection (with the exception of "python", where the current python in + use is the default). Additionally, values that start with an underscore + are omitted from all ids and tags. Values that refer to Python versions + wlil be expanded to the full Python executable name when passed as + arguments to the session function, which is currently a workaround to + allow free-threaded python interpreters to be located. + :param base_tag: optional tag that will be appended to all tags generated, + e.g. if the decorator yields tags like ``python314-x86-windows``, a + ``basetag`` value of ``all`` would yield the + tag as ``python314-x86-windows-all``. + :param filter\_: optional filtering function, must accept keyword arguments + matching the names in ``names``. Returns True or False indicating if + a certain tag combination should be included. + :param always_include_in_tag: list of names from ``names`` that indicate + parameters that should always be part of all tags, and not be omitted + as a "default" + + + """ + + PY_RE = re.compile(r"(?:python)?([234]\.\d+t?)") + + def _is_py_version(token): + return bool(PY_RE.match(token)) + + def _expand_python_version(token): + """expand pyx.y(t) tags into executable names. + + Works around nox issue fixed at + https://github.com/wntrblm/nox/pull/999 by providing full executable + name + + """ + m = PY_RE.match(token) + if m: + return f"python{m.group(1)}" + else: + return token + + def _python_to_tag(token): + m = PY_RE.match(token) + if m: + return f"py{m.group(1).replace('.', '')}" + else: + return token + + if always_include_in_tag: + name_to_list = dict(zip(names, token_lists)) + must_be_present = [ + name_to_list[name] for name in always_include_in_tag + ] + else: + must_be_present = None + + def _recur_param(prevtokens, prevtags, token_lists): + + if not token_lists: + return + + tokens = token_lists[0] + remainder = token_lists[1:] + + for i, token in enumerate(tokens): + + if _is_py_version(token): + is_our_python = token == OUR_PYTHON + tokentag = _python_to_tag(token) + is_default_token = is_our_python + else: + is_our_python = False + tokentag = token + is_default_token = i == 0 + + if is_our_python: + our_python_tags = ["py"] + else: + our_python_tags = [] + + if not tokentag.startswith("_"): + tags = ( + prevtags + + [tokentag] + + [tag + "-" + tokentag for tag in prevtags] + + our_python_tags + ) + else: + tags = prevtags + our_python_tags + + if remainder: + for args, newtags, ids in _recur_param( + prevtokens + [token], tags, remainder + ): + if not is_default_token: + newtags = [ + t + for t in newtags + if tokentag in t or t in our_python_tags + ] + + yield args, newtags, ids + else: + if not is_default_token: + newtags = [ + t + for t in tags + if tokentag in t or t in our_python_tags + ] + else: + newtags = tags + + if base_tag: + newtags = [t + f"-{base_tag}" for t in newtags] + if must_be_present: + for t in list(newtags): + for required_tokens in must_be_present: + if not any(r in t for r in required_tokens): + newtags.remove(t) + break + + yield prevtokens + [token], newtags, "-".join( + _python_to_tag(t) + for t in prevtokens + [token] + if not t.startswith("_") + ) + + params = [ + nox.param( + *[_expand_python_version(a) for a in args], tags=tags, id=ids + ) + for args, tags, ids in _recur_param([], [], token_lists) + if filter_ is None or filter_(**dict(zip(names, args))) + ] + + # for p in params: + # print(f"PARAM {'-'.join(p.args)} TAGS {p.tags}") + # breakpoint() + + return nox.parametrize(names, params) + + +def extract_opts(posargs: List[str], *args: str) -> Tuple[List[str], Any]: + + underscore_args = [arg.replace("-", "_") for arg in args] + return_tuple = collections.namedtuple("options", underscore_args) + + look_for_args = {f"--{arg}": idx for idx, arg in enumerate(args)} + return_args = [False for arg in args] + + def extract(arg: str): + if arg in look_for_args: + return_args[look_for_args[arg]] = True + return True + else: + return False + + return [arg for arg in posargs if not extract(arg)], return_tuple( + *return_args + ) diff --git a/tools/warn_tox.py b/tools/warn_tox.py new file mode 100644 index 00000000..a4530b89 --- /dev/null +++ b/tools/warn_tox.py @@ -0,0 +1,12 @@ +def warn_tox(): + print( + "\n" + + "=" * 80 + + "\n\033[1;31m ⚠️ NOTE: TOX IS DEPRECATED IN THIS PROJECT! ⚠️" + "\033[0m\n\033[1;33m " + "Please use nox instead for running tests.\033[0m\n" + "=" * 80 + "\n" + ) + + +if __name__ == "__main__": + warn_tox() diff --git a/tox.ini b/tox.ini index cc254c33..6f3f8af8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,3 +1,8 @@ +### +### NOTE: use of tox is deprecated, and this file is no longer maintained and +### will be removed at some point. Testing is now run via nox. +### + [tox] envlist = py-sqlalchemy @@ -63,6 +68,7 @@ passenv= commands= {env:BASECOMMAND} {env:WORKERS} {env:SQLITE:} {env:POSTGRESQL:} {env:MYSQL:} {env:ORACLE:} {env:MSSQL:} {env:BACKENDONLY:} {env:COVERAGE:} {env:LIMITTESTS:} {posargs} {oracle,mssql}: python reap_dbs.py db_idents.txt + python tools/warn_tox.py [testenv:pep484] -- 2.47.3