From 70463e9cae6cf32fb3b0e881a6213fcc690fce12 Mon Sep 17 00:00:00 2001 From: Mihail Milushev Date: Tue, 22 Aug 2023 14:07:30 -0400 Subject: [PATCH] Implement new `exec` write-hook runner that will execute arbitrary binaries Added new feature to the "code formatter" function which allows standalone executable tools to be run against code, without going through the Python interpreter. Known as the ``exec`` runner, it complements the existing ``console_scripts`` runner by allowing non-Python tools such as ``ruff`` to be used. Pull request courtesy Mihail Milushev. Fixes: #1275 Closes: #1276 Pull-request: https://github.com/sqlalchemy/alembic/pull/1276 Pull-request-sha: 24e576b9d8aa0d3931f2b0c0c15bb370577d7485 Change-Id: Ie28f2b7faf2fb8493c8e704f51e3e3524982e346 --- alembic/__init__.py | 2 +- alembic/script/write_hooks.py | 27 ++++ alembic/templates/async/alembic.ini.mako | 6 + alembic/templates/generic/alembic.ini.mako | 6 + alembic/templates/multidb/alembic.ini.mako | 6 + docs/build/autogenerate.rst | 32 +++-- docs/build/tutorial.rst | 6 + docs/build/unreleased/1275.rst | 15 +++ tests/test_post_write.py | 145 +++++++++++++++++++++ 9 files changed, 234 insertions(+), 11 deletions(-) create mode 100644 docs/build/unreleased/1275.rst diff --git a/alembic/__init__.py b/alembic/__init__.py index e9698de5..f391826a 100644 --- a/alembic/__init__.py +++ b/alembic/__init__.py @@ -3,4 +3,4 @@ import sys from . import context from . import op -__version__ = "1.11.4" +__version__ = "1.12.0" diff --git a/alembic/script/write_hooks.py b/alembic/script/write_hooks.py index 6ca19115..b44ce644 100644 --- a/alembic/script/write_hooks.py +++ b/alembic/script/write_hooks.py @@ -147,3 +147,30 @@ def console_scripts( cwd=cwd, **kw, ) + + +@register("exec") +def exec_(path: str, options: dict, ignore_output: bool = False) -> None: + try: + executable = options["executable"] + except KeyError as ke: + raise util.CommandError( + f"Key {options['_hook_name']}.executable is required for post " + f"write hook {options['_hook_name']!r}" + ) from ke + cwd: Optional[str] = options.get("cwd", None) + cmdline_options_str = options.get("options", "") + cmdline_options_list = _parse_cmdline_options(cmdline_options_str, path) + + kw: Dict[str, Any] = {} + if ignore_output: + kw["stdout"] = kw["stderr"] = subprocess.DEVNULL + + subprocess.run( + [ + executable, + *cmdline_options_list, + ], + cwd=cwd, + **kw, + ) diff --git a/alembic/templates/async/alembic.ini.mako b/alembic/templates/async/alembic.ini.mako index 64c7b6b9..bc9f2d50 100644 --- a/alembic/templates/async/alembic.ini.mako +++ b/alembic/templates/async/alembic.ini.mako @@ -72,6 +72,12 @@ sqlalchemy.url = driver://user:pass@localhost/dbname # black.entrypoint = black # black.options = -l 79 REVISION_SCRIPT_FILENAME +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + # Logging configuration [loggers] keys = root,sqlalchemy,alembic diff --git a/alembic/templates/generic/alembic.ini.mako b/alembic/templates/generic/alembic.ini.mako index f541b179..c18ddb4e 100644 --- a/alembic/templates/generic/alembic.ini.mako +++ b/alembic/templates/generic/alembic.ini.mako @@ -74,6 +74,12 @@ sqlalchemy.url = driver://user:pass@localhost/dbname # black.entrypoint = black # black.options = -l 79 REVISION_SCRIPT_FILENAME +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + # Logging configuration [loggers] keys = root,sqlalchemy,alembic diff --git a/alembic/templates/multidb/alembic.ini.mako b/alembic/templates/multidb/alembic.ini.mako index 4230fe13..a9ea0755 100644 --- a/alembic/templates/multidb/alembic.ini.mako +++ b/alembic/templates/multidb/alembic.ini.mako @@ -79,6 +79,12 @@ sqlalchemy.url = driver://user:pass@localhost/dbname2 # black.entrypoint = black # black.options = -l 79 REVISION_SCRIPT_FILENAME +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + # Logging configuration [loggers] keys = root,sqlalchemy,alembic diff --git a/docs/build/autogenerate.rst b/docs/build/autogenerate.rst index 5f34c0a0..33a8abd6 100644 --- a/docs/build/autogenerate.rst +++ b/docs/build/autogenerate.rst @@ -714,13 +714,14 @@ regardless of whether or not the autogenerate feature was used. rather than at commit time, and also can be useful for projects that prefer not to use pre-commit. +.. _post_write_hooks_config: -Basic Formatter Configuration -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Basic Post Processor Configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The ``alembic.ini`` samples now include commented-out configuration -illustrating how to configure code-formatting tools to run against the newly -generated file path. Example:: +illustrating how to configure code-formatting tools, or other tools like linters +to run against the newly generated file path. Example:: [post_write_hooks] @@ -736,13 +737,16 @@ Above, we configure ``hooks`` to be a single post write hook labeled configuration for the ``"black"`` post write hook, which includes: * ``type`` - this is the type of hook we are running. Alembic includes - a hook runner called ``"console_scripts"``, which is specifically a + two hook runners: ``"console_scripts"``, which is specifically a Python function that uses ``subprocess.run()`` to invoke a separate - Python script against the revision file. For a custom-written hook - function, this configuration variable would refer to the name under + Python script against the revision file; and ``"exec"``, which uses + ``subprocess.run()`` to execute an arbitrary binary. For a custom-written + hook function, this configuration variable would refer to the name under which the custom hook was registered; see the next section for an example. -The following configuration options are specific to the ``"console_scripts"`` +.. versionadded:: 1.12 added new ``exec`` runner + +The following configuration option is specific to the ``"console_scripts"`` hook runner: * ``entrypoint`` - the name of the `setuptools entrypoint `_ @@ -750,6 +754,14 @@ hook runner: Python console scripts, this name will match the name of the shell command that is usually run for the code formatting tool, in this case ``black``. +The following configuration option is specific to the ``"exec"`` hook runner: + +* ``executable`` - the name of the executable to invoke. Can be either a +bare executable name which will be searched in ``$PATH``, or a full pathname +to avoid potential issues with path interception. + +The following options are supported by both ``"console_scripts"`` and ``"exec"``: + * ``options`` - a line of command-line options that will be passed to the code formatting tool. In this case, we want to run the command ``black /path/to/revision.py -l 79``. By default, the revision path is @@ -767,7 +779,7 @@ hook runner: autopep8.entrypoint = autopep8 autopep8.options = --in-place REVISION_SCRIPT_FILENAME -* ``cwd`` - optional working directory from which the console script is run. +* ``cwd`` - optional working directory from which the code processing tool is run. When running ``alembic revision -m "rev1"``, we will now see the ``black`` tool's output as well:: @@ -916,4 +928,4 @@ by contrast, when no new operations are detected:: such as :paramref:`.EnvironmentContext.configure.compare_type` and :paramref:`.EnvironmentContext.configure.compare_server_default` are in play as usual, as well as that limitations in autogenerate - detection are the same when running ``alembic check``. \ No newline at end of file + detection are the same when running ``alembic check``. diff --git a/docs/build/tutorial.rst b/docs/build/tutorial.rst index 2974768d..6e9552c5 100644 --- a/docs/build/tutorial.rst +++ b/docs/build/tutorial.rst @@ -199,6 +199,12 @@ The file generated with the "generic" configuration looks like:: # black.entrypoint = black # black.options = -l 79 REVISION_SCRIPT_FILENAME + # lint with attempts to fix using "ruff" - use the exec runner, execute a binary + # hooks = ruff + # ruff.type = exec + # ruff.executable = %(here)s/.venv/bin/ruff + # ruff.options = --fix REVISION_SCRIPT_FILENAME + # Logging configuration [loggers] keys = root,sqlalchemy,alembic diff --git a/docs/build/unreleased/1275.rst b/docs/build/unreleased/1275.rst new file mode 100644 index 00000000..6ac40092 --- /dev/null +++ b/docs/build/unreleased/1275.rst @@ -0,0 +1,15 @@ +.. change:: + :tags: feature, autogenerate + :tickets: 1275 + + Added new feature to the "code formatter" function which allows standalone + executable tools to be run against code, without going through the Python + interpreter. Known as the ``exec`` runner, it complements the existing + ``console_scripts`` runner by allowing non-Python tools such as ``ruff`` to + be used. Pull request courtesy Mihail Milushev. + + .. seealso:: + + :ref:`post_write_hooks_config` + + diff --git a/tests/test_post_write.py b/tests/test_post_write.py index 85e95fb9..2027ab9a 100644 --- a/tests/test_post_write.py +++ b/tests/test_post_write.py @@ -256,3 +256,148 @@ black.cwd = /path/to/cwd self._run_black_with_config( input_config, expected_additional_arguments_fn, cwd="/path/to/cwd" ) + + def test_exec_executable_missing(self): + self.cfg = _no_sql_testing_config( + directives=( + "\n[post_write_hooks]\n" "hooks=ruff\n" "ruff.type=exec\n" + ) + ) + assert_raises_message( + util.CommandError, + "Key ruff.executable is required for post write hook 'ruff'", + command.revision, + self.cfg, + message="x", + ) + + def _run_ruff_with_config( + self, + input_config, + expected_additional_arguments_fn, + executable="ruff", + cwd=None, + ): + self.cfg = _no_sql_testing_config(directives=input_config) + + with mock.patch( + "alembic.script.write_hooks.subprocess" + ) as mock_subprocess: + rev = command.revision(self.cfg, message="x") + + eq_( + mock_subprocess.mock_calls, + [ + mock.call.run( + [ + executable, + ] + + expected_additional_arguments_fn(rev.path), + cwd=cwd, + ) + ], + ) + + def test_exec_with_path_search(self): + input_config = """ +[post_write_hooks] +hooks = ruff +ruff.type = exec +ruff.executable = ruff +ruff.options = --fix + """ + + def expected_additional_arguments_fn(rev_path): + return [rev_path, "--fix"] + + self._run_ruff_with_config( + input_config, expected_additional_arguments_fn + ) + + def test_exec_with_full_pathname(self): + input_config = """ +[post_write_hooks] +hooks = ruff +ruff.type = exec +ruff.executable = %(here)s/.venv/bin/ruff +ruff.options = --fix + """ + + def expected_additional_arguments_fn(rev_path): + return [rev_path, "--fix"] + + self._run_ruff_with_config( + input_config, + expected_additional_arguments_fn, + executable=os.path.abspath(_get_staging_directory()) + + "/.venv/bin/ruff", + ) + + @combinations(True, False) + def test_exec_with_filename_interpolation(self, posix): + input_config = """ +[post_write_hooks] +hooks = ruff +ruff.type = exec +ruff.executable = ruff +ruff.options = arg1 REVISION_SCRIPT_FILENAME 'multi-word arg' \ + --flag1='REVISION_SCRIPT_FILENAME' + """ + + def expected_additional_arguments_fn(rev_path): + if compat.is_posix: + return [ + "arg1", + rev_path, + "multi-word arg", + "--flag1=" + rev_path, + ] + else: + return [ + "arg1", + rev_path, + "'multi-word arg'", + "--flag1='%s'" % rev_path, + ] + + with mock.patch("alembic.util.compat.is_posix", posix): + self._run_ruff_with_config( + input_config, expected_additional_arguments_fn + ) + + def test_exec_with_path_in_config(self): + input_config = """ +[post_write_hooks] +hooks = ruff +ruff.type = exec +ruff.executable = ruff +ruff.options = arg1 REVISION_SCRIPT_FILENAME --config %(here)s/pyproject.toml + """ + + def expected_additional_arguments_fn(rev_path): + return [ + "arg1", + rev_path, + "--config", + os.path.abspath(_get_staging_directory()) + "/pyproject.toml", + ] + + self._run_ruff_with_config( + input_config, expected_additional_arguments_fn + ) + + def test_exec_with_cwd(self): + input_config = """ +[post_write_hooks] +hooks = ruff +ruff.type = exec +ruff.executable = ruff +ruff.cwd = /path/to/cwd + """ + + def expected_additional_arguments_fn(rev_path): + return [rev_path] + + self._run_ruff_with_config( + input_config, expected_additional_arguments_fn, cwd="/path/to/cwd" + ) -- 2.47.2