From: Mike Bayer Date: Tue, 13 May 2025 14:14:10 +0000 (-0400) Subject: allow pep 621 configuration X-Git-Tag: rel_1_16_0~16^2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a52a18f7f7d34148d869de442c9b448809c49672;p=thirdparty%2Fsqlalchemy%2Falembic.git allow pep 621 configuration Added optional :pep:`621` support to Alembic, where a subset of the project-centric configuration normally found in the ``alembic.ini`` file can now be retrieved from the project-wide ``pyproject.toml`` file. A new init template ``pyproject`` is added which illustrates a basic :pep:`621` setup. The :pep:`621` feature supports configuration values that are relevant to code locations and code production only; it does not accommodate database connectivity, configuration, or logging configuration. These latter configurational elements remain as elements that can be present either in the ``alembic.ini`` file, or retrieved elsewhere within the ``env.py`` file. The change also allows the ``alembic.ini`` file to be completely optional if the ``pyproject.toml`` file contains a base alembic configuration section. Fixes: #1082 Change-Id: I7c6d1e68eb570bc5273e3500a17b342dfbd69d1b --- diff --git a/alembic/command.py b/alembic/command.py index 0ae1d9a8..40f23610 100644 --- a/alembic/command.py +++ b/alembic/command.py @@ -12,6 +12,7 @@ from . import autogenerate as autogen from . import util from .runtime.environment import EnvironmentContext from .script import ScriptDirectory +from .util import compat if TYPE_CHECKING: from alembic.config import Config @@ -82,8 +83,32 @@ def init( ): os.makedirs(versions) + if not os.path.isabs(directory): + # for non-absolute path, state config file in .ini / pyproject + # as relative to the %(here)s token, which is where the config + # file itself would be + + if config.config_file_name is not None: + rel_dir = util.relpath_via_abs_root( + os.path.abspath(config.config_file_name), directory + ) + + ini_script_location_directory = os.path.join("%(here)s", rel_dir) + if config.toml_file_name is not None: + rel_dir = util.relpath_via_abs_root( + os.path.abspath(config.toml_file_name), directory + ) + + toml_script_location_directory = os.path.join("%(here)s", rel_dir) + + else: + ini_script_location_directory = directory + toml_script_location_directory = directory + script = ScriptDirectory(directory) + has_toml = False + config_file: str | None = None for file_ in os.listdir(template_dir): file_path = os.path.join(template_dir, file_) @@ -97,8 +122,38 @@ def init( ) else: script._generate_template( - file_path, config_file, script_location=directory + file_path, + config_file, + script_location=ini_script_location_directory, ) + elif file_ == "pyproject.toml.mako": + has_toml = True + assert config.toml_file_name is not None + toml_file = os.path.abspath(config.toml_file_name) + + if os.access(toml_file, os.F_OK): + with open(toml_file, "rb") as f: + toml_data = compat.tomllib.load(f) + if "tool" in toml_data and "alembic" in toml_data["tool"]: + + util.msg( + f"File {config.toml_file_name!r} already exists " + "and already has a [tool.alembic] section, " + "skipping", + ) + continue + script._append_template( + file_path, + toml_file, + script_location=toml_script_location_directory, + ) + else: + script._generate_template( + file_path, + toml_file, + script_location=toml_script_location_directory, + ) + elif os.path.isfile(file_path): output_file = os.path.join(directory, file_) script._copy_file(file_path, output_file) @@ -113,11 +168,20 @@ def init( pass assert config_file is not None - util.msg( - "Please edit configuration/connection/logging " - f"settings in {config_file!r} before proceeding.", - **config.messaging_opts, - ) + + if has_toml: + util.msg( + f"Please edit configuration settings in {toml_file!r} and " + "configuration/connection/logging " + f"settings in {config_file!r} before proceeding.", + **config.messaging_opts, + ) + else: + util.msg( + "Please edit configuration/connection/logging " + f"settings in {config_file!r} before proceeding.", + **config.messaging_opts, + ) def revision( diff --git a/alembic/config.py b/alembic/config.py index dc7d3f81..3b3f455c 100644 --- a/alembic/config.py +++ b/alembic/config.py @@ -83,12 +83,13 @@ class Config: Defaults to ``sys.stdout``. :param config_args: A dictionary of keys and values that will be used - for substitution in the alembic config file. The dictionary as given - is **copied** to a new one, stored locally as the attribute - ``.config_args``. When the :attr:`.Config.file_config` attribute is - first invoked, the replacement variable ``here`` will be added to this - dictionary before the dictionary is passed to ``ConfigParser()`` - to parse the .ini file. + for substitution in the alembic config file, as well as the pyproject.toml + file, depending on which / both are used. The dictionary as given is + **copied** to two new, independent dictionaries, stored locally under the + attributes ``.config_args`` and ``.toml_args``. Both of these + dictionaries will also be populated with the replacement variable + ``%(here)s``, which refers to the location of the .ini and/or .toml file + as appropriate. :param attributes: optional dictionary of arbitrary Python keys/values, which will be populated into the :attr:`.Config.attributes` dictionary. @@ -102,6 +103,7 @@ class Config: def __init__( self, file_: Union[str, os.PathLike[str], None] = None, + toml_file: Union[str, os.PathLike[str], None] = None, ini_section: str = "alembic", output_buffer: Optional[TextIO] = None, stdout: TextIO = sys.stdout, @@ -111,11 +113,13 @@ class Config: ) -> None: """Construct a new :class:`.Config`""" self.config_file_name = file_ + self.toml_file_name = toml_file self.config_ini_section = ini_section self.output_buffer = output_buffer self.stdout = stdout self.cmd_opts = cmd_opts self.config_args = dict(config_args) + self.toml_args = dict(config_args) if attributes: self.attributes.update(attributes) @@ -208,6 +212,26 @@ class Config: file_config.add_section(self.config_ini_section) return file_config + @util.memoized_property + def toml_alembic_config(self) -> Mapping[str, Any]: + """Return a dictionary of the [tool.alembic] section from + pyproject.toml""" + + if self.toml_file_name and os.path.exists(self.toml_file_name): + + here = os.path.abspath(os.path.dirname(self.toml_file_name)) + self.toml_args["here"] = here + + with open(self.toml_file_name, "rb") as f: + toml_data = compat.tomllib.load(f) + data = toml_data.get("tool", {}).get("alembic", {}) + if not isinstance(data, dict): + raise util.CommandError("Incorrect TOML format") + return data + + else: + return {} + def get_template_directory(self) -> str: """Return the directory where Alembic setup templates are found. @@ -280,6 +304,12 @@ class Config: The value here will override whatever was in the .ini file. + Does **NOT** consume from the pyproject.toml file. + + .. seealso:: + + :meth:`.Config.get_alembic_option` - includes pyproject support + :param section: name of the section :param name: name of the value @@ -328,9 +358,74 @@ class Config: section, unless the ``-n/--name`` flag were used to indicate a different section. + Does **NOT** consume from the pyproject.toml file. + + .. seealso:: + + :meth:`.Config.get_alembic_option` - includes pyproject support + """ return self.get_section_option(self.config_ini_section, name, default) + @overload + def get_alembic_option(self, name: str, default: str) -> str: ... + + @overload + def get_alembic_option( + self, name: str, default: Optional[str] = None + ) -> Optional[str]: ... + + def get_alembic_option( + self, name: str, default: Optional[str] = None + ) -> Union[None, str, list[str], dict[str, str]]: + """Return an option from the "[alembic]" or "[tool.alembic]" section + of the configparser-parsed .ini file (e.g. ``alembic.ini``) or + toml-parsed ``pyproject.toml`` file. + + The value returned is expected to be None, string, list of strings, + or dictionary of strings. Within each type of string value, the + ``%(here)s`` token is substituted out with the absolute path of the + ``pyproject.toml`` file, as are other tokens which are extracted from + the :paramref:`.Config.config_args` dictionary. + + Searches always prioritize the configparser namespace first, before + searching in the toml namespace. + + If Alembic was run using the ``-n/--name`` flag to indicate an + alternate main section name, this is taken into account **only** for + the configparser-parsed .ini file. The section name in toml is always + ``[tool.alembic]``. + + + .. versionadded:: 1.16.0 + + """ + + if self.file_config.has_option(self.config_ini_section, name): + return self.file_config.get(self.config_ini_section, name) + else: + USE_DEFAULT = object() + value: Union[None, str, list[str], dict[str, str]] = ( + self.toml_alembic_config.get(name, USE_DEFAULT) + ) + if value is USE_DEFAULT: + return default + if value is not None: + if isinstance(value, str): + value = value % (self.toml_args) + elif isinstance(value, list): + value = cast( + "list[str]", [v % (self.toml_args) for v in value] + ) + elif isinstance(value, dict): + value = cast( + "dict[str, str]", + {k: v % (self.toml_args) for k, v in value.items()}, + ) + else: + raise util.CommandError("unsupported TOML value type") + return value + @util.memoized_property def messaging_opts(self) -> MessagingOptions: """The messaging options.""" @@ -401,9 +496,10 @@ class Config: if x ] else: - return None + return self.toml_alembic_config.get("version_locations", None) def get_prepend_sys_paths_list(self) -> Optional[list[str]]: + prepend_sys_path_str = self.get_main_option("prepend_sys_path") if prepend_sys_path_str: @@ -428,26 +524,35 @@ class Config: if x ] else: - return None + return self.toml_alembic_config.get("prepend_sys_path", None) def get_hooks_list(self) -> list[PostWriteHookConfig]: - _split_on_space_comma = re.compile(r", *|(?: +)") - - hook_config = self.get_section("post_write_hooks", {}) - names = _split_on_space_comma.split(hook_config.get("hooks", "")) hooks: list[PostWriteHookConfig] = [] - for name in names: - if not name: - continue - opts = { - key[len(name) + 1 :]: hook_config[key] - for key in hook_config - if key.startswith(name + ".") - } - opts["_hook_name"] = name - hooks.append(opts) + if not self.file_config.has_section("post_write_hooks"): + hook_config = self.toml_alembic_config.get("post_write_hooks", {}) + for cfg in hook_config: + opts = dict(cfg) + opts["_hook_name"] = opts.pop("name") + hooks.append(opts) + + else: + _split_on_space_comma = re.compile(r", *|(?: +)") + hook_config = self.get_section("post_write_hooks", {}) + names = _split_on_space_comma.split(hook_config.get("hooks", "")) + + for name in names: + if not name: + continue + opts = { + key[len(name) + 1 :]: hook_config[key] + for key in hook_config + if key.startswith(name + ".") + } + + opts["_hook_name"] = name + hooks.append(opts) return hooks @@ -632,17 +737,19 @@ class CommandLine: parser.add_argument( "-c", "--config", - type=str, - default=os.environ.get("ALEMBIC_CONFIG", "alembic.ini"), + action="append", help="Alternate config file; defaults to value of " - 'ALEMBIC_CONFIG environment variable, or "alembic.ini"', + 'ALEMBIC_CONFIG environment variable, or "alembic.ini". ' + "May also refer to pyproject.toml file. May be specified twice " + "to reference both files separately", ) parser.add_argument( "-n", "--name", type=str, default="alembic", - help="Name of section in .ini file to use for Alembic config", + help="Name of section in .ini file to use for Alembic config " + "(only applies to configparser config, not toml)", ) parser.add_argument( "-x", @@ -752,6 +859,46 @@ class CommandLine: else: util.err(str(e), **config.messaging_opts) + def _inis_from_config(self, options: Namespace) -> tuple[str, str]: + names = options.config + + alembic_config_env = os.environ.get("ALEMBIC_CONFIG") + if ( + alembic_config_env + and os.path.basename(alembic_config_env) == "pyproject.toml" + ): + default_pyproject_toml = alembic_config_env + default_alembic_config = "alembic.ini" + elif alembic_config_env: + default_pyproject_toml = "pyproject.toml" + default_alembic_config = alembic_config_env + else: + default_alembic_config = "alembic.ini" + default_pyproject_toml = "pyproject.toml" + + if not names: + return default_pyproject_toml, default_alembic_config + + toml = ini = None + + for name in names: + if os.path.basename(name) == "pyproject.toml": + if toml is not None: + raise util.CommandError( + "pyproject.toml indicated more than once" + ) + toml = name + else: + if ini is not None: + raise util.CommandError( + "only one ini file may be indicated" + ) + ini = name + + return toml if toml else default_pyproject_toml, ( + ini if ini else default_alembic_config + ) + def main(self, argv: Optional[Sequence[str]] = None) -> None: """Executes the command line with the provided arguments.""" options = self.parser.parse_args(argv) @@ -760,8 +907,10 @@ class CommandLine: # behavior changed incompatibly in py3.3 self.parser.error("too few arguments") else: + toml, ini = self._inis_from_config(options) cfg = Config( - file_=options.config, + file_=ini, + toml_file=toml, ini_section=options.name, cmd_opts=options, ) diff --git a/alembic/script/base.py b/alembic/script/base.py index 3fa3c282..e1a88b78 100644 --- a/alembic/script/base.py +++ b/alembic/script/base.py @@ -162,13 +162,13 @@ class ScriptDirectory: present. """ - script_location = config.get_main_option("script_location") + script_location = config.get_alembic_option("script_location") if script_location is None: raise util.CommandError( "No 'script_location' key found in configuration." ) truncate_slug_length: Optional[int] - tsl = config.get_main_option("truncate_slug_length") + tsl = config.get_alembic_option("truncate_slug_length") if tsl is not None: truncate_slug_length = int(tsl) else: @@ -178,17 +178,21 @@ class ScriptDirectory: if prepend_sys_path: sys.path[:0] = prepend_sys_path - rvl = config.get_main_option("recursive_version_locations") == "true" + rvl = ( + config.get_alembic_option("recursive_version_locations") == "true" + ) return ScriptDirectory( util.coerce_resource_to_filename(script_location), - file_template=config.get_main_option( + file_template=config.get_alembic_option( "file_template", _default_file_template ), truncate_slug_length=truncate_slug_length, - sourceless=config.get_main_option("sourceless") == "true", - output_encoding=config.get_main_option("output_encoding", "utf-8"), + sourceless=config.get_alembic_option("sourceless") == "true", + output_encoding=config.get_alembic_option( + "output_encoding", "utf-8" + ), version_locations=config.get_version_locations_list(), - timezone=config.get_main_option("timezone"), + timezone=config.get_alembic_option("timezone"), hooks=config.get_hooks_list(), recursive_version_locations=rvl, messaging_opts=config.messaging_opts, @@ -542,6 +546,15 @@ class ScriptDirectory: def env_py_location(self) -> str: return os.path.abspath(os.path.join(self.dir, "env.py")) + def _append_template(self, src: str, dest: str, **kw: Any) -> None: + with util.status( + f"Appending to existing {os.path.abspath(dest)}", + **self.messaging_opts, + ): + util.template_to_file( + src, dest, self.output_encoding, append=True, **kw + ) + def _generate_template(self, src: str, dest: str, **kw: Any) -> None: with util.status( f"Generating {os.path.abspath(dest)}", **self.messaging_opts diff --git a/alembic/script/write_hooks.py b/alembic/script/write_hooks.py index 64bd3873..1877bdd8 100644 --- a/alembic/script/write_hooks.py +++ b/alembic/script/write_hooks.py @@ -71,7 +71,8 @@ def _run_hooks(path: str, hooks: list[PostWriteHookConfig]) -> None: type_ = hook["type"] except KeyError as ke: raise util.CommandError( - f"Key {name}.type is required for post write hook {name!r}" + f"Key '{name}.type' (or 'type' in toml) is required " + f"for post write hook {name!r}" ) from ke else: with util.status( diff --git a/alembic/templates/async/alembic.ini.mako b/alembic/templates/async/alembic.ini.mako index 1eb347c0..c46fe744 100644 --- a/alembic/templates/async/alembic.ini.mako +++ b/alembic/templates/async/alembic.ini.mako @@ -2,11 +2,15 @@ [alembic] # path to migration scripts. -# Use forward slashes (/) also on windows to provide an os agnostic path +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file script_location = ${script_location} # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s # Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens # file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s # sys.path path, will be prepended to sys.path if present. @@ -35,11 +39,11 @@ prepend_sys_path = . # sourceless = false # version location specification; This defaults -# to ${script_location}/versions. When using multiple version +# to /versions. When using multiple version # directories, initial revisions must be specified with --version-path. # The path separator used here should be the separator specified by "path_separator" # below. -# version_locations = %(here)s/bar:%(here)s/bat:${script_location}/versions +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions # path_separator; This indicates what character is used to split lists of file # paths, including version_locations and prepend_sys_path within configparser diff --git a/alembic/templates/generic/alembic.ini.mako b/alembic/templates/generic/alembic.ini.mako index 0a5a5754..af12a0db 100644 --- a/alembic/templates/generic/alembic.ini.mako +++ b/alembic/templates/generic/alembic.ini.mako @@ -1,8 +1,10 @@ # A generic, single database configuration. [alembic] -# path to migration scripts -# Use forward slashes (/) also on windows to provide an os agnostic path +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file script_location = ${script_location} # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s @@ -38,11 +40,11 @@ prepend_sys_path = . # sourceless = false # version location specification; This defaults -# to ${script_location}/versions. When using multiple version +# to /versions. When using multiple version # directories, initial revisions must be specified with --version-path. # The path separator used here should be the separator specified by "path_separator" # below. -# version_locations = %(here)s/bar:%(here)s/bat:${script_location}/versions +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions # path_separator; This indicates what character is used to split lists of file # paths, including version_locations and prepend_sys_path within configparser diff --git a/alembic/templates/multidb/alembic.ini.mako b/alembic/templates/multidb/alembic.ini.mako index a0cae1d5..699765c5 100644 --- a/alembic/templates/multidb/alembic.ini.mako +++ b/alembic/templates/multidb/alembic.ini.mako @@ -1,8 +1,10 @@ # a multi-database configuration. [alembic] -# path to migration scripts -# Use forward slashes (/) also on windows to provide an os agnostic path +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file script_location = ${script_location} # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s @@ -37,11 +39,11 @@ prepend_sys_path = . # sourceless = false # version location specification; This defaults -# to ${script_location}/versions. When using multiple version +# to /versions. When using multiple version # directories, initial revisions must be specified with --version-path. # The path separator used here should be the separator specified by "path_separator" # below. -# version_locations = %(here)s/bar:%(here)s/bat:${script_location}/versions +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions # path_separator; This indicates what character is used to split lists of file # paths, including version_locations and prepend_sys_path within configparser diff --git a/alembic/templates/pyproject/README b/alembic/templates/pyproject/README new file mode 100644 index 00000000..fdacc05f --- /dev/null +++ b/alembic/templates/pyproject/README @@ -0,0 +1 @@ +pyproject configuration, based on the generic configuration. \ No newline at end of file diff --git a/alembic/templates/pyproject/alembic.ini.mako b/alembic/templates/pyproject/alembic.ini.mako new file mode 100644 index 00000000..fae3c141 --- /dev/null +++ b/alembic/templates/pyproject/alembic.ini.mako @@ -0,0 +1,41 @@ +# A generic, single database configuration. + +[alembic] + +sqlalchemy.url = driver://user:pass@localhost/dbname + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/templates/pyproject/env.py b/alembic/templates/pyproject/env.py new file mode 100644 index 00000000..36112a3c --- /dev/null +++ b/alembic/templates/pyproject/env.py @@ -0,0 +1,78 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/templates/pyproject/pyproject.toml.mako b/alembic/templates/pyproject/pyproject.toml.mako new file mode 100644 index 00000000..cfc56ceb --- /dev/null +++ b/alembic/templates/pyproject/pyproject.toml.mako @@ -0,0 +1,76 @@ +[tool.alembic] + +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file +script_location = "${script_location}" + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = "%%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s" + +# additional paths to be prepended to sys.path. defaults to the current working directory. +prepend_sys_path = [ + "." +] + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library and tzdata library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to /versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# version_locations = [ +# "%(here)s/alembic/versions", +# "%(here)s/foo/bar" +# ] + + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = "utf-8" + +# This section defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples +# [[tool.alembic.post_write_hooks]] +# format using "black" - use the console_scripts runner, +# against the "black" entrypoint +# name = "black" +# type = "console_scripts" +# entrypoint = "black" +# options = "-l 79 REVISION_SCRIPT_FILENAME" +# +# [[tool.alembic.post_write_hooks]] +# lint with attempts to fix using "ruff" - use the exec runner, +# execute a binary +# name = "ruff" +# type = "exec" +# executable = "%(here)s/.venv/bin/ruff" +# options = "check --fix REVISION_SCRIPT_FILENAME" + diff --git a/alembic/templates/pyproject/script.py.mako b/alembic/templates/pyproject/script.py.mako new file mode 100644 index 00000000..480b130d --- /dev/null +++ b/alembic/templates/pyproject/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"} diff --git a/alembic/testing/__init__.py b/alembic/testing/__init__.py index 809de7e4..316ea91b 100644 --- a/alembic/testing/__init__.py +++ b/alembic/testing/__init__.py @@ -9,6 +9,7 @@ from sqlalchemy.testing import uses_deprecated from sqlalchemy.testing.config import combinations from sqlalchemy.testing.config import fixture from sqlalchemy.testing.config import requirements as requires +from sqlalchemy.testing.config import variation from .assertions import assert_raises from .assertions import assert_raises_message diff --git a/alembic/testing/env.py b/alembic/testing/env.py index a97990e6..72a5e424 100644 --- a/alembic/testing/env.py +++ b/alembic/testing/env.py @@ -188,6 +188,50 @@ datefmt = %%H:%%M:%%S ) +def _no_sql_pyproject_config(dialect="postgresql", directives=""): + """use a postgresql url with no host so that + connections guaranteed to fail""" + dir_ = _join_path(_get_staging_directory(), "scripts") + + return _write_toml_config( + f""" +[tool.alembic] +script_location ="{dir_}" +{textwrap.dedent(directives)} + + """, + f""" +[alembic] +sqlalchemy.url = {dialect}:// + +[loggers] +keys = root + +[handlers] +keys = console + +[logger_root] +level = WARNING +handlers = console +qualname = + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatters] +keys = generic + +[formatter_generic] +format = %%(levelname)-5.5s [%%(name)s] %%(message)s +datefmt = %%H:%%M:%%S + +""", + ) + + def _no_sql_testing_config(dialect="postgresql", directives=""): """use a postgresql url with no host so that connections guaranteed to fail""" @@ -227,6 +271,13 @@ datefmt = %%H:%%M:%%S ) +def _write_toml_config(tomltext, initext): + cfg = _write_config_file(initext) + with open(cfg.toml_file_name, "w") as f: + f.write(tomltext) + return cfg + + def _write_config_file(text): cfg = _testing_config() with open(cfg.config_file_name, "w") as f: @@ -239,7 +290,10 @@ def _testing_config(): if not os.access(_get_staging_directory(), os.F_OK): os.mkdir(_get_staging_directory()) - return Config(_join_path(_get_staging_directory(), "test_alembic.ini")) + return Config( + _join_path(_get_staging_directory(), "test_alembic.ini"), + _join_path(_get_staging_directory(), "pyproject.toml"), + ) def write_script( diff --git a/alembic/testing/fixtures.py b/alembic/testing/fixtures.py index 17d732f8..61bcd7e9 100644 --- a/alembic/testing/fixtures.py +++ b/alembic/testing/fixtures.py @@ -3,7 +3,9 @@ from __future__ import annotations import configparser from contextlib import contextmanager import io +import os import re +import shutil from typing import Any from typing import Dict @@ -24,13 +26,13 @@ from sqlalchemy.testing.fixtures import TestBase as SQLAlchemyTestBase import alembic from .assertions import _get_dialect +from .env import _get_staging_directory from ..environment import EnvironmentContext from ..migration import MigrationContext from ..operations import Operations from ..util import sqla_compat from ..util.sqla_compat import sqla_2 - testing_config = configparser.ConfigParser() testing_config.read(["test.cfg"]) @@ -38,6 +40,31 @@ testing_config.read(["test.cfg"]) class TestBase(SQLAlchemyTestBase): is_sqlalchemy_future = sqla_2 + @testing.fixture() + def clear_staging_dir(self): + yield + location = _get_staging_directory() + for filename in os.listdir(location): + file_path = os.path.join(location, filename) + if os.path.isfile(file_path) or os.path.islink(file_path): + os.unlink(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + + @contextmanager + def pushd(self, dirname): + current_dir = os.getcwd() + try: + os.chdir(dirname) + yield + finally: + os.chdir(current_dir) + + @testing.fixture() + def pop_alembic_config_env(self): + yield + os.environ.pop("ALEMBIC_CONFIG", None) + @testing.fixture() def ops_context(self, migration_context): with migration_context.begin_transaction(_per_migration=True): diff --git a/alembic/util/__init__.py b/alembic/util/__init__.py index 1d3a2179..99555d8e 100644 --- a/alembic/util/__init__.py +++ b/alembic/util/__init__.py @@ -25,5 +25,6 @@ from .messaging import write_outstream as write_outstream from .pyfiles import coerce_resource_to_filename as coerce_resource_to_filename from .pyfiles import load_python_file as load_python_file from .pyfiles import pyc_file_from_path as pyc_file_from_path +from .pyfiles import relpath_via_abs_root as relpath_via_abs_root from .pyfiles import template_to_file as template_to_file from .sqla_compat import sqla_2 as sqla_2 diff --git a/alembic/util/compat.py b/alembic/util/compat.py index 15d49cac..670e43aa 100644 --- a/alembic/util/compat.py +++ b/alembic/util/compat.py @@ -52,6 +52,11 @@ else: import importlib_metadata # type:ignore # noqa from importlib_metadata import EntryPoint # type:ignore # noqa +if py311: + import tomllib as tomllib +else: + import tomli as tomllib # type: ignore # noqa + def importlib_metadata_get(group: str) -> Sequence[EntryPoint]: ep = importlib_metadata.entry_points() diff --git a/alembic/util/pyfiles.py b/alembic/util/pyfiles.py index 973bd458..c97f7ef9 100644 --- a/alembic/util/pyfiles.py +++ b/alembic/util/pyfiles.py @@ -20,7 +20,12 @@ from .exc import CommandError def template_to_file( - template_file: str, dest: str, output_encoding: str, **kw: Any + template_file: str, + dest: str, + output_encoding: str, + *, + append: bool = False, + **kw: Any, ) -> None: template = Template(filename=template_file) try: @@ -38,7 +43,7 @@ def template_to_file( "template-oriented traceback." % fname ) else: - with open(dest, "wb") as f: + with open(dest, "ab" if append else "wb") as f: f.write(output) @@ -112,3 +117,9 @@ def load_module_py(module_id: str, path: str) -> ModuleType: module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) # type: ignore return module + + +def relpath_via_abs_root(root: str, relative_path: str) -> str: + abs_root = os.path.abspath(root) + abs_path = os.path.abspath(relative_path) + return abs_path[len(os.path.commonpath([abs_root, abs_path])) + 1 :] diff --git a/docs/build/autogenerate.rst b/docs/build/autogenerate.rst index ad88d475..aca8683b 100644 --- a/docs/build/autogenerate.rst +++ b/docs/build/autogenerate.rst @@ -692,18 +692,29 @@ regardless of whether or not the autogenerate feature was used. Basic Post Processor Configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The ``alembic.ini`` samples now include commented-out configuration +The template samples for ``alembic.ini`` as well as ``pyproject.toml`` for +applicable templates now include commented-out configuration illustrating how to configure code-formatting tools, or other tools like linters -to run against the newly generated file path. Example:: +to run against the newly generated file path. Example from an alembic.ini file:: - [post_write_hooks] + [post_write_hooks] + + # format using "black" + hooks=black + + black.type = console_scripts + black.entrypoint = black + black.options = -l 79 REVISION_SCRIPT_FILENAME - # format using "black" - hooks=black +The same example configured in a pyproject.toml file would look like:: - black.type = console_scripts - black.entrypoint = black - black.options = -l 79 + [[tool.alembic.post_write_hooks]] + + # format using "black" + name = "black" + type = "console_scripts" + entrypoint = "black" + options = "-l 79 REVISION_SCRIPT_FILENAME" Above, we configure ``hooks`` to be a single post write hook labeled ``"black"``. Note that this label is arbitrary. We then define the @@ -768,21 +779,37 @@ tool's output as well:: Hooks may also be specified as a list of names, which correspond to hook runners that will run sequentially. As an example, we can also run the `zimports `_ import rewriting tool (written -by Alembic's author) subsequent to running the ``black`` tool, using a -configuration as follows:: +by Alembic's author) subsequent to running the ``black`` tool. The +alembic.ini configuration would be as follows:: - [post_write_hooks] + [post_write_hooks] + + # format using "black", then "zimports" + hooks=black, zimports + + black.type = console_scripts + black.entrypoint = black + black.options = -l 79 REVISION_SCRIPT_FILENAME + + zimports.type = console_scripts + zimports.entrypoint = zimports + zimports.options = --style google REVISION_SCRIPT_FILENAME + +The equivalent pyproject.toml configuration would be:: - # format using "black", then "zimports" - hooks=black, zimports + # format using "black", then "zimports" - black.type = console_scripts - black.entrypoint = black - black.options = -l 79 REVISION_SCRIPT_FILENAME + [[tool.alembic.post_write_hooks]] + name = "black" + type="console_scripts" + entrypoint = "black" + options = "-l 79 REVISION_SCRIPT_FILENAME" - zimports.type = console_scripts - zimports.entrypoint = zimports - zimports.options = --style google REVISION_SCRIPT_FILENAME + [[tool.alembic.post_write_hooks]] + name = "zimports" + type="console_scripts" + entrypoint = "zimports" + options = "--style google REVISION_SCRIPT_FILENAME" When using the above configuration, a newly generated revision file will be processed first by the "black" tool, then by the "zimports" tool. diff --git a/docs/build/cookbook.rst b/docs/build/cookbook.rst index e52753a8..83c838ce 100644 --- a/docs/build/cookbook.rst +++ b/docs/build/cookbook.rst @@ -1213,7 +1213,10 @@ projects had a need to maintain more than one Alembic version history in a singl project, where these version histories are completely independent of each other and each refer to their own alembic_version table, either across multiple databases, schemas, or namespaces. A simple approach was added to support this, the -``--name`` flag on the commandline. +``--name`` flag on the commandline. This flag allows named sections within +the ``alembic.ini`` file to be present (but note it does **not apply** to +``pyproject.toml`` configuration, where only the ``[tool.alembic]`` section +is used). First, one would create an alembic.ini file of this form:: diff --git a/docs/build/tutorial.rst b/docs/build/tutorial.rst index 74b5252a..507f3e8e 100644 --- a/docs/build/tutorial.rst +++ b/docs/build/tutorial.rst @@ -32,6 +32,8 @@ needs of the application. The structure of this environment, including some generated migration scripts, looks like:: yourproject/ + alembic.ini + pyproject.toml alembic/ env.py README @@ -43,6 +45,11 @@ The structure of this environment, including some generated migration scripts, l The directory includes these directories/files: +* ``alembic.ini`` - this is Alembic's main configuration file which is genereated by all templates. + A detailed walkthrough of this file is later in the section :ref:`tutorial_alembic_ini`. +* ``pyproject.toml`` - most modern Python projects have a ``pyproject.toml`` file. Alembic may + optionally store project related configuration in this file as well; to use a ``pyproject.toml`` + configuration, see the section :ref:`using_pep_621`. * ``yourproject`` - this is the root of your application's source code, or some directory within it. * ``alembic`` - this directory lives within your application's source tree and is the home of the migration environment. It can be named anything, and a project that uses multiple databases @@ -106,6 +113,7 @@ command:: Available templates: generic - Generic single-database configuration. + pyproject - pep-621 compliant configuration that includes pyproject.toml async - Generic single-database configuration with an async dbapi. multidb - Rudimentary multi-database configuration. @@ -115,6 +123,12 @@ command:: .. versionchanged:: 1.8 The "pylons" environment template has been removed. +.. versionchanged:: 1.16.0 A new ``pyproject`` template has been added. See + the section :re3f:`using_pep_621` for background. + +` +.. _tutorial_alembic_ini: + Editing the .ini File ===================== @@ -128,8 +142,11 @@ The file generated with the "generic" configuration looks like:: # A generic, single database configuration. [alembic] - # path to migration scripts - script_location = alembic + # path to migration scripts. + # this is typically a path given in POSIX (e.g. forward slashes) + # format, relative to the token %(here)s which refers to the location of this + # ini file + script_location = %(here)s/alembic # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s # Uncomment the line below if you want the files to be prepended with date and time @@ -161,10 +178,13 @@ The file generated with the "generic" configuration looks like:: # sourceless = false # version location specification; This defaults - # to ${script_location}/versions. When using multiple version + # to /versions. When using multiple version # directories, initial revisions must be specified with --version-path. + # the special token `%(here)s` is available which indicates the absolute path + # to this configuration file. + # # The path separator used here should be the separator specified by "version_path_separator" below. - # version_locations = %(here)s/bar:%(here)s/bat:${script_location}/versions + # version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions # path_separator (New in Alembic 1.16.0, supersedes version_path_separator); # This indicates what character is used to @@ -269,6 +289,7 @@ This file contains the following features: core implementation does not directly read any other areas of the file, not including additional directives that may be consumed from the end-user-customizable ``env.py`` file (see note below). The name "alembic" + (for configparser config only, not ``pyproject.toml``) can be customized using the ``--name`` commandline flag; see :ref:`multiple_environments` for a basic example of this. @@ -280,8 +301,10 @@ This file contains the following features: logging. * ``script_location`` - this is the location of the Alembic environment. It is normally - specified as a filesystem location, either relative or absolute. If the location is - a relative path, it's interpreted as relative to the current directory. + specified as a filesystem location relative to the ``%(here)s`` token, which + indicates where the config file itself is located. The location may also + be a plain relative path, where it's interpreted as relative to the current directory, + or an absolute path. This is the only key required by Alembic in all cases. The generation of the .ini file by the command ``alembic init alembic`` automatically placed the @@ -359,7 +382,9 @@ This file contains the following features: See :ref:`multiple_bases` for examples. * ``path_separator`` - a separator character for the ``version_locations`` - and ``prepend_sys_path`` path lists. See :ref:`multiple_bases` for examples. + and ``prepend_sys_path`` path lists. Only applies to configparser config, + not needed if ``pyproject.toml`` configuration is used. + See :ref:`multiple_bases` for examples. * ``recursive_version_locations`` - when set to 'true', revision files are searched recursively in each "version_locations" directory. @@ -381,6 +406,173 @@ the SQLAlchemy URL is all that's needed:: sqlalchemy.url = postgresql://scott:tiger@localhost/test +.. _using_pep_621: + +Using pyproject.toml for configuration +====================================== + +.. versionadded:: 1.16.0 + +As the ``alembic.ini`` file includes a subset of options that are specific to +the organization and production of Python code within the local environment, +these specific options may alternatively be placed in the application's +``pyproject.toml`` file, to allow for :pep:`621` compliant configuration. + +Use of ``pyproject.toml`` does not preclude having an ``alembic.ini`` file as +well, as ``alembic.ini`` is still the default location for **deployment** +details such as database URLs, connectivity options, and logging to be present. +However, as connectivity and logging is consumed only by user-managed code +within the ``env.py`` file, it is feasible that the environment would not +require the ``alembic.ini`` file itself to be present at all, if these +configurational elements are consumed from other places elsewhere in the +application. + +To start with a pyproject configuration, the most straightforward approach is +to use the ``pyproject`` template:: + + alembic init --template pyproject alembic + +The output states that the existing pyproject file is being augmented with +additional directives:: + + Creating directory /path/to/yourproject/alembic...done + Creating directory /path/to/yourproject/alembic/versions...done + Appending to /path/to/yourproject/pyproject.toml...done + Generating /path/to/yourproject/alembic.ini...done + Generating /path/to/yourproject/alembic/env.py...done + Generating /path/to/yourproject/alembic/README...done + Generating /path/to/yourproject/alembic/script.py.mako...done + Please edit configuration/connection/logging settings in + '/path/to/yourproject/pyproject.toml' and + '/path/to/yourproject/alembic.ini' before proceeding. + +Alembic's template runner will generate a new ``pyproject.toml`` file if +one does not exist, or it will append directives to an existing ``pyproject.toml`` +file that does not already include alembic directives. + +Within the ``pyproject.toml`` file, the default section generated looks mostly +like the ``alembic.ini`` file, with the welcome exception that lists of values +are supported directly; this means the values ``prepend_sys_path`` and +``version_locations`` are specified as lists. The ``%(here)s`` token also +remains available as the absolute path to the ``pyproject.toml`` file:: + + [tool.alembic] + # path to migration scripts + script_location = "%(here)s/alembic" + + # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s + # Uncomment the line below if you want the files to be prepended with date and time + # file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + + # additional paths to be prepended to sys.path. defaults to the current working directory. + prepend_sys_path = [ + "." + ] + + # timezone to use when rendering the date within the migration file + # as well as the filename. + # If specified, requires the python>=3.9 or backports.zoneinfo library and tzdata library. + # Any required deps can installed by adding `alembic[tz]` to the pip requirements + # string value is passed to ZoneInfo() + # leave blank for localtime + # timezone = + + # max length of characters to apply to the + # "slug" field + # truncate_slug_length = 40 + + # set to 'true' to run the environment during + # the 'revision' command, regardless of autogenerate + # revision_environment = false + + # set to 'true' to allow .pyc and .pyo files without + # a source .py file to be detected as revisions in the + # versions/ directory + # sourceless = false + + # version location specification; This defaults + # to /versions. When using multiple version + # directories, initial revisions must be specified with --version-path. + # version_locations = [ + # "%(here)s/alembic/versions", + # "%(here)s/foo/bar" + # ] + + # set to 'true' to search source files recursively + # in each "version_locations" directory + # new in Alembic version 1.10 + # recursive_version_locations = false + + # the output encoding used when revision files + # are written from script.py.mako + # output_encoding = "utf-8" + + + # This section defines scripts or Python functions that are run + # on newly generated revision scripts. See the documentation for further + # detail and examples + # [[tool.alembic.post_write_hooks]] + # format using "black" - use the console_scripts runner, + # against the "black" entrypoint + # name = "black" + # type = "console_scripts" + # entrypoint = "black" + # options = "-l 79 REVISION_SCRIPT_FILENAME" + # + # [[tool.alembic.post_write_hooks]] + # lint with attempts to fix using "ruff" - use the exec runner, + # execute a binary + # name = "ruff" + # type = "exec" + # executable = "%(here)s/.venv/bin/ruff" + # options = "check --fix REVISION_SCRIPT_FILENAME" + + +The ``alembic.ini`` file for this template is truncated and contains +only database configuration and logging configuration:: + + [alembic] + + sqlalchemy.url = driver://user:pass@localhost/dbname + + # Logging configuration + [loggers] + keys = root,sqlalchemy,alembic + + [handlers] + keys = console + + [formatters] + keys = generic + + [logger_root] + level = WARNING + handlers = console + qualname = + + [logger_sqlalchemy] + level = WARNING + handlers = + qualname = sqlalchemy.engine + + [logger_alembic] + level = INFO + handlers = + qualname = alembic + + [handler_console] + class = StreamHandler + args = (sys.stderr,) + level = NOTSET + formatter = generic + + [formatter_generic] + format = %(levelname)-5.5s [%(name)s] %(message)s + datefmt = %H:%M:%S + +When ``env.py`` is configured to obtain database connectivity and logging +configuration from places other than ``alembic.ini``, the file can be +omitted altogether. .. _create_migration: diff --git a/docs/build/unreleased/1082.rst b/docs/build/unreleased/1082.rst new file mode 100644 index 00000000..8b58c5e8 --- /dev/null +++ b/docs/build/unreleased/1082.rst @@ -0,0 +1,21 @@ +.. change:: + :tags: feature, environment + :tickets: 1082 + + Added optional :pep:`621` support to Alembic, where a subset of the + project-centric configuration normally found in the ``alembic.ini`` file + can now be retrieved from the project-wide ``pyproject.toml`` file. A new + init template ``pyproject`` is added which illustrates a basic :pep:`621` + setup. The :pep:`621` feature supports configuration values that are + relevant to code locations and code production only; it does not + accommodate database connectivity, configuration, or logging configuration. + These latter configurational elements remain as elements that can be + present either in the ``alembic.ini`` file, or retrieved elsewhere within + the ``env.py`` file. The change also allows the ``alembic.ini`` file to + be completely optional if the ``pyproject.toml`` file contains a base + alembic configuration section. + + + .. seealso:: + + :ref:`using_pep_621` \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 389f95c9..3f94dbb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "SQLAlchemy>=1.4.0", "Mako", "typing-extensions>=4.12", + "tomli;python_version<'3.11'", ] dynamic = ["version"] @@ -111,3 +112,4 @@ module = [ 'sqlalchemy.testing.*' ] ignore_missing_imports = true + diff --git a/tests/test_command.py b/tests/test_command.py index 89398c24..3602eb25 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -1,3 +1,4 @@ +from configparser import RawConfigParser from contextlib import contextmanager import inspect from io import BytesIO @@ -22,6 +23,7 @@ from alembic.script import ScriptDirectory from alembic.testing import assert_raises from alembic.testing import assert_raises_message from alembic.testing import eq_ +from alembic.testing import expect_raises_message from alembic.testing import is_false from alembic.testing import is_true from alembic.testing import mock @@ -1187,6 +1189,106 @@ class CommandLineTest(TestBase): config.command.revision = orig_revision eq_(canary.mock_calls, [mock.call(self.cfg, message="foo")]) + def test_config_file_failure_modes(self): + """with two config files supported at the same time, test failure + modes with multiple --config directives + + """ + c1 = config.CommandLine() + + with expect_raises_message( + util.CommandError, "only one ini file may be indicated" + ): + c1.main( + argv=[ + "--config", + "inione", + "--config", + "initwo.ini", + "history", + ] + ) + + with expect_raises_message( + util.CommandError, "pyproject.toml indicated more than once" + ): + c1.main( + argv=[ + "--config", + "pyproject.toml", + "--config", + "a/b/pyproject.toml", + "history", + ] + ) + + @testing.combinations( + ( + {"ALEMBIC_CONFIG": "some/pyproject.toml", "argv": []}, + "some/pyproject.toml", + "alembic.ini", + ), + ( + {"ALEMBIC_CONFIG": "some/path_to_alembic.ini", "argv": []}, + "pyproject.toml", + "some/path_to_alembic.ini", + ), + ( + { + "ALEMBIC_CONFIG": "some/path_to_alembic.ini", + "argv": [ + "--config", + "foo/pyproject.toml", + "--config", + "bar/alembic.ini", + ], + }, + "foo/pyproject.toml", + "bar/alembic.ini", + ), + ( + { + "argv": [ + "--config", + "foo/pyproject.toml", + "--config", + "bar/alembic.ini", + ], + }, + "foo/pyproject.toml", + "bar/alembic.ini", + ), + ( + {"argv": []}, + "pyproject.toml", + "alembic.ini", + ), + ( + {"argv": ["--config", "foo/pyproject.toml"]}, + "foo/pyproject.toml", + "alembic.ini", + ), + ( + {"argv": ["--config", "foo/some_alembic.ini"]}, + "pyproject.toml", + "foo/some_alembic.ini", + ), + argnames=("args, expected_toml, expected_conf"), + ) + def test_config_file_resolution( + self, args, expected_toml, expected_conf, pop_alembic_config_env + ): + """with two config files supported at the same time, test resolution + of --config / ALEMBIC_CONFIG to always "do what's expected" + + """ + c1 = config.CommandLine() + if "ALEMBIC_CONFIG" in args: + os.environ["ALEMBIC_CONFIG"] = args["ALEMBIC_CONFIG"] + + options = c1.parser.parse_args(args["argv"]) + eq_(c1._inis_from_config(options), (expected_toml, expected_conf)) + def test_help_text(self): commands = { fn.__name__ @@ -1276,6 +1378,77 @@ class CommandLineTest(TestBase): cfg = run_cmd.mock_calls[0][1][0] eq_(cfg.config_file_name, "myconf.conf") + @testing.combinations( + ( + "pyproject", + "somepath/foobar", + "pyproject.toml", + "alembic.ini", + "%(here)s/somepath/foobar", + None, + ), + ( + "pyproject", + "somepath/foobar", + "somepath/pyproject.toml", + "alembic.ini", + "%(here)s/foobar", + None, + ), + ( + "generic", + "somepath/foobar", + "pyproject.toml", + "alembic.ini", + None, + "%(here)s/somepath/foobar", + ), + ( + "generic", + "somepath/foobar", + "pyproject.toml", + "somepath/alembic.ini", + None, + "%(here)s/foobar", + ), + argnames="template,directory,toml_file_name,config_file_name," + "expected_toml_location,expected_ini_location", + ) + def test_init_file_relative_version_token( + self, + template, + directory, + toml_file_name, + config_file_name, + expected_toml_location, + expected_ini_location, + clear_staging_dir, + ): + """in 1.16.0 with the advent of pyproject.toml, we are also rendering + the script_location value relative to the ``%(here)s`` token, if + the given path is a relative path. ``%(here)s`` is relative to the + owning config file either alembic.ini or pyproject.toml. + + """ + self.cfg.config_file_name = config_file_name + self.cfg.toml_file_name = toml_file_name + with self.pushd(os.path.join(_get_staging_directory())): + command.init(self.cfg, directory=directory, template=template) + if expected_toml_location is not None: + with open(self.cfg.toml_file_name, "rb") as f: + toml = util.compat.tomllib.load(f) + eq_( + toml["tool"]["alembic"]["script_location"], + expected_toml_location, + ) + + cfg = RawConfigParser() + util.compat.read_config_parser(cfg, config_file_name) + eq_( + cfg.get("alembic", "script_location", fallback=None), + expected_ini_location, + ) + def test_init_file_exists_and_is_empty(self): def access_(path, mode): if "generic" in path or path == "foobar": diff --git a/tests/test_post_write.py b/tests/test_post_write.py index 9f019ca3..a63ccd9f 100644 --- a/tests/test_post_write.py +++ b/tests/test_post_write.py @@ -2,6 +2,7 @@ import os import sys from alembic import command +from alembic import testing from alembic import util from alembic.script import write_hooks from alembic.testing import assert_raises_message @@ -10,6 +11,7 @@ from alembic.testing import eq_ from alembic.testing import mock from alembic.testing import TestBase from alembic.testing.env import _get_staging_directory +from alembic.testing.env import _no_sql_pyproject_config from alembic.testing.env import _no_sql_testing_config from alembic.testing.env import clear_staging_env from alembic.testing.env import staging_env @@ -40,24 +42,40 @@ class RunHookTest(TestBase): def tearDown(self): clear_staging_env() - def test_generic(self): + @testing.variation("config", ["ini", "toml"]) + def test_generic(self, config): hook1 = mock.Mock() hook2 = mock.Mock() write_hooks.register("hook1")(hook1) write_hooks.register("hook2")(hook2) - self.cfg = _no_sql_testing_config( - directives=( - "\n[post_write_hooks]\n" - "hooks=hook1,hook2\n" - "hook1.type=hook1\n" - "hook1.arg1=foo\n" - "hook2.type=hook2\n" - "hook2.arg1=bar\n" + if config.ini: + self.cfg = _no_sql_testing_config( + directives=( + "\n[post_write_hooks]\n" + "hooks=hook1,hook2\n" + "hook1.type=hook1\n" + "hook1.arg1=foo\n" + "hook2.type=hook2\n" + "hook2.arg1=bar\n" + ) + ) + else: + self.cfg = _no_sql_pyproject_config( + directives=""" + + [[tool.alembic.post_write_hooks]] + name="hook1" + type="hook1" + arg1="foo" + + [[tool.alembic.post_write_hooks]] + name="hook2" + type="hook2" + arg1="bar" + """ ) - ) - rev = command.revision(self.cfg, message="x") eq_( @@ -83,11 +101,18 @@ class RunHookTest(TestBase): self.cfg = _no_sql_testing_config( directives=("\n[post_write_hooks]\n") ) - command.revision(self.cfg, message="x") - def test_no_section(self): - self.cfg = _no_sql_testing_config(directives="") + @testing.variation("config", ["ini", "toml"]) + def test_no_section(self, config): + if config.ini: + self.cfg = _no_sql_testing_config(directives="") + else: + self.cfg = _no_sql_pyproject_config( + directives=""" + + """ + ) command.revision(self.cfg, message="x") @@ -98,16 +123,28 @@ class RunHookTest(TestBase): command.revision(self.cfg, message="x") - def test_no_type(self): - self.cfg = _no_sql_testing_config( - directives=( - "\n[post_write_hooks]\n" "hooks=foo\n" "foo.bar=somebar\n" + @testing.variation("config", ["ini", "toml"]) + def test_no_type(self, config): + if config.ini: + self.cfg = _no_sql_testing_config( + directives=( + "\n[post_write_hooks]\n" "hooks=foo\n" "foo.bar=somebar\n" + ) + ) + else: + self.cfg = _no_sql_pyproject_config( + directives=""" + + [[tool.alembic.post_write_hooks]] + name="foo" + bar="somebar" + """ ) - ) assert_raises_message( util.CommandError, - "Key foo.type is required for post write hook 'foo'", + r"Key 'foo.type' \(or 'type' in toml\) is required " + "for post write hook 'foo'", command.revision, self.cfg, message="x", @@ -130,9 +167,16 @@ class RunHookTest(TestBase): ) def _run_black_with_config( - self, input_config, expected_additional_arguments_fn, cwd=None + self, + input_config, + expected_additional_arguments_fn, + cwd=None, + use_toml=False, ): - self.cfg = _no_sql_testing_config(directives=input_config) + if use_toml: + self.cfg = _no_sql_pyproject_config(directives=input_config) + else: + self.cfg = _no_sql_testing_config(directives=input_config) retVal = [ compat.EntryPoint( @@ -175,25 +219,39 @@ class RunHookTest(TestBase): ], ) - def test_console_scripts(self): - input_config = """ + @testing.variation("config", ["ini", "toml"]) + def test_console_scripts(self, config): + if config.ini: + input_config = """ [post_write_hooks] hooks = black black.type = console_scripts black.entrypoint = black black.options = -l 79 - """ + """ + else: + input_config = """ + [[tool.alembic.post_write_hooks]] + name = "black" + type = "console_scripts" + entrypoint = "black" + options = "-l 79" + """ def expected_additional_arguments_fn(rev_path): return [rev_path, "-l", "79"] self._run_black_with_config( - input_config, expected_additional_arguments_fn + input_config, + expected_additional_arguments_fn, + use_toml=config.toml, ) - @combinations(True, False) - def test_filename_interpolation(self, posix): - input_config = """ + @combinations(True, False, argnames="posix") + @testing.variation("config", ["ini", "toml"]) + def test_filename_interpolation(self, posix, config): + if config.ini: + input_config = """ [post_write_hooks] hooks = black black.type = console_scripts @@ -201,6 +259,14 @@ black.entrypoint = black black.options = arg1 REVISION_SCRIPT_FILENAME 'multi-word arg' \ --flag1='REVISION_SCRIPT_FILENAME' """ + else: + input_config = """ +[[tool.alembic.post_write_hooks]] +name = "black" +type = "console_scripts" +entrypoint = "black" +options = "arg1 REVISION_SCRIPT_FILENAME 'multi-word arg' --flag1='REVISION_SCRIPT_FILENAME'" +""" # noqa: E501 def expected_additional_arguments_fn(rev_path): if compat.is_posix: @@ -220,7 +286,9 @@ black.options = arg1 REVISION_SCRIPT_FILENAME 'multi-word arg' \ with mock.patch("alembic.util.compat.is_posix", posix): self._run_black_with_config( - input_config, expected_additional_arguments_fn + input_config, + expected_additional_arguments_fn, + use_toml=config.toml, ) def test_path_in_config(self):