From: Daan De Meyer Date: Tue, 8 Aug 2023 13:57:14 +0000 (+0200) Subject: Introduce preset dependencies X-Git-Tag: v15~7 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d24a4b228659908056eb6a191c8c858c837e62e5;p=thirdparty%2Fmkosi.git Introduce preset dependencies Instead of building presets in alphanumerical order, let's introduce a proper concept of dependencies. Dependencies are configured with the new "Dependencies=" option in the new "[Preset]" section. All presets configured with "Dependencies=" are built before the current preset. We drop the alphanumerical preset in favor of explicit dependencies. --- diff --git a/mkosi.md b/mkosi.md index 39ba9163d..1ebe4ed59 100644 --- a/mkosi.md +++ b/mkosi.md @@ -190,9 +190,10 @@ Those settings cannot be configured in the configuration files. `--preset=` -: If specified, only build the given preset. Can be specified multiple - times to build multiple presets. If not specified, all presets are - built. See the `Presets` section for more information. +: If specified, only build the given presets. Can be specified multiple + times to build multiple presets. All the given presets and their + dependencies are built. If not specified, all presets are built. See + the `Presets` section for more information. ## Supported output formats @@ -300,6 +301,15 @@ they should be specified with a boolean argument: either "1", "yes", or "true" t | `ImageVersion=` | no | yes | match fails | | `Bootable=` | no | no | match auto feature | +### [Preset] Section + +`Dependencies=`, `--dependency=` + +: The presets that this preset depends on specified as a comma-separated + list. All presets configured in this option will be built before this + preset and will be pulled in as dependencies of this preset when + `--preset` is used. + ### [Distribution] Section `Distribution=`, `--distribution=`, `-d` @@ -1337,22 +1347,18 @@ directories containing mkosi configuration files or regular files with the `.conf` extension. When presets are found in `mkosi.presets/`, mkosi will build the -configured presets (or all of them if none were explicitly configured -using `--preset=`) in alphanumerical order. To enforce a certain build -order, preset names can be numerically prefixed (e.g. `00-initrd.conf`). -The numerical prefix will be removed from the preset name during parsing, -along with the `.conf` suffix (`00-initrd.conf` becomes `initrd`). The -preset name is used for display purposes and also as the default output -name if none is explicitly configured. +configured preset and its dependencies (or all of them if no presets +were explicitly configured using `--preset=`). To add dependencies +between presets, the `Dependencies=` setting can be used. When presets are defined, mkosi will first read the global configuration (configuration outside of the `mkosi.presets/` directory), followed by the preset specific configuration. This means that global configuration takes precedence over preset specific configuration. -Later presets can refer to outputs of earlier presets. Specifically, for -the following options, mkosi will only check whether the inputs exist -just before building the preset: +Presets can refer to outputs of presets they depend on. Specifically, +for the following options, mkosi will only check whether the inputs +exist just before building the preset: - `BaseTrees=` - `PackageManagerTrees=` @@ -1361,9 +1367,9 @@ just before building the preset: - `ToolsTree=` - `Initrds=` -To refer to outputs of earlier presets, simply configure any of these -options with a relative path to the location of the output to use in the -earlier preset's output directory. +To refer to outputs of a preset's dependencies, simply configure any of +these options with a relative path to the output to use in the output +directory of the dependency. A good example on how to use presets can be found in the systemd repository: https://github.com/systemd/systemd/tree/main/mkosi.presets. diff --git a/mkosi/config.py b/mkosi/config.py index 808f97ee0..50de80fec 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -8,13 +8,13 @@ import dataclasses import enum import fnmatch import functools +import graphlib import inspect import operator import os.path import platform import shlex import shutil -import string import subprocess import sys import textwrap @@ -628,6 +628,7 @@ class MkosiConfig: access the value from state. """ + dependencies: tuple[str] distribution: Distribution release: str mirror: Optional[str] @@ -798,6 +799,13 @@ class MkosiConfig: class MkosiConfigParser: SETTINGS = ( + MkosiConfigSetting( + dest="dependencies", + long="--dependency", + section="Preset", + parse=config_make_list_parser(delimiter=","), + help="Specify other presets that this preset depends on", + ), MkosiConfigSetting( dest="distribution", short="-d", @@ -1703,8 +1711,9 @@ class MkosiConfigParser: "--preset", action="append", dest="presets", + metavar="PRESET", default=[], - help="Build the specified preset", + help="Build the specified presets and their dependencies", ) parser.add_argument( "--nspawn-keep-unit", @@ -1767,6 +1776,32 @@ class MkosiConfigParser: delattr(namespace, "cache") print("Warning: --cache is no longer supported") + def resolve_deps(self, args: MkosiArgs, presets: Sequence[MkosiConfig]) -> list[MkosiConfig]: + graph = {p.preset: p.dependencies for p in presets} + + if args.presets: + if any((missing := p) not in graph for p in args.presets): + die(f"No preset found with name {missing}") + + deps = set() + queue = [*args.presets] + + while queue: + if (preset := queue.pop(0)) not in deps: + deps.add(preset) + queue.extend(graph[preset]) + + presets = [p for p in presets if p.preset in deps] + + graph = {p.preset: p.dependencies for p in presets} + + try: + order = list(graphlib.TopologicalSorter(graph).static_order()) + except graphlib.CycleError as e: + die(f"Preset dependency cycle detected: {' => '.join(e.args[1])}") + + return sorted(presets, key=lambda p: order.index(p.preset)) + def parse(self, argv: Optional[Sequence[str]] = None) -> tuple[MkosiArgs, tuple[MkosiConfig, ...]]: presets = [] namespace = argparse.Namespace() @@ -1808,15 +1843,13 @@ class MkosiConfigParser: self.parse_config(Path("."), namespace) if Path("mkosi.presets").exists(): - for p in sorted(Path("mkosi.presets").iterdir()): + for p in Path("mkosi.presets").iterdir(): if not p.is_dir() and not p.suffix == ".conf": continue - name = p.name.lstrip(string.digits + "-").removesuffix(".conf") + name = p.name.removesuffix(".conf") if not name: die(f"{p} is not a valid preset name") - if args.presets and name not in args.presets: - continue cp = copy.deepcopy(namespace) @@ -1853,7 +1886,10 @@ class MkosiConfigParser: # infrastructure scripts rather than image-specific configuration. self.backward_compat_stubs(namespace) - return args, tuple(load_config(ns) for ns in presets) + presets = [load_config(ns) for ns in presets] + presets = self.resolve_deps(args, presets) + + return args, tuple(presets) def load_credentials(args: argparse.Namespace) -> dict[str, str]: @@ -2090,6 +2126,9 @@ def summary(args: MkosiArgs, config: MkosiConfig) -> str: Verb: {bold(args.verb)} Cmdline: {bold(" ".join(args.cmdline))} + {bold("PRESET")}: + Dependencies: {line_join_list(config.dependencies)} + {bold("DISTRIBUTION")}: Distribution: {bold(config.distribution)} Release: {bold(none_to_na(config.release))}