From: Daan De Meyer Date: Sat, 28 Oct 2023 10:42:58 +0000 (+0200) Subject: Add BuildSources= match X-Git-Tag: v19~44^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F2018%2Fhead;p=thirdparty%2Fmkosi.git Add BuildSources= match One pattern I've started using a lot when I have to build multiple projects from source in a build script is to check if a source tree has been mounted at some location using BuildSources= and to only build the project if that's the case. The problem with this is that this only allows me to skip the build, it doesn't allow me to skip installing the necessary build and runtime packages for that particular project. Let's add a BuildSources= match so that everything related to the project can be skipped if the project is not configured to be mounted at some location using BuildSources=. --- diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 49f677fb2..0a6c0357c 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -313,9 +313,9 @@ def mount_volatile_overlay(state: MkosiState) -> Iterator[Path]: def finalize_mounts(config: MkosiConfig) -> list[PathString]: sources = [ - (src, Path.cwd() / (str(target).lstrip("/") if target else ".")) + (src, Path.cwd() / target) for src, target - in ((Path.cwd(), None), *config.build_sources) + in ((Path.cwd(), Path(".")), *config.build_sources) ] # bwrap() mounts /home and /var read-only during execution. So let's add the bind mount options for the diff --git a/mkosi/config.py b/mkosi/config.py index 72a781c08..d14fc242c 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -183,21 +183,24 @@ def parse_path(value: str, return path -def make_source_target_paths_parser(absolute: bool = True) -> Callable[[str], tuple[Path, Optional[Path]]]: - def parse_source_target_paths(value: str) -> tuple[Path, Optional[Path]]: +def make_source_target_paths_parser(absolute: bool = True) -> Callable[[str], tuple[Path, Path]]: + def parse_source_target_paths(value: str) -> tuple[Path, Path]: src, sep, target = value.partition(':') src_path = parse_path(src, required=False) if sep: target_path = parse_path(target, required=False, resolve=False, expanduser=False) - if absolute and not target_path.is_absolute(): - die("Target path must be absolute") + target_path = Path("/") / target_path if absolute else Path(str(target_path).lstrip("/")) else: - target_path = None + target_path = Path("/") if absolute else Path(".") return src_path, target_path return parse_source_target_paths +def config_match_build_sources(match: str, value: list[tuple[Path, Path]]) -> bool: + return Path(match.lstrip("/")) in [t for _, t in value] + + def config_parse_string(value: Optional[str], old: Optional[str]) -> Optional[str]: return value or None @@ -728,7 +731,7 @@ class MkosiConfig: repository_key_check: bool repositories: list[str] cache_only: bool - package_manager_trees: list[tuple[Path, Optional[Path]]] + package_manager_trees: list[tuple[Path, Path]] output_format: OutputFormat manifest_format: list[ManifestFormat] @@ -753,8 +756,8 @@ class MkosiConfig: with_docs: bool base_trees: list[Path] - skeleton_trees: list[tuple[Path, Optional[Path]]] - extra_trees: list[tuple[Path, Optional[Path]]] + skeleton_trees: list[tuple[Path, Path]] + extra_trees: list[tuple[Path, Path]] remove_packages: list[str] remove_files: list[str] @@ -765,7 +768,7 @@ class MkosiConfig: build_scripts: list[Path] postinst_scripts: list[Path] finalize_scripts: list[Path] - build_sources: list[tuple[Path, Optional[Path]]] + build_sources: list[tuple[Path, Path]] environment: dict[str, str] with_tests: bool with_network: bool @@ -819,7 +822,7 @@ class MkosiConfig: tools_tree_release: Optional[str] tools_tree_mirror: Optional[str] tools_tree_packages: list[str] - runtime_trees: list[tuple[Path, Optional[Path]]] + runtime_trees: list[tuple[Path, Path]] runtime_size: Optional[int] # QEMU-specific options @@ -1434,6 +1437,7 @@ SETTINGS = ( metavar="PATH", section="Content", parse=config_make_list_parser(delimiter=",", parse=make_source_target_paths_parser(absolute=False)), + match=config_match_build_sources, help="Path for sources to build", ), MkosiConfigSetting( @@ -2882,14 +2886,15 @@ def json_type_transformer(refcls: Union[type[MkosiArgs], type[MkosiConfig]]) -> return (cast(str, rootpw[0]), cast(bool, rootpw[1])) def source_target_transformer( - trees: list[list[Optional[str]]], fieldtype: type[list[tuple[Path, Optional[Path]]]] - ) -> list[tuple[Path, Optional[Path]]]: + trees: list[list[Optional[str]]], fieldtype: type[list[tuple[Path, Path]]] + ) -> list[tuple[Path, Path]]: # TODO: exchange for TypeGuard and list comprehension once on 3.10 ret = [] for src, tgt in trees: assert src is not None + assert tgt is not None source = Path(src) - target = Path(tgt) if tgt is not None else None + target = Path(tgt) ret.append((source, target)) return ret @@ -2912,7 +2917,7 @@ def json_type_transformer(refcls: Union[type[MkosiArgs], type[MkosiConfig]]) -> list[Path]: path_list_transformer, Optional[uuid.UUID]: optional_uuid_transformer, Optional[tuple[str, bool]]: root_password_transformer, - list[tuple[Path, Optional[Path]]]: source_target_transformer, + list[tuple[Path, Path]]: source_target_transformer, tuple[str, ...]: str_tuple_transformer, Architecture: enum_transformer, BiosBootloader: enum_transformer, diff --git a/mkosi/resources/mkosi.md b/mkosi/resources/mkosi.md index 3ed010dad..9897bc5d0 100644 --- a/mkosi/resources/mkosi.md +++ b/mkosi/resources/mkosi.md @@ -359,6 +359,29 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, the UAPI group version format specification. If no operator is prepended, the equality operator is assumed by default. +`BuildSources=` + +: Takes a build source target path (see `BuildSources=`). This match is + satisfied if any of the configured build sources uses this target + path. For example, if we have a `mkosi.conf` file containing: + + ```conf + [Content] + BuildSources=../abc/qed:kernel + ``` + + and a drop-in containing: + + ```conf + [Match] + BuildSources=kernel + ``` + + The drop-in will be included. + +: Any absolute paths passed to this setting are interpreted relative to + the current working directory. + | Matcher | Globs | Rich Comparisons | Default | |-------------------|-------|------------------|-------------------------| | `Profile=` | no | no | match fails | @@ -371,6 +394,7 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, | `Bootable=` | no | no | match auto feature | | `Format=` | no | no | match default format | | `SystemdVersion=` | no | yes | match fails | +| `BuildSources=` | no | no | match fails | ### [Config] Section @@ -747,31 +771,34 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, `BaseTrees=`, `--base-tree=` -: Takes a colon separated pair of directories to use as base images. When - used, these base images are each copied into the OS tree and form the - base distribution instead of installing the distribution from scratch. - Only extra packages are installed on top of the ones already installed - in the base images. Note that for this to work properly, the base image +: Takes a comma separated list of paths to use as base trees. When used, + these base trees are each copied into the OS tree and form the base + distribution instead of installing the distribution from scratch. Only + extra packages are installed on top of the ones already installed in + the base trees. Note that for this to work properly, the base image still needs to contain the package manager metadata (see `CleanPackageMetadata=`). : Instead of a directory, a tar file or a disk image may be provided. In - this case it is unpacked into the OS tree. This mode of operation allows - setting permissions and file ownership explicitly, in particular for projects - stored in a version control system such as `git` which retain full file - ownership and access mode metadata for committed files. + this case it is unpacked into the OS tree. This mode of operation + allows setting permissions and file ownership explicitly, in + particular for projects stored in a version control system such as + `git` which retain full file ownership and access mode metadata for + committed files. `SkeletonTrees=`, `--skeleton-tree=` -: Takes a colon separated pair of paths. The first path refers to a - directory to copy into the OS tree before invoking the package - manager. The second path refers to the target directory inside the - image. If the second path is not provided, the directory is copied - on top of the root directory of the image. Use this to insert files - and directories into the OS tree before the package manager installs - any packages. If the `mkosi.skeleton/` directory is found in the local - directory it is also used for this purpose with the root directory as - target (also see the **Files** section below). +: Takes a comma separated list of colon separated path pairs. The first + path of each pair refers to a directory to copy into the OS tree + before invoking the package manager. The second path of each pair + refers to the target directory inside the image. If the second path is + not provided, the directory is copied on top of the root directory of + the image. The second path is always interpreted as an absolute path. + Use this to insert files and directories into the OS tree before the + package manager installs any packages. If the `mkosi.skeleton/` + directory is found in the local directory it is also used for this + purpose with the root directory as target (also see the **Files** + section below). : As with the base tree logic above, instead of a directory, a tar file may be provided too. `mkosi.skeleton.tar` will be automatically @@ -779,14 +806,16 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, `ExtraTrees=`, `--extra-tree=` -: Takes a colon separated pair of paths. The first path refers to a - directory to copy from the host into the image. The second path refers - to the target directory inside the image. If the second path is not - provided, the directory is copied on top of the root directory of the - image. Use this to override any default configuration files shipped - with the distribution. If the `mkosi.extra/` directory is found in the - local directory it is also used for this purpose with the root - directory as target. (also see the **Files** section below). +: Takes a comma separated list of colon separated path pairs. The first + path of each pair refers to a directory to copy from the host into the + image. The second path of each pair refers to the target directory + inside the image. If the second path is not provided, the directory is + copied on top of the root directory of the image. The second path is + always interpreted as an absolute path. Use this to override any + default configuration files shipped with the distribution. If the + `mkosi.extra/` directory is found in the local directory it is also + used for this purpose with the root directory as target. (also see the + **Files** section below). : As with the base tree logic above, instead of a directory, a tar file may be provided too. `mkosi.extra.tar` will be automatically @@ -838,14 +867,16 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, `BuildSources=`, `--build-sources=` -: Takes a list of colon-separated pairs of paths to source trees and - where to mount them when running scripts. Every target path is - prefixed with the current working directory and all build sources are - sorted lexicographically by mount target before mounting so that top - level paths are mounted first. When using the `mkosi-chroot` script ( - see the **Scripts** section), the current working directory with all - build sources mounted in it is mounted to `/work/src` inside the - image's root directory. +: Takes a comma separated list of colon separated path pairs. The first + path of each pair refers to a directory to mount from the host. The + second path of each pair refers to the directory where the source + directory should be mounted when running scripts. Every target path + is prefixed with the current working directory and all build sources + are sorted lexicographically by their target before mounting so that + top level paths are mounted first. When using the `mkosi-chroot` + script ( see the **Scripts** section), the current working directory + with all build sources mounted in it is mounted to `/work/src` inside + the image's root directory. `Environment=`, `--environment=` diff --git a/tests/test_config.py b/tests/test_config.py index 5c1fe3004..bc33912d3 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -461,6 +461,26 @@ def test_match_release(tmp_path: Path, release1: int, release2: int) -> None: assert "testpkg3" in conf.packages +def test_match_build_sources(tmp_path: Path) -> None: + d = tmp_path + + (d / "mkosi.conf").write_text( + """\ + [Match] + BuildSources=kernel + BuildSources=/kernel + + [Output] + Output=abc + """ + ) + + with chdir(d): + _, [config] = parse_config(["--build-sources", ".:kernel"]) + + assert config.output == "abc" + + @pytest.mark.parametrize( "image1,image2", itertools.combinations_with_replacement( ["image_a", "image_b", "image_c"], 2 @@ -617,8 +637,8 @@ def test_package_manager_tree(tmp_path: Path, skel: Optional[Path], pkgmngr: Opt _, [conf] = parse_config() - skel_expected = [(skel, None)] if skel is not None else [] - pkgmngr_expected = [(pkgmngr, None)] if pkgmngr is not None else skel_expected + skel_expected = [(skel, Path("/"))] if skel is not None else [] + pkgmngr_expected = [(pkgmngr, Path("/"))] if pkgmngr is not None else skel_expected assert conf.skeleton_trees == skel_expected assert conf.package_manager_trees == pkgmngr_expected diff --git a/tests/test_json.py b/tests/test_json.py index dde6f9995..7007a979c 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -168,7 +168,7 @@ def test_config() -> None: "PackageManagerTrees": [ [ "/foo/bar", - null + "/" ] ], "Packages": [], @@ -225,7 +225,7 @@ def test_config() -> None: "SkeletonTrees": [ [ "/foo/bar", - null + "/" ], [ "/bar/baz", @@ -307,7 +307,7 @@ def test_config() -> None: output_dir = Path("/your/output/here"), output_format = OutputFormat.uki, overlay = True, - package_manager_trees = [(Path("/foo/bar"), None)], + package_manager_trees = [(Path("/foo/bar"), Path("/"))], packages = [], passphrase = None, postinst_scripts = [Path("/bar/qux")], @@ -341,7 +341,7 @@ def test_config() -> None: seed = uuid.UUID("7496d7d8-7f08-4a2b-96c6-ec8c43791b60"), sign = False, sign_expected_pcr = ConfigFeature.disabled, - skeleton_trees = [(Path("/foo/bar"), None), (Path("/bar/baz"), Path("/qux"))], + skeleton_trees = [(Path("/foo/bar"), Path("/")), (Path("/bar/baz"), Path("/qux"))], source_date_epoch = 12345, split_artifacts = True, ssh = False,