From: Daan De Meyer Date: Mon, 6 Nov 2023 14:40:19 +0000 (+0100) Subject: Fix RuntimeTrees= X-Git-Tag: v19~26^2~6 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b3a59f403f989a7e8b5a5965d7fd8fcc3584c0c1;p=thirdparty%2Fmkosi.git Fix RuntimeTrees= We previously made the target path non-optional but this doesn't actually work for RuntimeTrees=, where we need optional paths. So instead, let's introduce ConfigTree to abstract the tree concept and have a class where we can define methods on to make working with trees easier. --- diff --git a/mkosi/__init__.py b/mkosi/__init__.py index 01d2d71ce..f2e5fc37c 100644 --- a/mkosi/__init__.py +++ b/mkosi/__init__.py @@ -37,7 +37,7 @@ from mkosi.config import ( SecureBootSignTool, Verb, format_bytes, - format_source_target, + format_tree, parse_config, summary, ) @@ -319,9 +319,9 @@ def mount_volatile_overlay(state: MkosiState) -> Iterator[Path]: def finalize_mounts(config: MkosiConfig) -> list[PathString]: sources = [ - (src, Path.cwd() / target) - for src, target - in ((Path.cwd(), Path(".")), *config.build_sources) + (source, target) + for source, target + in [(Path.cwd(), Path.cwd())] + [t.with_prefix(Path.cwd()) for t in config.build_sources] ] # bwrap() mounts /home and /var read-only during execution. So let's add the bind mount options for the @@ -1036,7 +1036,8 @@ def install_skeleton_trees(state: MkosiState) -> None: return with complete_step("Copying in skeleton file trees…"): - for source, target in state.config.skeleton_trees: + for tree in state.config.skeleton_trees: + source, target = tree.with_prefix() install_tree(state.config, source, state.root, target) @@ -1045,7 +1046,8 @@ def install_package_manager_trees(state: MkosiState) -> None: return with complete_step("Copying in package manager file trees…"): - for source, target in state.config.package_manager_trees: + for tree in state.config.package_manager_trees: + source, target = tree.with_prefix() install_tree(state.config, source, state.workspace / "pkgmngr", target) @@ -1054,7 +1056,8 @@ def install_extra_trees(state: MkosiState) -> None: return with complete_step("Copying in extra file trees…"): - for source, target in state.config.extra_trees: + for tree in state.config.extra_trees: + source, target = tree.with_prefix() install_tree(state.config, source, state.root, target) @@ -1106,7 +1109,7 @@ def build_initrd(state: MkosiState) -> Path: *(["--mirror", state.config.mirror] if state.config.mirror else []), "--repository-key-check", str(state.config.repository_key_check), "--repositories", ",".join(state.config.repositories), - "--package-manager-tree", ",".join(format_source_target(s, t) for s, t in state.config.package_manager_trees), + "--package-manager-tree", ",".join(format_tree(t) for t in state.config.package_manager_trees), *(["--compress-output", str(state.config.compress_output)] if state.config.compress_output else []), "--with-network", str(state.config.with_network), "--cache-only", str(state.config.cache_only), @@ -1620,9 +1623,9 @@ def check_inputs(config: MkosiConfig) -> None: for name, trees in (("Skeleton", config.skeleton_trees), ("Package manager", config.package_manager_trees), ("Extra", config.extra_trees)): - for src, _ in trees: - if not src.exists(): - die(f"{name} tree {src} not found") + for tree in trees: + if not tree.source.exists(): + die(f"{name} tree {tree.source} not found") if config.bootable != ConfigFeature.disabled: for p in config.initrds: @@ -2241,8 +2244,8 @@ def acl_toggle_build(config: MkosiConfig, uid: int) -> Iterator[None]: yield return - extras = [e[0] for e in config.extra_trees] - skeletons = [s[0] for s in config.skeleton_trees] + extras = [t.source for t in config.extra_trees] + skeletons = [t.source for t in config.skeleton_trees] with contextlib.ExitStack() as stack: for p in (*config.base_trees, *extras, *skeletons): @@ -2315,13 +2318,14 @@ def run_shell(args: MkosiArgs, config: MkosiConfig) -> None: else: cmdline += ["--image", fname] - for src, tgt in config.runtime_trees: + for tree in config.runtime_trees: + target = Path("/root/src") / (tree.target or tree.source.name) # We add norbind because very often RuntimeTrees= will be used to mount the source directory into the # container and the output directory from which we're running will very likely be a subdirectory of the # source directory which would mean we'd be mounting the container root directory as a subdirectory in # itself which tends to lead to all kinds of weird issues, which we avoid by not doing a recursive mount # which means the container root directory mounts will be skipped. - cmdline += ["--bind", f"{src}:{tgt or f'/root/src/{src.name}'}:norbind,rootidmap"] + cmdline += ["--bind", f"{tree.source}:{target}:norbind,rootidmap"] if args.verb == Verb.boot: # Add nspawn options first since systemd-nspawn ignores all options after the first argument. diff --git a/mkosi/config.py b/mkosi/config.py index 82b3b1594..8d5456db3 100644 --- a/mkosi/config.py +++ b/mkosi/config.py @@ -75,6 +75,15 @@ class ConfigFeature(StrEnum): disabled = enum.auto() +@dataclasses.dataclass(frozen=True) +class ConfigTree: + source: Path + target: Optional[Path] + + def with_prefix(self, prefix: Path = Path("/")) -> tuple[Path, Path]: + return (self.source, prefix / os.fspath(self.target).lstrip("/") if self.target else prefix) + + class SecureBootSignTool(StrEnum): auto = enum.auto() sbsign = enum.auto() @@ -185,22 +194,17 @@ def parse_path(value: str, return path -def make_source_target_paths_parser(absolute: bool = True) -> Callable[[str], tuple[Path, Path]]: - def parse_source_target_paths(value: str) -> tuple[Path, Path]: - src, sep, target = value.partition(':') - src_path = parse_path(src, required=False) - if sep: - target_path = parse_path(target, required=False, resolve=False, expanduser=False) - target_path = Path("/") / target_path if absolute else Path(str(target_path).lstrip("/")) - else: - target_path = Path("/") if absolute else Path(".") - return src_path, target_path +def parse_tree(value: str) -> ConfigTree: + src, sep, tgt = value.partition(':') - return parse_source_target_paths + return ConfigTree( + source=parse_path(src, required=False), + target=parse_path(tgt, required=False, resolve=False, expanduser=False) if sep else None, + ) -def config_match_build_sources(match: str, value: list[tuple[Path, Path]]) -> bool: - return Path(match.lstrip("/")) in [t for _, t in value] +def config_match_build_sources(match: str, value: list[ConfigTree]) -> bool: + return Path(match.lstrip("/")) in [tree.target for tree in value if tree.target] def config_parse_string(value: Optional[str], old: Optional[str]) -> Optional[str]: @@ -733,7 +737,7 @@ class MkosiConfig: repository_key_check: bool repositories: list[str] cache_only: bool - package_manager_trees: list[tuple[Path, Path]] + package_manager_trees: list[ConfigTree] output_format: OutputFormat manifest_format: list[ManifestFormat] @@ -758,8 +762,8 @@ class MkosiConfig: with_docs: bool base_trees: list[Path] - skeleton_trees: list[tuple[Path, Path]] - extra_trees: list[tuple[Path, Path]] + skeleton_trees: list[ConfigTree] + extra_trees: list[ConfigTree] remove_packages: list[str] remove_files: list[str] @@ -770,7 +774,7 @@ class MkosiConfig: build_scripts: list[Path] postinst_scripts: list[Path] finalize_scripts: list[Path] - build_sources: list[tuple[Path, Path]] + build_sources: list[ConfigTree] environment: dict[str, str] with_tests: bool with_network: bool @@ -824,7 +828,7 @@ class MkosiConfig: tools_tree_release: Optional[str] tools_tree_mirror: Optional[str] tools_tree_packages: list[str] - runtime_trees: list[tuple[Path, Path]] + runtime_trees: list[ConfigTree] runtime_size: Optional[int] # QEMU-specific options @@ -1151,7 +1155,7 @@ SETTINGS = ( long="--package-manager-tree", metavar="PATH", section="Distribution", - parse=config_make_list_parser(delimiter=",", parse=make_source_target_paths_parser()), + parse=config_make_list_parser(delimiter=",", parse=parse_tree), default_factory=lambda ns: ns.skeleton_trees, default_factory_depends=("skeleton_trees",), help="Use a package manager tree to configure the package manager", @@ -1346,7 +1350,7 @@ SETTINGS = ( long="--skeleton-tree", metavar="PATH", section="Content", - parse=config_make_list_parser(delimiter=",", parse=make_source_target_paths_parser()), + parse=config_make_list_parser(delimiter=",", parse=parse_tree), paths=("mkosi.skeleton", "mkosi.skeleton.tar"), path_default=False, help="Use a skeleton tree to bootstrap the image before installing anything", @@ -1356,7 +1360,7 @@ SETTINGS = ( long="--extra-tree", metavar="PATH", section="Content", - parse=config_make_list_parser(delimiter=",", parse=make_source_target_paths_parser()), + parse=config_make_list_parser(delimiter=",", parse=parse_tree), paths=("mkosi.extra", "mkosi.extra.tar"), path_default=False, help="Copy an extra tree on top of image", @@ -1441,7 +1445,7 @@ SETTINGS = ( dest="build_sources", metavar="PATH", section="Content", - parse=config_make_list_parser(delimiter=",", parse=make_source_target_paths_parser(absolute=False)), + parse=config_make_list_parser(delimiter=",", parse=parse_tree), match=config_match_build_sources, help="Path for sources to build", ), @@ -1902,7 +1906,7 @@ SETTINGS = ( long="--runtime-tree", metavar="SOURCE:[TARGET]", section="Host", - parse=config_make_list_parser(delimiter=",", parse=make_source_target_paths_parser()), + parse=config_make_list_parser(delimiter=",", parse=parse_tree), help="Additional mounts to add when booting the image", ), MkosiConfigSetting( @@ -2692,15 +2696,15 @@ def line_join_list(array: Iterable[PathString]) -> str: return "\n ".join(items) -def format_source_target(source: Path, target: Optional[Path]) -> str: - return f"{source}:{target}" if target else f"{source}" +def format_tree(tree: ConfigTree) -> str: + return f"{tree.source}:{tree.target}" if tree.target else f"{tree.source}" -def line_join_source_target_list(array: Sequence[tuple[Path, Optional[Path]]]) -> str: +def line_join_tree_list(array: Sequence[ConfigTree]) -> str: if not array: return "none" - items = [format_source_target(source, target) for source, target in array] + items = [format_tree(tree) for tree in array] return "\n ".join(items) @@ -2744,7 +2748,7 @@ def summary(config: MkosiConfig) -> str: Repo Signature/Key check: {yes_no(config.repository_key_check)} Repositories: {line_join_list(config.repositories)} Use Only Package Cache: {yes_no(config.cache_only)} - Package Manager Trees: {line_join_source_target_list(config.package_manager_trees)} + Package Manager Trees: {line_join_tree_list(config.package_manager_trees)} {bold("OUTPUT")}: Output Format: {config.output_format} @@ -2770,8 +2774,8 @@ def summary(config: MkosiConfig) -> str: With Documentation: {yes_no(config.with_docs)} Base Trees: {line_join_list(config.base_trees)} - Skeleton Trees: {line_join_source_target_list(config.skeleton_trees)} - Extra Trees: {line_join_source_target_list(config.extra_trees)} + Skeleton Trees: {line_join_tree_list(config.skeleton_trees)} + Extra Trees: {line_join_tree_list(config.extra_trees)} Remove Packages: {line_join_list(config.remove_packages)} Remove Files: {line_join_list(config.remove_files)} @@ -2782,7 +2786,7 @@ Clean Package Manager Metadata: {yes_no_auto(config.clean_package_metadata)} Build Scripts: {line_join_list(config.build_scripts)} Postinstall Scripts: {line_join_list(config.postinst_scripts)} Finalize Scripts: {line_join_list(config.finalize_scripts)} - Build Sources: {line_join_source_target_list(config.build_sources)} + Build Sources: {line_join_tree_list(config.build_sources)} Script Environment: {line_join_list(env)} Run Tests in Build Scripts: {yes_no(config.with_tests)} Scripts With Network: {yes_no(config.with_network)} @@ -2845,7 +2849,7 @@ Clean Package Manager Metadata: {yes_no_auto(config.clean_package_metadata)} Tools Tree Release: {none_to_none(config.tools_tree_release)} Tools Tree Mirror: {none_to_default(config.tools_tree_mirror)} Tools Tree Packages: {line_join_list(config.tools_tree_packages)} - Runtime Trees: {line_join_source_target_list(config.runtime_trees)} + Runtime Trees: {line_join_tree_list(config.runtime_trees)} Runtime Size: {format_bytes_or_none(config.runtime_size)} QEMU GUI: {yes_no(config.qemu_gui)} @@ -2900,17 +2904,18 @@ def json_type_transformer(refcls: Union[type[MkosiArgs], type[MkosiConfig]]) -> return None return (cast(str, rootpw[0]), cast(bool, rootpw[1])) - def source_target_transformer( - trees: list[list[Optional[str]]], fieldtype: type[list[tuple[Path, Path]]] - ) -> list[tuple[Path, Path]]: + def config_tree_transformer(trees: list[dict[str, Any]], fieldtype: type[ConfigTree]) -> list[ConfigTree]: # TODO: exchange for TypeGuard and list comprehension once on 3.10 ret = [] - for src, tgt in trees: - assert src is not None - assert tgt is not None - source = Path(src) - target = Path(tgt) - ret.append((source, target)) + for d in trees: + assert "source" in d + assert "target" in d + ret.append( + ConfigTree( + source=Path(d["source"]), + target=Path(d["target"]) if d["target"] is not None else None, + ) + ) return ret def enum_transformer(enumval: str, fieldtype: type[E]) -> E: @@ -2932,7 +2937,7 @@ def json_type_transformer(refcls: Union[type[MkosiArgs], type[MkosiConfig]]) -> list[Path]: path_list_transformer, Optional[uuid.UUID]: optional_uuid_transformer, Optional[tuple[str, bool]]: root_password_transformer, - list[tuple[Path, Path]]: source_target_transformer, + list[ConfigTree]: config_tree_transformer, tuple[str, ...]: str_tuple_transformer, Architecture: enum_transformer, BiosBootloader: enum_transformer, diff --git a/mkosi/qemu.py b/mkosi/qemu.py index 3fb03490e..f09a37c5d 100644 --- a/mkosi/qemu.py +++ b/mkosi/qemu.py @@ -549,13 +549,14 @@ def run_qemu(args: MkosiArgs, config: MkosiConfig, qemu_device_fds: Mapping[Qemu if root: kcl += [root] - for src, target in config.runtime_trees: - sock = stack.enter_context(start_virtiofsd(src, uidmap=True)) + for tree in config.runtime_trees: + sock = stack.enter_context(start_virtiofsd(tree.source, uidmap=True)) cmdline += [ "-chardev", f"socket,id={sock.name},path={sock}", "-device", f"vhost-user-fs-pci,queue-size=1024,chardev={sock.name},tag={sock.name}", ] - kcl += [f"systemd.mount-extra={sock.name}:{target or f'/root/src/{src.name}'}:virtiofs"] + target = Path("/root/src") / (tree.target or tree.source.name) + kcl += [f"systemd.mount-extra={sock.name}:{target}:virtiofs"] if kernel and (KernelType.identify(kernel) != KernelType.uki or not config.architecture.supports_smbios()): cmdline += ["-append", " ".join(kcl)] diff --git a/mkosi/resources/mkosi.md b/mkosi/resources/mkosi.md index 670972bae..35c606e39 100644 --- a/mkosi/resources/mkosi.md +++ b/mkosi/resources/mkosi.md @@ -1348,7 +1348,8 @@ boolean argument: either `1`, `yes`, or `true` to enable, or `0`, `no`, directory to mount into any machine (container or VM) started by mkosi. The second path refers to the target directory inside the machine. If the second path is not provided, the directory is mounted - below `/root/src` in the machine. + below `/root/src` in the machine. If the second path is relative, it + is interpreted relative to `/root/src` in the machine. : For each mounted directory, the uid and gid of the user running mkosi are mapped to the root user in the machine. This means that all the diff --git a/tests/test_config.py b/tests/test_config.py index f2bfa13d0..f5bc6afe3 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -14,6 +14,7 @@ from mkosi.architecture import Architecture from mkosi.config import ( Compression, ConfigFeature, + ConfigTree, MkosiConfig, OutputFormat, Verb, @@ -639,8 +640,8 @@ def test_package_manager_tree(tmp_path: Path, skel: Optional[Path], pkgmngr: Opt _, [conf] = parse_config() - skel_expected = [(skel, Path("/"))] if skel is not None else [] - pkgmngr_expected = [(pkgmngr, Path("/"))] if pkgmngr is not None else skel_expected + skel_expected = [ConfigTree(skel, None)] if skel is not None else [] + pkgmngr_expected = [ConfigTree(pkgmngr, None)] if pkgmngr is not None else skel_expected assert conf.skeleton_trees == skel_expected assert conf.package_manager_trees == pkgmngr_expected diff --git a/tests/test_json.py b/tests/test_json.py index 7007a979c..d0c1663f3 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -14,6 +14,7 @@ from mkosi.config import ( Bootloader, Compression, ConfigFeature, + ConfigTree, DocFormat, ManifestFormat, MkosiArgs, @@ -91,10 +92,10 @@ def test_config() -> None: "/path/to/buildscript" ], "BuildSources": [ - [ - "/qux", - "/frob" - ] + { + "source": "/qux", + "target": "/frob" + } ], "CacheDirectory": "/is/this/the/cachedir", "CacheOnly": true, @@ -166,10 +167,10 @@ def test_config() -> None: "OutputDirectory": "/your/output/here", "Overlay": true, "PackageManagerTrees": [ - [ - "/foo/bar", - "/" - ] + { + "source": "/foo/bar", + "target": null + } ], "Packages": [], "Passphrase": null, @@ -205,14 +206,14 @@ def test_config() -> None: "RootShell": "/bin/tcsh", "RuntimeSize": 8589934592, "RuntimeTrees": [ - [ - "/foo/bar", - "/baz" - ], - [ - "/bar/baz", - "/qux" - ] + { + "source": "/foo/bar", + "target": "/baz" + }, + { + "source": "/bar/baz", + "target": "/qux" + } ], "SectorSize": null, "SecureBoot": true, @@ -223,14 +224,14 @@ def test_config() -> None: "Sign": false, "SignExpectedPcr": "disabled", "SkeletonTrees": [ - [ - "/foo/bar", - "/" - ], - [ - "/bar/baz", - "/qux" - ] + { + "source": "/foo/bar", + "target": "/" + }, + { + "source": "/bar/baz", + "target": "/qux" + } ], "SourceDateEpoch": 12345, "SplitArtifacts": true, @@ -264,7 +265,7 @@ def test_config() -> None: build_dir = None, build_packages = ["pkg1", "pkg2"], build_scripts = [Path("/path/to/buildscript")], - build_sources = [(Path("/qux"), Path("/frob"))], + build_sources = [ConfigTree(Path("/qux"), Path("/frob"))], cache_dir = Path("/is/this/the/cachedir"), cache_only = True, checksum = False, @@ -307,7 +308,7 @@ def test_config() -> None: output_dir = Path("/your/output/here"), output_format = OutputFormat.uki, overlay = True, - package_manager_trees = [(Path("/foo/bar"), Path("/"))], + package_manager_trees = [ConfigTree(Path("/foo/bar"), None)], packages = [], passphrase = None, postinst_scripts = [Path("/bar/qux")], @@ -332,7 +333,7 @@ def test_config() -> None: root_password = ("test1234", False), root_shell = "/bin/tcsh", runtime_size = 8589934592, - runtime_trees = [(Path("/foo/bar"), Path("/baz")), (Path("/bar/baz"), Path("/qux"))], + runtime_trees = [ConfigTree(Path("/foo/bar"), Path("/baz")), ConfigTree(Path("/bar/baz"), Path("/qux"))], sector_size = None, secure_boot = True, secure_boot_certificate = None, @@ -341,7 +342,7 @@ def test_config() -> None: seed = uuid.UUID("7496d7d8-7f08-4a2b-96c6-ec8c43791b60"), sign = False, sign_expected_pcr = ConfigFeature.disabled, - skeleton_trees = [(Path("/foo/bar"), Path("/")), (Path("/bar/baz"), Path("/qux"))], + skeleton_trees = [ConfigTree(Path("/foo/bar"), Path("/")), ConfigTree(Path("/bar/baz"), Path("/qux"))], source_date_epoch = 12345, split_artifacts = True, ssh = False,