From: Richard Purdie Date: Tue, 10 Jun 2025 14:53:27 +0000 (+0100) Subject: classes/lib: Switch classextend to use new filter API X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=24a9858a8927e91d499ee342ed93a0dbb44d83bc;p=thirdparty%2Fopenembedded%2Fopenembedded-core.git classes/lib: Switch classextend to use new filter API Currently, class extensions are implmented using shadow variables and access indirection which is horribly ineffient and ugly. Switch to using the new bitbake filter API, which allows a translation of the variable before the expanded value is returned. This allows us to drop the shadow variable accesses. It also avoids the need to iterate PACKAGES and make many variable changes since a filter against RDEPENDS applies to RDEPENDS:${PN} and all of it's other overridden values. Since data expansion happens at access, it also avoids many of the race conditions this code has tranditionally been plagued with. Signed-off-by: Richard Purdie --- diff --git a/meta/classes-recipe/native.bbclass b/meta/classes-recipe/native.bbclass index 625975a694..7d1fe343fa 100644 --- a/meta/classes-recipe/native.bbclass +++ b/meta/classes-recipe/native.bbclass @@ -138,38 +138,18 @@ python native_virtclass_handler () { if "native" not in classextend: return - def map_dependencies(varname, d, suffix = "", selfref=True, regex=False): - if suffix: - varname = varname + ":" + suffix - deps = d.getVar(varname) - if not deps: - return - deps = bb.utils.explode_deps(deps) - newdeps = [] - for dep in deps: - if regex and dep.startswith("^") and dep.endswith("$"): - newdeps.append(dep[:-1].replace(pn, bpn) + "-native$") - elif dep == pn: - if not selfref: - continue - newdeps.append(dep) - elif "-cross-" in dep: - newdeps.append(dep.replace("-cross", "-native")) - elif not dep.endswith("-native"): - # Replace ${PN} with ${BPN} in the dependency to make sure - # dependencies on, e.g., ${PN}-foo become ${BPN}-foo-native - # rather than ${BPN}-native-foo-native. - newdeps.append(dep.replace(pn, bpn) + "-native") - else: - newdeps.append(dep) - output_varname = varname + def map_dependencies(varname, d, suffix, selfref=True, regex=False): + varname = varname + ":" + suffix # Handle ${PN}-xxx -> ${BPN}-xxx-native if suffix != "${PN}" and "${PN}" in suffix: output_varname = varname.replace("${PN}", "${BPN}") + "-native" d.renameVar(varname, output_varname) - d.setVar(output_varname, " ".join(newdeps)) - map_dependencies("DEPENDS", e.data, selfref=False) + d.setVarFilter("DEPENDS", "native_filter(val, '" + pn + "', '" + bpn + "', selfref=False)") + + for varname in ["RDEPENDS", "RRECOMMENDS", "RSUGGESTS", "RPROVIDES", "RREPLACES"]: + d.setVarFilter(varname, "native_filter(val, '" + pn + "', '" + bpn + "')") + # We need to handle things like ${@bb.utils.contains('PTEST_ENABLED', '1', '${PN}-ptest', '', d)} # and not pass ${PN}-test since in the native case it would be ignored. This does mean we ignore # anonymous python derived PACKAGES entries. @@ -181,8 +161,9 @@ python native_virtclass_handler () { map_dependencies("RSUGGESTS", e.data, pkg) map_dependencies("RPROVIDES", e.data, pkg) map_dependencies("RREPLACES", e.data, pkg) - map_dependencies("PACKAGES", e.data) - map_dependencies("PACKAGES_DYNAMIC", e.data, regex=True) + + d.setVarFilter("PACKAGES", "native_filter(val, '" + pn + "', '" + bpn + "')") + d.setVarFilter("PACKAGES_DYNAMIC", "native_filter(val, '" + pn + "', '" + bpn + "', regex=True)") provides = e.data.getVar("PROVIDES") nprovides = [] diff --git a/meta/classes-recipe/nativesdk.bbclass b/meta/classes-recipe/nativesdk.bbclass index 7ecb4c12c1..9838d5a54b 100644 --- a/meta/classes-recipe/nativesdk.bbclass +++ b/meta/classes-recipe/nativesdk.bbclass @@ -99,15 +99,14 @@ python () { import oe.classextend - clsextend = oe.classextend.NativesdkClassExtender("nativesdk", d) - clsextend.rename_packages() + clsextend = oe.classextend.ClassExtender("nativesdk", [], d) clsextend.rename_package_variables((d.getVar("PACKAGEVARS") or "").split()) - clsextend.map_depends_variable("DEPENDS") - clsextend.map_depends_variable("PACKAGE_WRITE_DEPS") + clsextend.set_filter("DEPENDS", deps=True) + clsextend.set_filter("PACKAGE_WRITE_DEPS", deps=False) clsextend.map_packagevars() - clsextend.map_variable("PROVIDES") - clsextend.map_regexp_variable("PACKAGES_DYNAMIC") + clsextend.set_filter("PROVIDES", deps=False) + d.setVar("LIBCEXTENSION", "") d.setVar("ABIEXTENSION", "") } diff --git a/meta/classes-recipe/toolchain-scripts.bbclass b/meta/classes-recipe/toolchain-scripts.bbclass index 5874a48af8..5d28df845b 100644 --- a/meta/classes-recipe/toolchain-scripts.bbclass +++ b/meta/classes-recipe/toolchain-scripts.bbclass @@ -238,11 +238,11 @@ toolchain_create_sdk_siteconfig () { python __anonymous () { import oe.classextend deps = "" + prefixes = (d.getVar("MULTILIB_VARIANTS") or "").split() for dep in (d.getVar('TOOLCHAIN_NEED_CONFIGSITE_CACHE') or "").split(): deps += " %s:do_populate_sysroot" % dep for variant in (d.getVar('MULTILIB_VARIANTS') or "").split(): - clsextend = oe.classextend.ClassExtender(variant, d) - newdep = clsextend.extend_name(dep) + newdep = oe.classextend.add_suffix(dep, variant, prefixes) deps += " %s:do_populate_sysroot" % newdep d.appendVarFlag('do_configure', 'depends', deps) } diff --git a/meta/classes/multilib.bbclass b/meta/classes/multilib.bbclass index b959bbd93c..677dbc854a 100644 --- a/meta/classes/multilib.bbclass +++ b/meta/classes/multilib.bbclass @@ -112,11 +112,11 @@ python __anonymous () { variant = d.getVar("BBEXTENDVARIANT") import oe.classextend - clsextend = oe.classextend.ClassExtender(variant, d) - - clsextend.map_depends_variable("PACKAGE_INSTALL") - clsextend.map_depends_variable("LINGUAS_INSTALL") - clsextend.map_depends_variable("RDEPENDS") + prefixes = (d.getVar("MULTILIB_VARIANTS") or "").split() + clsextend = oe.classextend.ClassExtender(variant, prefixes, d) + clsextend.set_filter("PACKAGE_INSTALL", deps=False) + clsextend.set_filter("LINGUAS_INSTALL", deps=False) + clsextend.set_filter("RDEPENDS", deps=True) pinstall = d.getVar("LINGUAS_INSTALL") + " " + d.getVar("PACKAGE_INSTALL") d.setVar("PACKAGE_INSTALL", pinstall) d.setVar("LINGUAS_INSTALL", "") @@ -136,27 +136,28 @@ python multilib_virtclass_handler_postkeyexp () { import oe.classextend - clsextend = oe.classextend.ClassExtender(variant, d) - if bb.data.inherits_class('image', d): return - clsextend.map_depends_variable("DEPENDS") - clsextend.map_depends_variable("PACKAGE_WRITE_DEPS") - clsextend.map_variable("PROVIDES") + prefixes = (d.getVar("MULTILIB_VARIANTS") or "").split() + clsextend = oe.classextend.ClassExtender(variant, prefixes, d) + + clsextend.set_filter("DEPENDS", deps=True) + clsextend.set_filter("PACKAGE_WRITE_DEPS", deps=False) + + clsextend.set_filter("PROVIDES", deps=False) if bb.data.inherits_class('cross-canadian', d): return - clsextend.rename_packages() clsextend.rename_package_variables((d.getVar("PACKAGEVARS") or "").split()) clsextend.map_packagevars() - clsextend.map_regexp_variable("PACKAGES_DYNAMIC") - clsextend.map_variable("INITSCRIPT_PACKAGES") - clsextend.map_variable("USERADD_PACKAGES") - clsextend.map_variable("SYSTEMD_PACKAGES") - clsextend.map_variable("UPDATERCPN") + + clsextend.set_filter("INITSCRIPT_PACKAGES", deps=False) + clsextend.set_filter("USERADD_PACKAGES", deps=False) + clsextend.set_filter("SYSTEMD_PACKAGES", deps=False) + clsextend.set_filter("UPDATERCPN", deps=False) reset_alternative_priority(d) } diff --git a/meta/classes/multilib_global.bbclass b/meta/classes/multilib_global.bbclass index c1d6de100c..d9372d9ed1 100644 --- a/meta/classes/multilib_global.bbclass +++ b/meta/classes/multilib_global.bbclass @@ -207,30 +207,25 @@ python multilib_virtclass_handler_global () { variants = (d.getVar("MULTILIB_VARIANTS") or "").split() - import oe.classextend - clsextends = [] - for variant in variants: - clsextends.append(oe.classextend.ClassExtender(variant, localdata)) - # Process PROVIDES origprovs = provs = localdata.getVar("PROVIDES") or "" - for clsextend in clsextends: - provs = provs + " " + clsextend.map_variable("PROVIDES", setvar=False) + for variant in variants: + provs = provs + " " + oe.classextend.suffix_filter_deps(localdata.getVar("PROVIDES") or "", variant, variants) d.setVar("PROVIDES", provs) # Process RPROVIDES origrprovs = rprovs = localdata.getVar("RPROVIDES") or "" - for clsextend in clsextends: - rprovs = rprovs + " " + clsextend.map_variable("RPROVIDES", setvar=False) + for variant in variants: + rprovs = rprovs + " " + oe.classextend.suffix_filter_deps(localdata.getVar("RPROVIDES") or "", variant, variants) if rprovs.strip(): d.setVar("RPROVIDES", rprovs) # Process RPROVIDES:${PN}... for pkg in (d.getVar("PACKAGES") or "").split(): origrprovs = rprovs = localdata.getVar("RPROVIDES:%s" % pkg) or "" - for clsextend in clsextends: - rprovs = rprovs + " " + clsextend.map_variable("RPROVIDES:%s" % pkg, setvar=False) - rprovs = rprovs + " " + clsextend.extname + "-" + pkg + for variant in variants: + rprovs = rprovs + " " + oe.classextend.suffix_filter_deps(localdata.getVar("RPROVIDES:%s" % pkg) or "", variant, variants) + rprovs = rprovs + " " + variant + "-" + pkg d.setVar("RPROVIDES:%s" % pkg, rprovs) } diff --git a/meta/lib/oe/__init__.py b/meta/lib/oe/__init__.py index 73de774266..9e4134c483 100644 --- a/meta/lib/oe/__init__.py +++ b/meta/lib/oe/__init__.py @@ -12,4 +12,4 @@ __path__ = extend_path(__path__, __name__) BBIMPORTS = ["qa", "data", "path", "utils", "types", "package", "packagedata", \ "packagegroup", "sstatesig", "lsb", "cachedpath", "license", "qemu", \ "reproducible", "rust", "buildcfg", "go", "spdx30_tasks", "spdx_common", \ - "cve_check", "tune"] + "cve_check", "tune", "classextend"] diff --git a/meta/lib/oe/classextend.py b/meta/lib/oe/classextend.py index 8ae5d3b715..ddca10dee5 100644 --- a/meta/lib/oe/classextend.py +++ b/meta/lib/oe/classextend.py @@ -5,155 +5,136 @@ # import collections +import bb.filter + +@bb.filter.filter_proc() +def native_filter(val, pn, bpn, regex=False, selfref=True): + deps = val + if not deps: + return + deps = bb.utils.explode_deps(deps) + newdeps = [] + for dep in deps: + if regex and dep.startswith("^") and dep.endswith("$"): + if not dep.endswith("-native$"): + newdeps.append(dep[:-1].replace(pn, bpn) + "-native$") + else: + newdeps.append(dep) + elif dep == pn: + if not selfref: + continue + newdeps.append(dep) + elif "-cross-" in dep: + newdeps.append(dep.replace("-cross", "-native")) + elif not dep.endswith("-native"): + # Replace ${PN} with ${BPN} in the dependency to make sure + # dependencies on, e.g., ${PN}-foo become ${BPN}-foo-native + # rather than ${BPN}-native-foo-native. + newdeps.append(dep.replace(pn, bpn) + "-native") + else: + newdeps.append(dep) + return " ".join(newdeps) + +def add_suffix(val, extname, prefixes): + if val.startswith(extname + "-"): + return val + if val.endswith(("-native", "-native-runtime")) or ('nativesdk-' in val) or ('-cross-' in val) or ('-crosssdk-' in val): + return val + # If it starts with a known prefix (e.g. multilibs), just pass it through + for prefix in prefixes: + if val.startswith(prefix + "-"): + return val + if val.startswith("kernel-") or val == "virtual/kernel": + return val + if val.startswith("rtld"): + return val + if val.endswith("-crosssdk"): + return val + if val.endswith("-" + extname): + val = val.replace("-" + extname, "") + if val.startswith("virtual/"): + # Assume large numbers of dashes means a triplet is present and we don't need to convert + if val.count("-") >= 3 and val.endswith(("-go",)): + return val + subs = val.split("/", 1)[1] + if not subs.startswith(extname): + return "virtual/" + extname + "-" + subs + return val + if val.startswith("/") or (val.startswith("${") and val.endswith("}")): + return val + if not val.startswith(extname): + return extname + "-" + val + return val + +def get_package_mappings(packages, extname): + pkgs_mapping = [] + for pkg in packages.split(): + if pkg.startswith(extname): + pkgs_mapping.append([pkg.split(extname + "-")[1], pkg]) + continue + pkgs_mapping.append([pkg, add_suffix(pkg, extname, [])]) + return pkgs_mapping + +@bb.filter.filter_proc() +def package_suffix_filter(val, extname): + pkgs_mapping = get_package_mappings(val, extname) + return " ".join([row[1] for row in pkgs_mapping]) + +@bb.filter.filter_proc() +def suffix_filter(val, extname, prefixes): + newdeps = [] + for dep in val.split(): + newdeps.append(add_suffix(dep, extname, prefixes)) + return " ".join(newdeps) + +@bb.filter.filter_proc() +def suffix_filter_regex(val, extname, prefixes): + newvar = [] + for v in val.split(): + if v.startswith("^" + extname): + newvar.append(v) + elif v.startswith("^"): + newvar.append("^" + extname + "-" + v[1:]) + else: + newvar.append(add_suffix(v, extname, prefixes)) + return " ".join(newvar) -def get_packages(d): - pkgs = d.getVar("PACKAGES_NONML") - extcls = d.getVar("EXTENDERCLASS") - return extcls.rename_packages_internal(pkgs) - -def get_depends(varprefix, d): - extcls = d.getVar("EXTENDERCLASS") - return extcls.map_depends_variable(varprefix + "_NONML") +@bb.filter.filter_proc() +def suffix_filter_deps(val, extname, prefixes): + deps = bb.utils.explode_dep_versions2(val) + newdeps = collections.OrderedDict() + for dep in deps: + newdeps[add_suffix(dep, extname, prefixes)] = deps[dep] + return bb.utils.join_deps(newdeps, False) class ClassExtender(object): - def __init__(self, extname, d): + def __init__(self, extname, prefixes, d): self.extname = extname self.d = d - self.pkgs_mapping = [] - self.d.setVar("EXTENDERCLASS", self) - - def extend_name(self, name): - if name.startswith("kernel-") or name == "virtual/kernel": - return name - if name.startswith("rtld"): - return name - if name.endswith("-crosssdk"): - return name - if name.endswith("-" + self.extname): - name = name.replace("-" + self.extname, "") - if name.startswith("virtual/"): - # Assume large numbers of dashes means a triplet is present and we don't need to convert - if name.count("-") >= 3 and name.endswith(("-go",)): - return name - subs = name.split("/", 1)[1] - if not subs.startswith(self.extname): - return "virtual/" + self.extname + "-" + subs - return name - if name.startswith("/") or (name.startswith("${") and name.endswith("}")): - return name - if not name.startswith(self.extname): - return self.extname + "-" + name - return name - - def map_variable(self, varname, setvar = True): - var = self.d.getVar(varname) - if not var: - return "" - var = var.split() - newvar = [] - for v in var: - newvar.append(self.extend_name(v)) - newdata = " ".join(newvar) - if setvar: - self.d.setVar(varname, newdata) - return newdata - - def map_regexp_variable(self, varname, setvar = True): - var = self.d.getVar(varname) - if not var: - return "" - var = var.split() - newvar = [] - for v in var: - if v.startswith("^" + self.extname): - newvar.append(v) - elif v.startswith("^"): - newvar.append("^" + self.extname + "-" + v[1:]) - else: - newvar.append(self.extend_name(v)) - newdata = " ".join(newvar) - if setvar: - self.d.setVar(varname, newdata) - return newdata + self.prefixes = prefixes - def map_depends(self, dep): - if dep.endswith(("-native", "-native-runtime")) or ('nativesdk-' in dep) or ('cross-canadian' in dep) or ('-crosssdk-' in dep): - return dep + def set_filter(self, var, deps): + if deps: + self.d.setVarFilter(var, "suffix_filter_deps(val, '" + self.extname + "', " + str(self.prefixes) + ")") else: - # Do not extend for that already have multilib prefix - var = self.d.getVar("MULTILIB_VARIANTS") - if var: - var = var.split() - for v in var: - if dep.startswith(v): - return dep - return self.extend_name(dep) - - def map_depends_variable(self, varname, suffix = ""): - # We need to preserve EXTENDPKGV so it can be expanded correctly later - if suffix: - varname = varname + ":" + suffix - orig = self.d.getVar("EXTENDPKGV", False) - self.d.setVar("EXTENDPKGV", "EXTENDPKGV") - deps = self.d.getVar(varname) - if not deps: - self.d.setVar("EXTENDPKGV", orig) - return - deps = bb.utils.explode_dep_versions2(deps) - newdeps = collections.OrderedDict() - for dep in deps: - newdeps[self.map_depends(dep)] = deps[dep] - - if not varname.endswith("_NONML"): - self.d.renameVar(varname, varname + "_NONML") - self.d.setVar(varname, "${@oe.classextend.get_depends('%s', d)}" % varname) - self.d.appendVarFlag(varname, "vardeps", " " + varname + "_NONML") - ret = bb.utils.join_deps(newdeps, False).replace("EXTENDPKGV", "${EXTENDPKGV}") - self.d.setVar("EXTENDPKGV", orig) - return ret + self.d.setVarFilter(var, "suffix_filter(val, '" + self.extname + "', " + str(self.prefixes) + ")") def map_packagevars(self): - for pkg in (self.d.getVar("PACKAGES").split() + [""]): - self.map_depends_variable("RDEPENDS", pkg) - self.map_depends_variable("RRECOMMENDS", pkg) - self.map_depends_variable("RSUGGESTS", pkg) - self.map_depends_variable("RPROVIDES", pkg) - self.map_depends_variable("RREPLACES", pkg) - self.map_depends_variable("RCONFLICTS", pkg) - self.map_depends_variable("PKG", pkg) - - def rename_packages(self): - for pkg in (self.d.getVar("PACKAGES") or "").split(): - if pkg.startswith(self.extname): - self.pkgs_mapping.append([pkg.split(self.extname + "-")[1], pkg]) - continue - self.pkgs_mapping.append([pkg, self.extend_name(pkg)]) - - self.d.renameVar("PACKAGES", "PACKAGES_NONML") - self.d.setVar("PACKAGES", "${@oe.classextend.get_packages(d)}") - - def rename_packages_internal(self, pkgs): - self.pkgs_mapping = [] - for pkg in (self.d.expand(pkgs) or "").split(): - if pkg.startswith(self.extname): - self.pkgs_mapping.append([pkg.split(self.extname + "-")[1], pkg]) - continue - self.pkgs_mapping.append([pkg, self.extend_name(pkg)]) - - return " ".join([row[1] for row in self.pkgs_mapping]) + self.set_filter("RDEPENDS", deps=True) + self.set_filter("RRECOMMENDS", deps=True) + self.set_filter("RSUGGESTS", deps=True) + self.set_filter("RPROVIDES", deps=True) + self.set_filter("RREPLACES", deps=True) + self.set_filter("RCONFLICTS", deps=True) + self.set_filter("PKG", deps=True) def rename_package_variables(self, variables): - for pkg_mapping in self.pkgs_mapping: + pkgs_mapping = get_package_mappings(self.d.getVar('PACKAGES'), self.extname) + self.d.setVarFilter('PACKAGES', "package_suffix_filter(val, '" + self.extname + "')") + self.d.setVarFilter('PACKAGES_DYNAMIC', "suffix_filter_regex(val, '" + self.extname + "', " + str(self.prefixes) + ")") + + for pkg_mapping in pkgs_mapping: if pkg_mapping[0].startswith("${") and pkg_mapping[0].endswith("}"): continue for subs in variables: self.d.renameVar("%s:%s" % (subs, pkg_mapping[0]), "%s:%s" % (subs, pkg_mapping[1])) - -class NativesdkClassExtender(ClassExtender): - def map_depends(self, dep): - if dep.startswith(self.extname): - return dep - if dep.endswith(("-native", "-native-runtime")) or ('nativesdk-' in dep) or ('-cross-' in dep) or ('-crosssdk-' in dep): - return dep - else: - return self.extend_name(dep)