]> git.ipfire.org Git - thirdparty/openembedded/openembedded-core.git/commitdiff
classes/lib: Switch classextend to use new filter API
authorRichard Purdie <richard.purdie@linuxfoundation.org>
Tue, 10 Jun 2025 14:53:27 +0000 (15:53 +0100)
committerRichard Purdie <richard.purdie@linuxfoundation.org>
Tue, 12 Aug 2025 09:02:52 +0000 (10:02 +0100)
Currently, class extensions are implmented using shadow variables and
access indirection which is horribly ineffient and ugly.

Switch to using the new bitbake filter API, which allows a translation
of the variable before the expanded value is returned. This allows us
to drop the shadow variable accesses. It also avoids the need to iterate
PACKAGES and make many variable changes since a filter against RDEPENDS
applies to RDEPENDS:${PN} and all of it's other overridden values.

Since data expansion happens at access, it also avoids many of the race
conditions this code has tranditionally been plagued with.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
meta/classes-recipe/native.bbclass
meta/classes-recipe/nativesdk.bbclass
meta/classes-recipe/toolchain-scripts.bbclass
meta/classes/multilib.bbclass
meta/classes/multilib_global.bbclass
meta/lib/oe/__init__.py
meta/lib/oe/classextend.py

index 625975a69429389a5b231a19b17e8ed7a12d80ad..7d1fe343fa03364da86cd1636e7df26ace412810 100644 (file)
@@ -138,38 +138,18 @@ python native_virtclass_handler () {
     if "native" not in classextend:
         return
 
-    def map_dependencies(varname, d, suffix = "", selfref=True, regex=False):
-        if suffix:
-            varname = varname + ":" + suffix
-        deps = d.getVar(varname)
-        if not deps:
-            return
-        deps = bb.utils.explode_deps(deps)
-        newdeps = []
-        for dep in deps:
-            if regex and dep.startswith("^") and dep.endswith("$"):
-                newdeps.append(dep[:-1].replace(pn, bpn) + "-native$")
-            elif dep == pn:
-                if not selfref:
-                    continue
-                newdeps.append(dep)
-            elif "-cross-" in dep:
-                newdeps.append(dep.replace("-cross", "-native"))
-            elif not dep.endswith("-native"):
-                # Replace ${PN} with ${BPN} in the dependency to make sure
-                # dependencies on, e.g., ${PN}-foo become ${BPN}-foo-native
-                # rather than ${BPN}-native-foo-native.
-                newdeps.append(dep.replace(pn, bpn) + "-native")
-            else:
-                newdeps.append(dep)
-        output_varname = varname
+    def map_dependencies(varname, d, suffix, selfref=True, regex=False):
+        varname = varname + ":" + suffix
         # Handle ${PN}-xxx -> ${BPN}-xxx-native
         if suffix != "${PN}" and "${PN}" in suffix:
             output_varname = varname.replace("${PN}", "${BPN}") + "-native"
             d.renameVar(varname, output_varname)
-        d.setVar(output_varname, " ".join(newdeps))
 
-    map_dependencies("DEPENDS", e.data, selfref=False)
+    d.setVarFilter("DEPENDS", "native_filter(val, '" + pn + "', '" + bpn + "', selfref=False)")
+
+    for varname in ["RDEPENDS", "RRECOMMENDS", "RSUGGESTS", "RPROVIDES", "RREPLACES"]:
+        d.setVarFilter(varname, "native_filter(val, '" + pn + "', '" + bpn + "')")
+
     # We need to handle things like ${@bb.utils.contains('PTEST_ENABLED', '1', '${PN}-ptest', '', d)}
     # and not pass ${PN}-test since in the native case it would be ignored. This does mean we ignore
     # anonymous python derived PACKAGES entries.
@@ -181,8 +161,9 @@ python native_virtclass_handler () {
         map_dependencies("RSUGGESTS", e.data, pkg)
         map_dependencies("RPROVIDES", e.data, pkg)
         map_dependencies("RREPLACES", e.data, pkg)
-    map_dependencies("PACKAGES", e.data)
-    map_dependencies("PACKAGES_DYNAMIC", e.data, regex=True)
+
+    d.setVarFilter("PACKAGES", "native_filter(val, '" + pn + "', '" + bpn + "')")
+    d.setVarFilter("PACKAGES_DYNAMIC", "native_filter(val, '" + pn + "', '" + bpn + "', regex=True)")
 
     provides = e.data.getVar("PROVIDES")
     nprovides = []
index 7ecb4c12c1f0f2b9cc48da215006a153679facf8..9838d5a54b7a133c1d0713a44c65e076be5cb408 100644 (file)
@@ -99,15 +99,14 @@ python () {
 
     import oe.classextend
 
-    clsextend = oe.classextend.NativesdkClassExtender("nativesdk", d)
-    clsextend.rename_packages()
+    clsextend = oe.classextend.ClassExtender("nativesdk", [], d)
     clsextend.rename_package_variables((d.getVar("PACKAGEVARS") or "").split())
 
-    clsextend.map_depends_variable("DEPENDS")
-    clsextend.map_depends_variable("PACKAGE_WRITE_DEPS")
+    clsextend.set_filter("DEPENDS", deps=True)
+    clsextend.set_filter("PACKAGE_WRITE_DEPS", deps=False)
     clsextend.map_packagevars()
-    clsextend.map_variable("PROVIDES")
-    clsextend.map_regexp_variable("PACKAGES_DYNAMIC")
+    clsextend.set_filter("PROVIDES", deps=False)
+
     d.setVar("LIBCEXTENSION", "")
     d.setVar("ABIEXTENSION", "")
 }
index 5874a48af8453f7b0bffbfeee3a37e094eff6774..5d28df845b88e614cb887bf0cfc56562705ca52d 100644 (file)
@@ -238,11 +238,11 @@ toolchain_create_sdk_siteconfig () {
 python __anonymous () {
     import oe.classextend
     deps = ""
+    prefixes = (d.getVar("MULTILIB_VARIANTS") or "").split()
     for dep in (d.getVar('TOOLCHAIN_NEED_CONFIGSITE_CACHE') or "").split():
         deps += " %s:do_populate_sysroot" % dep
         for variant in (d.getVar('MULTILIB_VARIANTS') or "").split():
-            clsextend = oe.classextend.ClassExtender(variant, d)
-            newdep = clsextend.extend_name(dep)
+            newdep = oe.classextend.add_suffix(dep, variant, prefixes)
             deps += " %s:do_populate_sysroot" % newdep
     d.appendVarFlag('do_configure', 'depends', deps)
 }
index b959bbd93c01c2cfb022515f2d71915e95a6db8d..677dbc854a1a15c509addc9a3c75850e0b18b6f9 100644 (file)
@@ -112,11 +112,11 @@ python __anonymous () {
         variant = d.getVar("BBEXTENDVARIANT")
         import oe.classextend
 
-        clsextend = oe.classextend.ClassExtender(variant, d)
-
-        clsextend.map_depends_variable("PACKAGE_INSTALL")
-        clsextend.map_depends_variable("LINGUAS_INSTALL")
-        clsextend.map_depends_variable("RDEPENDS")
+        prefixes = (d.getVar("MULTILIB_VARIANTS") or "").split()
+        clsextend = oe.classextend.ClassExtender(variant, prefixes, d)
+        clsextend.set_filter("PACKAGE_INSTALL", deps=False)
+        clsextend.set_filter("LINGUAS_INSTALL", deps=False)
+        clsextend.set_filter("RDEPENDS", deps=True)
         pinstall = d.getVar("LINGUAS_INSTALL") + " " + d.getVar("PACKAGE_INSTALL")
         d.setVar("PACKAGE_INSTALL", pinstall)
         d.setVar("LINGUAS_INSTALL", "")
@@ -136,27 +136,28 @@ python multilib_virtclass_handler_postkeyexp () {
 
     import oe.classextend
 
-    clsextend = oe.classextend.ClassExtender(variant, d)
-
     if bb.data.inherits_class('image', d):
         return
 
-    clsextend.map_depends_variable("DEPENDS")
-    clsextend.map_depends_variable("PACKAGE_WRITE_DEPS")
-    clsextend.map_variable("PROVIDES")
+    prefixes = (d.getVar("MULTILIB_VARIANTS") or "").split()
+    clsextend = oe.classextend.ClassExtender(variant, prefixes, d)
+
+    clsextend.set_filter("DEPENDS", deps=True)
+    clsextend.set_filter("PACKAGE_WRITE_DEPS", deps=False)
+
+    clsextend.set_filter("PROVIDES", deps=False)
 
     if bb.data.inherits_class('cross-canadian', d):
         return
 
-    clsextend.rename_packages()
     clsextend.rename_package_variables((d.getVar("PACKAGEVARS") or "").split())
 
     clsextend.map_packagevars()
-    clsextend.map_regexp_variable("PACKAGES_DYNAMIC")
-    clsextend.map_variable("INITSCRIPT_PACKAGES")
-    clsextend.map_variable("USERADD_PACKAGES")
-    clsextend.map_variable("SYSTEMD_PACKAGES")
-    clsextend.map_variable("UPDATERCPN")
+
+    clsextend.set_filter("INITSCRIPT_PACKAGES", deps=False)
+    clsextend.set_filter("USERADD_PACKAGES", deps=False)
+    clsextend.set_filter("SYSTEMD_PACKAGES", deps=False)
+    clsextend.set_filter("UPDATERCPN", deps=False)
 
     reset_alternative_priority(d)
 }
index c1d6de100c9bd4b6ef376396ede7cb17c8db5a94..d9372d9ed1ad515cc209c150b016bd49178850fc 100644 (file)
@@ -207,30 +207,25 @@ python multilib_virtclass_handler_global () {
 
             variants = (d.getVar("MULTILIB_VARIANTS") or "").split()
 
-            import oe.classextend
-            clsextends = []
-            for variant in variants:
-                clsextends.append(oe.classextend.ClassExtender(variant, localdata))
-
             # Process PROVIDES
             origprovs = provs = localdata.getVar("PROVIDES") or ""
-            for clsextend in clsextends:
-                provs = provs + " " + clsextend.map_variable("PROVIDES", setvar=False)
+            for variant in variants:
+                provs = provs + " " + oe.classextend.suffix_filter_deps(localdata.getVar("PROVIDES") or "", variant, variants)
             d.setVar("PROVIDES", provs)
 
             # Process RPROVIDES
             origrprovs = rprovs = localdata.getVar("RPROVIDES") or ""
-            for clsextend in clsextends:
-                rprovs = rprovs + " " + clsextend.map_variable("RPROVIDES", setvar=False)
+            for variant in variants:
+                rprovs = rprovs + " " + oe.classextend.suffix_filter_deps(localdata.getVar("RPROVIDES") or "", variant, variants)
             if rprovs.strip():
                 d.setVar("RPROVIDES", rprovs)
 
             # Process RPROVIDES:${PN}...
             for pkg in (d.getVar("PACKAGES") or "").split():
                 origrprovs = rprovs = localdata.getVar("RPROVIDES:%s" % pkg) or ""
-                for clsextend in clsextends:
-                    rprovs = rprovs + " " + clsextend.map_variable("RPROVIDES:%s" % pkg, setvar=False)
-                    rprovs = rprovs + " " + clsextend.extname + "-" + pkg
+                for variant in variants:
+                    rprovs = rprovs + " " + oe.classextend.suffix_filter_deps(localdata.getVar("RPROVIDES:%s" % pkg) or "", variant, variants)
+                    rprovs = rprovs + " " + variant + "-" + pkg
                 d.setVar("RPROVIDES:%s" % pkg, rprovs)
 }
 
index 73de774266b25e74b1a018fdccea73035ce20dcd..9e4134c483aafc6a0f00e44c2ff7d2ece16751d5 100644 (file)
@@ -12,4 +12,4 @@ __path__ = extend_path(__path__, __name__)
 BBIMPORTS = ["qa", "data", "path", "utils", "types", "package", "packagedata", \
              "packagegroup", "sstatesig", "lsb", "cachedpath", "license", "qemu", \
              "reproducible", "rust", "buildcfg", "go", "spdx30_tasks", "spdx_common", \
-             "cve_check", "tune"]
+             "cve_check", "tune", "classextend"]
index 8ae5d3b71527b500fee9e17a897b8db7b1413fcd..ddca10dee5441c6a8b659405d36c7476ec56ad3f 100644 (file)
 #
 
 import collections
+import bb.filter
+
+@bb.filter.filter_proc()
+def native_filter(val, pn, bpn, regex=False, selfref=True):
+    deps = val
+    if not deps:
+        return
+    deps = bb.utils.explode_deps(deps)
+    newdeps = []
+    for dep in deps:
+        if regex and dep.startswith("^") and dep.endswith("$"):
+            if not dep.endswith("-native$"):
+                newdeps.append(dep[:-1].replace(pn, bpn) + "-native$")
+            else:
+                newdeps.append(dep)
+        elif dep == pn:
+            if not selfref:
+                continue
+            newdeps.append(dep)
+        elif "-cross-" in dep:
+            newdeps.append(dep.replace("-cross", "-native"))
+        elif not dep.endswith("-native"):
+            # Replace ${PN} with ${BPN} in the dependency to make sure
+            # dependencies on, e.g., ${PN}-foo become ${BPN}-foo-native
+            # rather than ${BPN}-native-foo-native.
+            newdeps.append(dep.replace(pn, bpn) + "-native")
+        else:
+            newdeps.append(dep)
+    return " ".join(newdeps)
+
+def add_suffix(val, extname, prefixes):
+    if val.startswith(extname + "-"):
+        return val
+    if val.endswith(("-native", "-native-runtime")) or ('nativesdk-' in val) or ('-cross-' in val) or ('-crosssdk-' in val):
+        return val
+    # If it starts with a known prefix (e.g. multilibs), just pass it through
+    for prefix in prefixes:
+        if val.startswith(prefix + "-"):
+            return val
+    if val.startswith("kernel-") or val == "virtual/kernel":
+        return val
+    if val.startswith("rtld"):
+        return val
+    if val.endswith("-crosssdk"):
+        return val
+    if val.endswith("-" + extname):
+        val = val.replace("-" + extname, "")
+    if val.startswith("virtual/"):
+        # Assume large numbers of dashes means a triplet is present and we don't need to convert
+        if val.count("-") >= 3 and val.endswith(("-go",)):
+            return val
+        subs = val.split("/", 1)[1]
+        if not subs.startswith(extname):
+            return "virtual/" + extname + "-" + subs
+        return val
+    if val.startswith("/") or (val.startswith("${") and val.endswith("}")):
+        return val
+    if not val.startswith(extname):
+        return extname + "-" + val
+    return val
+
+def get_package_mappings(packages, extname):
+    pkgs_mapping = []
+    for pkg in packages.split():
+        if pkg.startswith(extname):
+           pkgs_mapping.append([pkg.split(extname + "-")[1], pkg])
+           continue
+        pkgs_mapping.append([pkg, add_suffix(pkg, extname, [])])
+    return pkgs_mapping
+
+@bb.filter.filter_proc()
+def package_suffix_filter(val, extname):
+    pkgs_mapping = get_package_mappings(val, extname)
+    return " ".join([row[1] for row in pkgs_mapping])
+
+@bb.filter.filter_proc()
+def suffix_filter(val, extname, prefixes):
+    newdeps = []
+    for dep in val.split():
+        newdeps.append(add_suffix(dep, extname, prefixes))
+    return " ".join(newdeps)
+
+@bb.filter.filter_proc()
+def suffix_filter_regex(val, extname, prefixes):
+    newvar = []
+    for v in val.split():
+        if v.startswith("^" + extname):
+            newvar.append(v)
+        elif v.startswith("^"):
+            newvar.append("^" + extname + "-" + v[1:])
+        else:
+            newvar.append(add_suffix(v, extname, prefixes))
+    return " ".join(newvar)
 
-def get_packages(d):
-    pkgs = d.getVar("PACKAGES_NONML")
-    extcls = d.getVar("EXTENDERCLASS")
-    return extcls.rename_packages_internal(pkgs)
-
-def get_depends(varprefix, d):
-    extcls = d.getVar("EXTENDERCLASS")
-    return extcls.map_depends_variable(varprefix + "_NONML")
+@bb.filter.filter_proc()
+def suffix_filter_deps(val, extname, prefixes):
+    deps = bb.utils.explode_dep_versions2(val)
+    newdeps = collections.OrderedDict()
+    for dep in deps:
+        newdeps[add_suffix(dep, extname, prefixes)] = deps[dep]
+    return bb.utils.join_deps(newdeps, False)
 
 class ClassExtender(object):
-    def __init__(self, extname, d):
+    def __init__(self, extname, prefixes, d):
         self.extname = extname
         self.d = d
-        self.pkgs_mapping = []
-        self.d.setVar("EXTENDERCLASS", self)
-
-    def extend_name(self, name):
-        if name.startswith("kernel-") or name == "virtual/kernel":
-            return name
-        if name.startswith("rtld"):
-            return name
-        if name.endswith("-crosssdk"):
-            return name
-        if name.endswith("-" + self.extname):
-            name = name.replace("-" + self.extname, "")
-        if name.startswith("virtual/"):
-            # Assume large numbers of dashes means a triplet is present and we don't need to convert
-            if name.count("-") >= 3 and name.endswith(("-go",)):
-                return name
-            subs = name.split("/", 1)[1]
-            if not subs.startswith(self.extname):
-                return "virtual/" + self.extname + "-" + subs
-            return name
-        if name.startswith("/") or (name.startswith("${") and name.endswith("}")):
-            return name
-        if not name.startswith(self.extname):
-            return self.extname + "-" + name
-        return name
-
-    def map_variable(self, varname, setvar = True):
-        var = self.d.getVar(varname)
-        if not var:
-            return ""
-        var = var.split()
-        newvar = []
-        for v in var:
-            newvar.append(self.extend_name(v))
-        newdata =  " ".join(newvar)
-        if setvar:
-            self.d.setVar(varname, newdata)
-        return newdata
-
-    def map_regexp_variable(self, varname, setvar = True):
-        var = self.d.getVar(varname)
-        if not var:
-            return ""
-        var = var.split()
-        newvar = []
-        for v in var:
-            if v.startswith("^" + self.extname):
-                newvar.append(v)
-            elif v.startswith("^"):
-                newvar.append("^" + self.extname + "-" + v[1:])
-            else:
-                newvar.append(self.extend_name(v))
-        newdata =  " ".join(newvar)
-        if setvar:
-            self.d.setVar(varname, newdata)
-        return newdata
+        self.prefixes = prefixes
 
-    def map_depends(self, dep):
-        if dep.endswith(("-native", "-native-runtime")) or ('nativesdk-' in dep) or ('cross-canadian' in dep) or ('-crosssdk-' in dep):
-            return dep
+    def set_filter(self, var, deps):
+        if deps:
+            self.d.setVarFilter(var, "suffix_filter_deps(val, '" + self.extname + "', " + str(self.prefixes) + ")")
         else:
-            # Do not extend for that already have multilib prefix
-            var = self.d.getVar("MULTILIB_VARIANTS")
-            if var:
-                var = var.split()
-                for v in var:
-                    if dep.startswith(v):
-                        return dep
-            return self.extend_name(dep)
-
-    def map_depends_variable(self, varname, suffix = ""):
-        # We need to preserve EXTENDPKGV so it can be expanded correctly later
-        if suffix:
-            varname = varname + ":" + suffix
-        orig = self.d.getVar("EXTENDPKGV", False)
-        self.d.setVar("EXTENDPKGV", "EXTENDPKGV")
-        deps = self.d.getVar(varname)
-        if not deps:
-            self.d.setVar("EXTENDPKGV", orig)
-            return
-        deps = bb.utils.explode_dep_versions2(deps)
-        newdeps = collections.OrderedDict()
-        for dep in deps:
-            newdeps[self.map_depends(dep)] = deps[dep]
-
-        if not varname.endswith("_NONML"):
-            self.d.renameVar(varname, varname + "_NONML")
-            self.d.setVar(varname, "${@oe.classextend.get_depends('%s', d)}" % varname)
-            self.d.appendVarFlag(varname, "vardeps", " " + varname + "_NONML")
-        ret = bb.utils.join_deps(newdeps, False).replace("EXTENDPKGV", "${EXTENDPKGV}")
-        self.d.setVar("EXTENDPKGV", orig)
-        return ret
+            self.d.setVarFilter(var, "suffix_filter(val, '" + self.extname + "', " + str(self.prefixes) + ")")
 
     def map_packagevars(self):
-        for pkg in (self.d.getVar("PACKAGES").split() + [""]):
-            self.map_depends_variable("RDEPENDS", pkg)
-            self.map_depends_variable("RRECOMMENDS", pkg)
-            self.map_depends_variable("RSUGGESTS", pkg)
-            self.map_depends_variable("RPROVIDES", pkg)
-            self.map_depends_variable("RREPLACES", pkg)
-            self.map_depends_variable("RCONFLICTS", pkg)
-            self.map_depends_variable("PKG", pkg)
-
-    def rename_packages(self):
-        for pkg in (self.d.getVar("PACKAGES") or "").split():
-            if pkg.startswith(self.extname):
-               self.pkgs_mapping.append([pkg.split(self.extname + "-")[1], pkg])
-               continue
-            self.pkgs_mapping.append([pkg, self.extend_name(pkg)])
-
-        self.d.renameVar("PACKAGES", "PACKAGES_NONML")
-        self.d.setVar("PACKAGES", "${@oe.classextend.get_packages(d)}")
-
-    def rename_packages_internal(self, pkgs):
-        self.pkgs_mapping = []
-        for pkg in (self.d.expand(pkgs) or "").split():
-            if pkg.startswith(self.extname):
-               self.pkgs_mapping.append([pkg.split(self.extname + "-")[1], pkg])
-               continue
-            self.pkgs_mapping.append([pkg, self.extend_name(pkg)])
-
-        return " ".join([row[1] for row in self.pkgs_mapping])
+        self.set_filter("RDEPENDS", deps=True)
+        self.set_filter("RRECOMMENDS", deps=True)
+        self.set_filter("RSUGGESTS", deps=True)
+        self.set_filter("RPROVIDES", deps=True)
+        self.set_filter("RREPLACES", deps=True)
+        self.set_filter("RCONFLICTS", deps=True)
+        self.set_filter("PKG", deps=True)
 
     def rename_package_variables(self, variables):
-        for pkg_mapping in self.pkgs_mapping:
+        pkgs_mapping = get_package_mappings(self.d.getVar('PACKAGES'), self.extname)
+        self.d.setVarFilter('PACKAGES', "package_suffix_filter(val, '" + self.extname + "')")
+        self.d.setVarFilter('PACKAGES_DYNAMIC', "suffix_filter_regex(val, '" + self.extname + "', " + str(self.prefixes) + ")")
+
+        for pkg_mapping in pkgs_mapping:
             if pkg_mapping[0].startswith("${") and pkg_mapping[0].endswith("}"):
                 continue
             for subs in variables:
                 self.d.renameVar("%s:%s" % (subs, pkg_mapping[0]), "%s:%s" % (subs, pkg_mapping[1]))
-
-class NativesdkClassExtender(ClassExtender):
-    def map_depends(self, dep):
-        if dep.startswith(self.extname):
-            return dep
-        if dep.endswith(("-native", "-native-runtime")) or ('nativesdk-' in dep) or ('-cross-' in dep) or ('-crosssdk-' in dep):
-            return dep
-        else:
-            return self.extend_name(dep)