spdx30: Improve os.walk() handling

There have been errors seen when assembling root file system SPDX
documents where they will references files that don't exist in the
package SPDX.

The speculation is that this is caused by os.walk() ignoring errors when
walking, causing files to be omitted. Improve the code by adding an
error handler to os.walk() to report errors when they occur.

In addition, sort the files and directories while walking to ensure
consistent ordering of the file SPDX IDs.

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

diff --git a/meta/lib/oe/spdx30_tasks.py b/meta/lib/oe/spdx30_tasks.py
index 6a39246fe1c32c9786801efdc579a8f2064578d4..e3e5dbc742764585df855441f3cbc790971fde74 100644 (file)
--- a/meta/lib/oe/spdx30_tasks.py
+++ b/meta/lib/oe/spdx30_tasks.py
@@ -19,6 +19,10 @@ from datetime import datetime, timezone
 from pathlib import Path
 
 
+def walk_error(err):
+    bb.error(f"ERROR walking {err.filename}: {err}")
+
+
 def set_timestamp_now(d, o, prop):
     if d.getVar("SPDX_INCLUDE_TIMESTAMPS") == "1":
         setattr(o, prop, datetime.now(timezone.utc))
@@ -148,11 +152,13 @@ def add_package_files(
     spdx_files = set()
 
     file_counter = 1
-    for subdir, dirs, files in os.walk(topdir):
+    for subdir, dirs, files in os.walk(topdir, onerror=walk_error):
         dirs[:] = [d for d in dirs if d not in ignore_dirs]
         if subdir == str(topdir):
             dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs]
 
+        dirs.sort()
+        files.sort()
         for file in files:
             filepath = Path(subdir) / file
             if filepath.is_symlink() or not filepath.is_file():
@@ -356,7 +362,9 @@ def add_download_files(d, objset):
             if fd.type == "file":
                 if os.path.isdir(fd.localpath):
                     walk_idx = 1
-                    for root, dirs, files in os.walk(fd.localpath):
+                    for root, dirs, files in os.walk(fd.localpath, onerror=walk_error):
+                        dirs.sort()
+                        files.sort()
                         for f in files:
                             f_path = os.path.join(root, f)
                             if os.path.islink(f_path):
@@ -1046,7 +1054,9 @@ def create_rootfs_spdx(d):
     collect_build_package_inputs(d, objset, rootfs_build, packages, files_by_hash)
 
     files = set()
-    for dirpath, dirnames, filenames in os.walk(image_rootfs):
+    for dirpath, dirnames, filenames in os.walk(image_rootfs, onerror=walk_error):
+        dirnames.sort()
+        filenames.sort()
         for fn in filenames:
             fpath = Path(dirpath) / fn
             if not fpath.is_file() or fpath.is_symlink():
@@ -1282,7 +1292,9 @@ def create_sdk_sbom(d, sdk_deploydir, spdx_work_dir, toolchain_outputname):
     root_files = []
 
     # NOTE: os.walk() doesn't return symlinks
-    for dirpath, dirnames, filenames in os.walk(sdk_deploydir):
+    for dirpath, dirnames, filenames in os.walk(sdk_deploydir, onerror=walk_error):
+        dirnames.sort()
+        filenames.sort()
         for fn in filenames:
             fpath = Path(dirpath) / fn
             if not fpath.is_file() or fpath.is_symlink():