bitbake/cooker/codeparser: Ensure the code parser cache is saved for each parsing...

Before this change, the codeparser cache was only being saved for the main
server process. This is suboptimal as it leaves code being re-evaluated at
task execution time and increases parse time.

We use the multiprocess Finalize() functionality to ensure each process
saves out its cache. We need to update the cache save function to be multiprocess
friendly with locking.

(From Poky rev: c8928e93dd8a08d6768623f6491c9ba31d0aa06f)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

diff --git a/lib/bb/codeparser.py b/lib/bb/codeparser.py
index 84d1c09f21ad8a7fca505ede164d6e19c97d4fc9..fdba06f6785e6c4fbed0ee6f208ec7cdf38b98ae 100644 (file)
--- a/lib/bb/codeparser.py
+++ b/lib/bb/codeparser.py
@@ -70,8 +70,22 @@ def parser_cache_save(d):
     if not cachefile:
         return
 
+    lf = bb.utils.lockfile(cachefile + ".lock")
+
+    p = pickle.Unpickler(file(cachefile, "rb"))
+    data, version = p.load()
+
+    if version == PARSERCACHE_VERSION:
+        for h in data[0]:
+            if h not in pythonparsecache:
+                pythonparsecache[h] = data[0][h]
+        for h in data[1]:
+            if h not in pythonparsecache:
+                shellparsecache[h] = data[1][h]
+
     p = pickle.Pickler(file(cachefile, "wb"), -1)
     p.dump([[pythonparsecache, shellparsecache], PARSERCACHE_VERSION])
+    bb.utils.unlockfile(lf)
 
 class PythonParser():
     class ValueVisitor():

diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index d4415d324df4f9ee8e601efea8848273c52a9ef7..9f1fa2a6bcdcb49af84f668521a2c121c39539e2 100644 (file)
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -1072,6 +1072,7 @@ class CookerParser(object):
     def start(self):
         def init(cfg):
             parse_file.cfg = cfg
+            multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, args=(self.cooker.configuration.data, ), exitpriority=1)
 
         self.results = self.load_cached()