bitbake/codeparser: Improve cache handling

The current codeparser cache handling hurts performance badly even
over a couple of cores and certainly on many core systems, it can
spent huge amounts of time in the codeparser cache save functions.

This patch reworks the cache handling so that each parsing thread
saves out its own "differences" file compared to any existing core
cache and then the main bitbake thread picks these up and merges
things back together.

This was tested on systems with small and large numbers of cores
and was found to perform orders of magnitude better in all cases
despite the more complex code.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

diff --git a/lib/bb/codeparser.py b/lib/bb/codeparser.py
index bb26167481028185f87ae897b1b51e2d1e3e73ed..936b742614671757a17c0da385d1950602c74cdc 100644 (file)
--- a/lib/bb/codeparser.py
+++ b/lib/bb/codeparser.py
@@ -70,7 +70,51 @@ def parser_cache_save(d):
     if not cachefile:
         return
 
-    lf = bb.utils.lockfile(cachefile + ".lock")
+    glf = bb.utils.lockfile(cachefile + ".lock", shared=True)
+
+    i = os.getpid()
+    lf = None
+    while not lf:
+        shellcache = {}
+        pythoncache = {}
+
+        lf = bb.utils.lockfile(cachefile + ".lock." + str(i), retry=False)
+        if not lf or os.path.exists(cachefile + "-" + str(i)):
+            if lf:
+               bb.utils.unlockfile(lf) 
+               lf = None
+            i = i + 1
+            continue
+
+        try:
+            p = pickle.Unpickler(file(cachefile, "rb"))
+            data, version = p.load()
+        except (IOError, EOFError):
+            data, version = None, None
+
+        if version != PARSERCACHE_VERSION:
+            shellcache = shellparsecache
+            pythoncache = pythonparsecache
+        else:
+            for h in pythonparsecache:
+                if h not in data[0]:
+                    pythoncache[h] = pythonparsecache[h]
+            for h in shellparsecache:
+                if h not in data[1]:
+                    shellcache[h] = shellparsecache[h]
+
+        p = pickle.Pickler(file(cachefile + "-" + str(i), "wb"), -1)
+        p.dump([[pythoncache, shellcache], PARSERCACHE_VERSION])
+
+    bb.utils.unlockfile(lf)
+    bb.utils.unlockfile(glf)
+
+def parser_cache_savemerge(d):
+    cachefile = parser_cachefile(d)
+    if not cachefile:
+        return
+
+    glf = bb.utils.lockfile(cachefile + ".lock")
 
     try:
         p = pickle.Unpickler(file(cachefile, "rb"))
@@ -78,17 +122,33 @@ def parser_cache_save(d):
     except (IOError, EOFError):
         data, version = None, None
 
-    if version == PARSERCACHE_VERSION:
-        for h in data[0]:
-            if h not in pythonparsecache:
-                pythonparsecache[h] = data[0][h]
-        for h in data[1]:
-            if h not in pythonparsecache:
-                shellparsecache[h] = data[1][h]
+    if version != PARSERCACHE_VERSION:
+        data = [{}, {}]
+
+    for f in [y for y in os.listdir(os.path.dirname(cachefile)) if y.startswith(os.path.basename(cachefile) + '-')]:
+        f = os.path.join(os.path.dirname(cachefile), f)
+        try:
+            p = pickle.Unpickler(file(f, "rb"))
+            extradata, version = p.load()
+        except (IOError, EOFError):
+            extradata, version = [{}, {}], None
+        
+        if version != PARSERCACHE_VERSION:
+            continue
+
+        for h in extradata[0]:
+            if h not in data[0]:
+                data[0][h] = extradata[0][h]
+        for h in extradata[1]:
+            if h not in data[1]:
+                data[1][h] = extradata[1][h]
+        os.unlink(f)
 
     p = pickle.Pickler(file(cachefile, "wb"), -1)
-    p.dump([[pythonparsecache, shellparsecache], PARSERCACHE_VERSION])
-    bb.utils.unlockfile(lf)
+    p.dump([data, PARSERCACHE_VERSION])
+
+    bb.utils.unlockfile(glf)
+
 
 class PythonParser():
     class ValueVisitor():

diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 43b36adcdebc187351ae1ac2fd196e6f6c762091..f00fdb2e991c79a8f4bd540fa6b50a413e0f7bbe 100644 (file)
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -1120,11 +1120,10 @@ class CookerParser(object):
             self.pool.terminate()
         self.pool.join()
 
-        bb.codeparser.parser_cache_save(self.cfgdata)
-
         sync = threading.Thread(target=self.bb_cache.sync)
         sync.start()
         multiprocessing.util.Finalize(None, sync.join, exitpriority=-100)
+        bb.codeparser.parser_cache_savemerge(self.cooker.configuration.data)
 
     def load_cached(self):
         for filename, appends in self.fromcache: