]> git.ipfire.org Git - thirdparty/openembedded/openembedded-core-contrib.git/commitdiff
bitbake: cooker/hashserv: Allow autostarting of a local hash server using BB_HASHSERVE
authorRichard Purdie <richard.purdie@linuxfoundation.org>
Wed, 24 Jul 2019 08:13:55 +0000 (09:13 +0100)
committerRichard Purdie <richard.purdie@linuxfoundation.org>
Tue, 6 Aug 2019 10:21:31 +0000 (11:21 +0100)
Its useful, particularly in the local developer model of usage, for
bitbake to start and stop a hash equivalence server on local port,
rather than relying on one being started by the user before the build.

The new BB_HASHSERVE variable supports this.

The database handling is moved internally into the hashserv code so that
different threads/processes can be used for the server without errors.

(Bitbake rev: a4fa8f1bd88995ae60e10430316fbed63d478587)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
bitbake/bin/bitbake-hashserv
bitbake/bin/bitbake-worker
bitbake/lib/bb/cooker.py
bitbake/lib/bb/runqueue.py
bitbake/lib/hashserv/__init__.py
bitbake/lib/hashserv/tests.py

index 848a0ca39f9d203da87b0bcc5cf1b5b5ddfcb9bc..6c911c098a73804d5c9fa00b2710f625cf642b43 100755 (executable)
@@ -41,9 +41,7 @@ def main():
     console.setLevel(level)
     logger.addHandler(console)
 
-    db = sqlite3.connect(args.database)
-
-    server = hashserv.create_server((args.address, args.port), db, args.prefix)
+    server = hashserv.create_server((args.address, args.port), args.database, args.prefix)
     server.serve_forever()
     return 0
 
index 3e502d5ca9f9172f4d140494980105653cd33288..96369199f233198e5bda52dca4b10f94c7429613 100755 (executable)
@@ -418,6 +418,7 @@ class BitbakeWorker(object):
         bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"]
         for mc in self.databuilder.mcdata:
             self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"])
+            self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.workerdata["hashservport"])
 
     def handle_newtaskhashes(self, data):
         self.workerdata["newhashes"] = pickle.loads(data)
index b4851e13ede6bcd99f0a25153384ade2af087e40..84f25ec94ebed5d2f14804a52474f04428872cc0 100644 (file)
@@ -31,6 +31,7 @@ import pyinotify
 import json
 import pickle
 import codecs
+import hashserv
 
 logger      = logging.getLogger("BitBake")
 collectlog  = logging.getLogger("BitBake.Collection")
@@ -230,6 +231,8 @@ class BBCooker:
         self.state = state.initial
 
         self.parser = None
+        self.hashserv = None
+        self.hashservport = None
 
         signal.signal(signal.SIGTERM, self.sigterm_exception)
         # Let SIGHUP exit as SIGTERM
@@ -368,6 +371,15 @@ class BBCooker:
 
         self.data.setVar('BB_CMDLINE', self.ui_cmdline)
 
+        if self.data.getVar("BB_HASHSERVE") == "localhost:0":
+            dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db"
+            self.hashserv = hashserv.create_server(('localhost', 0), dbfile, '')
+            self.hashservport = "localhost:" + str(self.hashserv.server_port)
+            thread = threading.Thread(target=self.hashserv.serve_forever)
+            thread.daemon = True
+            thread.start()
+            self.data.setVar("BB_HASHSERVE", self.hashservport)
+
         #
         # Copy of the data store which has been expanded.
         # Used for firing events and accessing variables where expansion needs to be accounted for
@@ -1645,9 +1657,10 @@ class BBCooker:
 
     def post_serve(self):
         prserv.serv.auto_shutdown()
+        if self.hashserv:
+            self.hashserv.shutdown()
         bb.event.fire(CookerExit(), self.data)
 
-
     def shutdown(self, force = False):
         if force:
             self.state = state.forceshutdown
index 05c1200da1030cd63bbca470b8408285815ff7ae..82cc9af81bce1312608288ffa123d9b7e4e6bcdd 100644 (file)
@@ -1254,6 +1254,7 @@ class RunQueue:
             "buildname" : self.cfgData.getVar("BUILDNAME"),
             "date" : self.cfgData.getVar("DATE"),
             "time" : self.cfgData.getVar("TIME"),
+            "hashservport" : self.cooker.hashservport,
         }
 
         worker.stdin.write(b"<cookerconfig>" + pickle.dumps(self.cooker.configuration) + b"</cookerconfig>")
index 768c5504cf2d0953ba60849f2157f806dc6866c6..7ec9b64419c0d2e8ae024d5e09202b992b28c646 100644 (file)
@@ -18,8 +18,15 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
     def log_message(self, f, *args):
         logger.debug(f, *args)
 
+    def opendb(self):
+        self.db = sqlite3.connect(self.dbname)
+        self.db.row_factory = sqlite3.Row
+
     def do_GET(self):
         try:
+            if not self.db:
+                self.opendb()
+
             p = urllib.parse.urlparse(self.path)
 
             if p.path != self.prefix + '/v1/equivalent':
@@ -52,6 +59,9 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
 
     def do_POST(self):
         try:
+            if not self.db:
+                self.opendb()
+
             p = urllib.parse.urlparse(self.path)
 
             if p.path != self.prefix + '/v1/equivalent':
@@ -123,14 +133,17 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
             self.send_error(400, explain=traceback.format_exc())
             return
 
-def create_server(addr, db, prefix=''):
+def create_server(addr, dbname, prefix=''):
     class Handler(HashEquivalenceServer):
         pass
 
-    Handler.prefix = prefix
-    Handler.db = db
+    db = sqlite3.connect(dbname)
     db.row_factory = sqlite3.Row
 
+    Handler.prefix = prefix
+    Handler.db = None
+    Handler.dbname = dbname
+
     with contextlib.closing(db.cursor()) as cursor:
         cursor.execute('''
             CREATE TABLE IF NOT EXISTS tasks_v2 (
index 8300a255990728396c8cce5edebda4ce55d3f55f..15f4a52ee961e175cf32cf7bbfc800bd9b7bd0fc 100644 (file)
@@ -11,14 +11,15 @@ import sqlite3
 import hashlib
 import urllib.request
 import json
+import tempfile
 from . import create_server
 
 class TestHashEquivalenceServer(unittest.TestCase):
     def setUp(self):
-        # Start an in memory hash equivalence server in the background bound to
+        # Start a hash equivalence server in the background bound to
         # an ephemeral port
-        db = sqlite3.connect(':memory:', check_same_thread=False)
-        self.server = create_server(('localhost', 0), db)
+        self.dbfile = tempfile.NamedTemporaryFile(prefix="bb-hashserv-db-")
+        self.server = create_server(('localhost', 0), self.dbfile.name)
         self.server_addr = 'http://localhost:%d' % self.server.socket.getsockname()[1]
         self.server_thread = threading.Thread(target=self.server.serve_forever)
         self.server_thread.start()