]> git.ipfire.org Git - thirdparty/openembedded/openembedded-core-contrib.git/commitdiff
bitbake: cooker,xmlrpc,servers: implement CookerFeatures
authorAlexandru DAMIAN <alexandru.damian@intel.com>
Wed, 18 Sep 2013 12:15:47 +0000 (13:15 +0100)
committerRichard Purdie <richard.purdie@linuxfoundation.org>
Wed, 18 Sep 2013 14:14:13 +0000 (15:14 +0100)
Implementing feature set selection that allows a client
to enable specific features in the server at connection time.

Only enabling of features is supported, as there is
no way to safely remove data loaded into the cooker.
Once enabled, a feature will remain enabled for the
life of the cooker.

Client-server connection now supports specifying the feature
set required by the client. This is implemented in the Process
server using a managed proxy list, so the server cooker
will now load dynamically needed features based on what client
connects to it.

In the XMLRPC server the feature set is requested by
using a parameter for registerUIHandler function.
This allows observer-only clients to also specify features
for the server.

The server code configuration now is completly separated
from the client code. All hardcoding of client knowledge is
removed from the server.

The extra_caches is removed as the client can now specify
the caches it needs using the feature. The UI modules
now need to specify the desired featureSet. HOB is modified
to conform to the featureSet specification.

The only feature available is CookerFeatures.HOB_EXTRA_CACHES
which forces loading the bb.cache_extra:HobRecipeInfo class.

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
bin/bitbake
lib/bb/cooker.py
lib/bb/cookerdata.py
lib/bb/server/__init__.py
lib/bb/server/process.py
lib/bb/server/xmlrpc.py
lib/bb/ui/hob.py
lib/bb/ui/knotty.py
lib/bb/ui/uievent.py

index 60c4b96c7018ccf472a496d90700a0be62b94988..b4506efb5281d12e9af184f51adb525f3c131923 100755 (executable)
@@ -75,18 +75,6 @@ def get_ui(config):
         sys.exit("FATAL: Invalid user interface '%s' specified.\n"
                  "Valid interfaces: depexp, goggle, ncurses, hob, knotty [default]." % interface)
 
-def gather_extra_cache_data():
-    extra = []
-    interfaces = ['depexp', 'goggle', 'ncurses', 'hob', 'knotty']
-    for i in interfaces:
-        try:
-            ui = __import__("bb.ui." + i, fromlist = [i])
-            if hasattr(ui, "extraCaches"):
-                extra = extra + ui.extraCaches
-            del ui
-        except:
-            pass
-    return extra
 
 # Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others"""
 warnlog = logging.getLogger("BitBake.Warnings")
@@ -302,25 +290,22 @@ def main():
     # Clear away any spurious environment variables while we stoke up the cooker
     cleanedvars = bb.utils.clean_environment()
 
-    # Collect all the caches we need
-    if configParams.server_only:
-        configuration.extra_caches = gather_extra_cache_data()
-    else:
-        configuration.extra_caches = getattr(ui_module, "extraCaches", [])
-
     if not configParams.remote_server:
         # we start a server with a given configuration
         server = start_server(servermodule, configParams, configuration)
     else:
-        # we start a stub server that is actually a XMLRPClient to
+        # we start a stub server that is actually a XMLRPClient that connects to a real server
         server = servermodule.BitBakeXMLRPCClient(configParams.observe_only)
         server.saveConnectionDetails(configParams.remote_server)
 
     logger.removeHandler(handler)
 
     if not configParams.server_only:
+        # Collect the feature set for the UI
+        featureset = getattr(ui_module, "featureSet", [])
+
         # Setup a connection to the server (cooker)
-        server_connection = server.establishConnection()
+        server_connection = server.establishConnection(featureset)
 
         # Restore the environment in case the UI needs it
         for k in cleanedvars:
index 1d5e5f66c2d4f331276e63fce19dc26ce038b127..8776e187eaca26ccd52ecb255c8a790a8a241d2b 100644 (file)
@@ -79,6 +79,29 @@ class SkippedPackage:
         elif reason:
             self.skipreason = reason
 
+
+class CookerFeatures(object):
+    _feature_list = [HOB_EXTRA_CACHES] = range(1)
+
+    def __init__(self):
+        self._features=set()
+
+    def setFeature(self, f):
+        # validate we got a request for a feature we support
+        if f not in CookerFeatures._feature_list:
+            return
+        self._features.add(f)
+
+    def __contains__(self, f):
+        return f in self._features
+
+    def __iter__(self):
+        return self._features.__iter__()
+
+    def next(self):
+        return self._features.next()
+
+
 #============================================================================#
 # BBCooker
 #============================================================================#
@@ -90,6 +113,7 @@ class BBCooker:
     def __init__(self, configuration):
         self.recipecache = None
         self.skiplist = {}
+        self.featureset = CookerFeatures()
 
         self.configuration = configuration
 
@@ -122,7 +146,13 @@ class BBCooker:
         self.state = state.initial
 
         self.caches_array = []
-        caches_name_array = ['bb.cache:CoreRecipeInfo'] + self.configuration.extra_caches
+
+        all_extra_cache_names = []
+        # We hardcode all known cache types in a single place, here.
+        if CookerFeatures.HOB_EXTRA_CACHES in self.featureset:
+            all_extra_cache_names.append("bb.cache_extra:HobRecipeInfo")
+
+        caches_name_array = ['bb.cache:CoreRecipeInfo'] + all_extra_cache_names
 
         # At least CoreRecipeInfo will be loaded, so caches_array will never be empty!
         # This is the entry point, no further check needed!
@@ -130,7 +160,7 @@ class BBCooker:
             try:
                 module_name, cache_name = var.split(':')
                 module = __import__(module_name, fromlist=(cache_name,))
-                self.caches_array.append(getattr(module, cache_name)) 
+                self.caches_array.append(getattr(module, cache_name))
             except ImportError as exc:
                 logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc))
                 sys.exit("FATAL: Failed to import extra cache class '%s'." % cache_name)
index 0b278b178efd754c12626fa46f4a727700758c70..e640ed0f35e40f2b9280c6e4c0ec747a8e90ad8b 100644 (file)
@@ -127,7 +127,6 @@ class CookerConfiguration(object):
         self.dump_signatures = False
         self.dry_run = False
         self.tracking = False
-        self.extra_caches = []
 
         self.env = {}
 
index 2e1c619b54f935e39401d5db2fa9280171821c88..da5e48074014472b39a1cf92c085cd711f0545f6 100644 (file)
@@ -89,7 +89,7 @@ class BitBakeBaseServer(object):
     def detach(self):
         return
 
-    def establishConnection(self):
+    def establishConnection(self, featureset):
         raise   "Must redefine the %s.establishConnection()" % self.__class__.__name__
 
     def endSession(self):
index e45e0c2f6dd3b7114160e28db9c7d802b231fc38..aa072020af2349a6e16dbb1191ec122ac4d04d10 100644 (file)
@@ -31,7 +31,7 @@ import sys
 import time
 import select
 from Queue import Empty
-from multiprocessing import Event, Process, util, Queue, Pipe, queues
+from multiprocessing import Event, Process, util, Queue, Pipe, queues, Manager
 
 from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer
 
@@ -78,12 +78,13 @@ class ProcessServer(Process, BaseImplServer):
     profile_filename = "profile.log"
     profile_processed_filename = "profile.log.processed"
 
-    def __init__(self, command_channel, event_queue):
+    def __init__(self, command_channel, event_queue, featurelist):
         BaseImplServer.__init__(self)
-        Process.__init__(self)
+        Process.__init__(self, args=(featurelist))
         self.command_channel = command_channel
         self.event_queue = event_queue
         self.event = EventAdapter(event_queue)
+        self.featurelist = featurelist
         self.quit = False
 
         self.keep_running = Event()
@@ -94,6 +95,14 @@ class ProcessServer(Process, BaseImplServer):
         for event in bb.event.ui_queue:
             self.event_queue.put(event)
         self.event_handle.value = bb.event.register_UIHhandler(self)
+
+        # process any feature changes based on what UI requested
+        original_featureset = list(self.cooker.featureset)
+        while len(self.featurelist)> 0:
+            self.cooker.featureset.setFeature(self.featurelist.pop())
+        if (original_featureset != list(self.cooker.featureset)):
+            self.cooker.reset()
+
         bb.cooker.server_main(self.cooker, self.main)
 
     def main(self):
@@ -198,13 +207,17 @@ class BitBakeServer(BitBakeBaseServer):
         #
         self.ui_channel, self.server_channel = Pipe()
         self.event_queue = ProcessEventQueue(0)
-        self.serverImpl = ProcessServer(self.server_channel, self.event_queue)
+        manager = Manager()
+        self.featurelist = manager.list()
+        self.serverImpl = ProcessServer(self.server_channel, self.event_queue, self.featurelist)
 
     def detach(self):
         self.serverImpl.start()
         return
 
-    def establishConnection(self):
+    def establishConnection(self, featureset):
+        for f in featureset:
+            self.featurelist.append(f)
         self.connection = BitBakeProcessServerConnection(self.serverImpl, self.ui_channel, self.event_queue)
         signal.signal(signal.SIGTERM, lambda i, s: self.connection.terminate())
         return self.connection
index 8326623520cab4df6892d56386b114ff925af2ab..389327a60f4bb9af210433906273dc0b30179926 100644 (file)
@@ -89,12 +89,23 @@ class BitBakeServerCommands():
         self.server = server
         self.has_client = False
 
-    def registerEventHandler(self, host, port):
+    def registerEventHandler(self, host, port, featureset = []):
         """
         Register a remote UI Event Handler
         """
         s, t = _create_server(host, port)
 
+        # we don't allow connections if the cooker is running
+        if (self.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]):
+            return None
+
+        original_featureset = list(self.cooker.featureset)
+        for f in featureset:
+            self.cooker.featureset.setFeature(f)
+
+        if (original_featureset != list(self.cooker.featureset)):
+            self.cooker.reset()
+
         self.event_handle = bb.event.register_UIHhandler(s)
         return self.event_handle
 
@@ -263,11 +274,12 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
         self.connection_token = token
 
 class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
-    def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False):
+    def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False, featureset = []):
         self.connection, self.transport = _create_server(serverImpl.host, serverImpl.port)
         self.clientinfo = clientinfo
         self.serverImpl = serverImpl
         self.observer_only = observer_only
+        self.featureset = featureset
 
     def connect(self):
         if not self.observer_only:
@@ -277,7 +289,8 @@ class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
         if token is None:
             return None
         self.transport.set_connection_token(token)
-        self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo)
+
+        self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo, self.featureset)
         for event in bb.event.ui_queue:
             self.events.queue_event(event)
         return self
@@ -301,14 +314,15 @@ class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
 
 class BitBakeServer(BitBakeBaseServer):
     def initServer(self, interface = ("localhost", 0)):
+        self.interface = interface
         self.serverImpl = XMLRPCServer(interface)
 
     def detach(self):
         daemonize.createDaemon(self.serverImpl.serve_forever, "bitbake-cookerdaemon.log")
         del self.cooker
 
-    def establishConnection(self):
-        self.connection = BitBakeXMLRPCServerConnection(self.serverImpl)
+    def establishConnection(self, featureset):
+        self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, self.interface, False, featureset)
         return self.connection.connect()
 
     def set_connection_token(self, token):
@@ -318,12 +332,13 @@ class BitBakeXMLRPCClient(BitBakeBaseServer):
 
     def __init__(self, observer_only = False):
         self.observer_only = observer_only
+        # if we need extra caches, just tell the server to load them all
         pass
 
     def saveConnectionDetails(self, remote):
         self.remote = remote
 
-    def establishConnection(self):
+    def establishConnection(self, featureset):
         # The format of "remote" must be "server:port"
         try:
             [host, port] = self.remote.split(":")
@@ -340,7 +355,7 @@ class BitBakeXMLRPCClient(BitBakeBaseServer):
         except:
             return None
         self.serverImpl = XMLRPCProxyServer(host, port)
-        self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only)
+        self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only, featureset)
         return self.connection.connect()
 
     def endSession(self):
index bc5ee1434a61a854ceca085bf270c06960f9b15d..154a3b3b4afc93c6aa30dbcf16eda1083c494889 100755 (executable)
@@ -46,7 +46,7 @@ from bb.ui.crumbs.hoblistmodel import RecipeListModel, PackageListModel
 from bb.ui.crumbs.hobeventhandler import HobHandler
 from bb.ui.crumbs.builder import Builder
 
-extraCaches = ['bb.cache_extra:HobRecipeInfo']
+featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES]
 
 def event_handle_idle_func(eventHandler, hobHandler):
     # Consume as many messages as we can in the time available to us
index 35590a2a7947096636f6df92d814d190952920f0..0211b50abaa20d3feeca8d19a9c601c2df42ae65 100644 (file)
@@ -472,6 +472,9 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                              event.taskid, event.taskstring, event.exitcode)
                 continue
 
+            if isinstance(event, bb.event.DepTreeGenerated):
+                continue
+
             # ignore
             if isinstance(event, (bb.event.BuildBase,
                                   bb.event.StampUpdate,
index 038029fcfa95c551073387e57e95d30eacc6ef5c..2133b44477b55571b43d75a0a4fbc5416448d611 100644 (file)
@@ -28,7 +28,7 @@ import socket, threading, pickle
 from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
 
 class BBUIEventQueue:
-    def __init__(self, BBServer, clientinfo=("localhost, 0")):
+    def __init__(self, BBServer, clientinfo=("localhost, 0"), featureset=[]):
 
         self.eventQueue = []
         self.eventQueueLock = threading.Lock()
@@ -44,7 +44,10 @@ class BBUIEventQueue:
         server.register_function( self.send_event, "event.sendpickle" )
         server.socket.settimeout(1)
 
-        self.EventHandle = self.BBServer.registerEventHandler(self.host, self.port)
+        self.EventHandle = self.BBServer.registerEventHandler(self.host, self.port, featureset)
+
+        if (self.EventHandle == None):
+            bb.fatal("Could not register UI event handler")
 
         self.server = server