]> git.ipfire.org Git - thirdparty/openembedded/openembedded-core-contrib.git/commitdiff
toaster: improve logging facilities for toaster
authorAlexandru DAMIAN <alexandru.damian@intel.com>
Thu, 5 Feb 2015 13:18:06 +0000 (13:18 +0000)
committerAlexandru DAMIAN <alexandru.damian@intel.com>
Mon, 9 Feb 2015 17:54:34 +0000 (17:54 +0000)
This patch improves the logging facilities for toaster in order
to help diagnose bugs that happen on user machines.

The logs are stored now under "/tmp/toaster_$$" where $$ is a
PID-based unique identifier. On shutdown, toaster will automatically
erase all logs unless errors are listed in the log file.

On error, Toaster provides suggestions on what to do.

This patch includes a minor fix found as a result of logging
improvements.

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
bin/toaster
lib/bb/ui/buildinfohelper.py
lib/toaster/bldcontrol/management/commands/runbuilds.py
lib/toaster/toastergui/views.py
lib/toaster/toastermain/settings.py
lib/toaster/toastermain/urls.py

index c3b8633c2582f7629753f82e6fb726bda3558e87..04f2e9937801e75c958704a4ac36f129c5180962 100755 (executable)
@@ -69,7 +69,7 @@ function webserverStartAll()
        fi
         if [ $retval -eq 0 ]; then
            echo "Starting webserver..."
-            python $BBBASEDIR/lib/toaster/manage.py runserver "0.0.0.0:$WEB_PORT" </dev/null >${BUILDDIR}/toaster_web_$$.log 2>&1 & echo $! >${BUILDDIR}/.toastermain.pid
+            python $BBBASEDIR/lib/toaster/manage.py runserver "0.0.0.0:$WEB_PORT" </dev/null >>${BUILDDIR}/toaster_web.log 2>&1 & echo $! >${BUILDDIR}/.toastermain.pid
             sleep 1
             if ! cat "${BUILDDIR}/.toastermain.pid" | xargs -I{} kill -0 {} ; then
                 retval=1
@@ -189,17 +189,43 @@ if [ -z "$ZSH_NAME" ] && [ `basename \"$0\"` = `basename \"$BASH_SOURCE\"` ]; th
     fi
 
     # Define a fake builddir where only the pid files are actually created. No real builds will take place here.
-    BUILDDIR=/tmp
+    BUILDDIR=/tmp/toaster_$$
+    if [ -d "$BUILDDIR" ]; then
+        echo -e "Previous toaster run directory $BUILDDIR found, cowardly refusing to start. Please remove the directory when that toaster instance is over" 2>&1
+        exit 1;
+    fi
+
+    mkdir -p "$BUILDDIR"
+
     RUNNING=1
     function trap_ctrlc() {
         echo "** Stopping system"
         webserverKillAll
         RUNNING=0
     }
+
+    function do_cleanup() {
+        find "$BUILDDIR" -type f | xargs rm
+        rmdir "$BUILDDIR"
+    }
+    function cleanup() {
+        if grep -ir error "$BUILDDIR" >/dev/null; then
+            if grep -irn "That port is already in use" "$BUILDDIR"; then
+                echo "You can use the \"webport=PORTNUMBER\" parameter to start Toaster on a different port (port $WEB_PORT is already in use)"
+                do_cleanup
+            else
+                echo -e "\nErrors found in the Toaster log files present in '$BUILDDIR'. Directory will not be cleaned.\n Please review the errors and notify toaster@yoctoproject.org or submit a bug https://bugzilla.yoctoproject.org/enter_bug.cgi?product=Toaster"
+            fi
+        else
+            echo "No errors found, removing the run directory '$BUILDDIR'"
+            do_cleanup
+        fi;
+    }
     TOASTER_MANAGED=1
     export TOASTER_MANAGED=1
     if [ $WEBSERVER -gt 0 ] && ! webserverStartAll; then
         echo "Failed to start the web server, stopping" 1>&2;
+        cleanup
         exit 1;
     fi
     if [ $WEBSERVER -gt 0 ]; then
@@ -209,9 +235,10 @@ if [ -z "$ZSH_NAME" ] && [ `basename \"$0\"` = `basename \"$BASH_SOURCE\"` ]; th
     trap trap_ctrlc SIGINT
     echo "Toaster is now running. You can stop it with Ctrl-C"
     while [ $RUNNING -gt 0 ]; do
-       python $BBBASEDIR/lib/toaster/manage.py runbuilds
-       sleep 1
+        python $BBBASEDIR/lib/toaster/manage.py runbuilds 2>&1 | tee -a "$BUILDDIR/toaster.log"
+        sleep 1
     done
+    cleanup
     echo "**** Exit"
     exit 0
 fi
@@ -275,8 +302,8 @@ case $CMD in
         fi
         unset BBSERVER
         PREREAD=""
-        if [ -e conf/toaster-pre.conf ]; then
-                PREREAD="--read conf/toaster-pre.conf"
+        if [ -e ${BUILDDIR}/conf/toaster-pre.conf ]; then
+               rm ${BUILDDIR}/conf/toaster-pre.conf
         fi
         bitbake $PREREAD --postread conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0
         if [ $? -ne 0 ]; then
@@ -285,7 +312,7 @@ case $CMD in
         else
             export BBSERVER=0.0.0.0:-1
             if [ $NOTOASTERUI == 0 ]; then        # we start the TOASTERUI only if not inhibited
-                bitbake --observe-only -u toasterui >${BUILDDIR}/toaster_ui.log 2>&1 & echo $! >${BUILDDIR}/.toasterui.pid
+                bitbake --observe-only -u toasterui >>${BUILDDIR}/toaster_ui.log 2>&1 & echo $! >${BUILDDIR}/.toasterui.pid
             fi
         fi
         if [ $start_success -eq 1 ]; then
index 4e2d4a7dec5ab02b4de4c4be0e8c6786fa726c3e..491fd1566db4b74b2a887ae891205e7eacace4ff 100644 (file)
@@ -656,18 +656,41 @@ class BuildInfoHelper(object):
         assert path.startswith("/")
         assert 'build' in self.internal_state
 
-        def _slkey(layer_version):
-            assert isinstance(layer_version, Layer_Version)
-            return len(layer_version.layer.local_path)
-
-        # Heuristics: we always match recipe to the deepest layer path that
-        # we can match to the recipe file path
-        for bl in sorted(self.orm_wrapper.layer_version_objects, reverse=True, key=_slkey):
-            if (path.startswith(bl.layer.local_path)):
-                return bl
-
-        #if we get here, we didn't read layers correctly; mockup the new layer
-        unknown_layer, created = Layer.objects.get_or_create(name="unknown", local_path="/", layer_index_url="")
+        if self.brbe is None:
+            def _slkey_interactive(layer_version):
+                assert isinstance(layer_version, Layer_Version)
+                return len(layer_version.layer.local_path)
+
+            # Heuristics: we always match recipe to the deepest layer path in the discovered layers
+            for lvo in sorted(self.orm_wrapper.layer_version_objects, reverse=True, key=_slkey_interactive):
+                # we can match to the recipe file path
+                if path.startswith(lvo.layer.local_path):
+                    return lvo
+
+        else:
+            br_id, be_id = self.brbe.split(":")
+            from bldcontrol.bbcontroller import getBuildEnvironmentController
+            from bldcontrol.models import BuildRequest
+            bc = getBuildEnvironmentController(pk = be_id)
+
+            def _slkey_managed(layer_version):
+                return len(bc.getGitCloneDirectory(layer_version.giturl, layer_version.commit) + layer_version.dirpath)
+
+            # Heuristics: we match the path to where the layers have been checked out
+            for brl in sorted(BuildRequest.objects.get(pk = br_id).brlayer_set.all(), reverse = True, key = _slkey_managed):
+                localdirname = os.path.join(os.path.join(bc.be.sourcedir, bc.getGitCloneDirectory(brl.giturl, brl.commit)), brl.dirpath)
+                if path.startswith(localdirname):
+                    #logger.warn("-- managed: matched path %s with layer %s " % (path, localdirname))
+                    # we matched the BRLayer, but we need the layer_version that generated this br
+                    for lvo in self.orm_wrapper.layer_version_objects:
+                        if brl.name == lvo.layer.name:
+                            return lvo
+
+        #if we get here, we didn't read layers correctly; dump whatever information we have on the error log
+        logger.error("Could not match layer version for recipe path %s : %s" % (path, self.orm_wrapper.layer_version_objects))
+
+        #mockup the new layer
+        unknown_layer, created = Layer.objects.get_or_create(name="__FIXME__unidentified_layer", local_path="/", layer_index_url="")
         unknown_layer_version_obj, created = Layer_Version.objects.get_or_create(layer = unknown_layer, build = self.internal_state['build'])
 
         return unknown_layer_version_obj
index 3b539b591aefea27cbc94c309f3306c163534d23..c4ab87bdc98b293bc72b6f99d7bfa83afb03846c 100644 (file)
@@ -68,7 +68,7 @@ class Command(NoArgsCommand):
                 task = None
             bbctrl.build(list(map(lambda x:x.target, br.brtarget_set.all())), task)
 
-            logger.debug("runbuilds: Build launched, exiting")
+            logger.debug("runbuilds: Build launched, exiting. Follow build logs at %s/toaster_ui.log" % bec.be.builddir)
             # disconnect from the server
             bbctrl.disconnect()
 
index 7353844bf1fc851cb82a147c4bfdfa8a1b6e8163..b67a6767b3ec2a247c0bee0c8686132ef2165de7 100755 (executable)
@@ -3248,7 +3248,7 @@ else:
     def xhr_build(request, pid):
         raise Exception("page not available in interactive mode")
 
-    def xhr_projectinfo(request, pid):
+    def xhr_projectinfo(request):
         raise Exception("page not available in interactive mode")
 
     def xhr_projectedit(request, pid):
index 7cf905266d7f2b3e7484a1899180a04fc7f1f538..ea7c3534da5427fcfde8c65b1434e35530e7bd56 100644 (file)
@@ -344,7 +344,7 @@ LOGGING = {
     },
     'formatters': {
         'datetime': {
-            'format': '%(levelname)s %(asctime)s %(message)s'
+            'format': '%(asctime)s %(levelname)s %(message)s'
         }
     },
     'handlers': {
@@ -365,8 +365,8 @@ LOGGING = {
             'level': 'DEBUG',
         },
         'django.request': {
-            'handlers': ['mail_admins'],
-            'level': 'ERROR',
+            'handlers': ['console'],
+            'level': 'WARN',
             'propagate': True,
         },
     }
index 61120675796f2754e626626ec04b0ade7cb17984..f66f11dcde59667cfaa53d05cae2d85d09de95b9 100644 (file)
@@ -23,6 +23,9 @@ from django.conf.urls import patterns, include, url
 from django.views.generic import RedirectView
 from django.views.decorators.cache import never_cache
 
+import logging
+
+logger = logging.getLogger("toaster")
 
 # Uncomment the next two lines to enable the admin:
 from django.contrib import admin
@@ -47,10 +50,12 @@ import toastermain.settings
 
 if toastermain.settings.FRESH_ENABLED:
     urlpatterns.insert(1, url(r'', include('fresh.urls')))
+    logger.info("Enabled django-fresh extension")
 
 if toastermain.settings.DEBUG_PANEL_ENABLED:
     import debug_toolbar
     urlpatterns.insert(1, url(r'', include(debug_toolbar.urls)))
+    logger.info("Enabled django_toolbar extension")
 
 
 if toastermain.settings.MANAGED:
@@ -70,4 +75,15 @@ for t in os.walk(os.path.dirname(currentdir)):
 
     if "urls.py" in t[2] and t[0] != currentdir:
         modulename = os.path.basename(t[0])
-        urlpatterns.insert(0, url(r'^' + modulename + '/', include ( modulename + '.urls')))
+        # make sure we don't have this module name in
+        conflict = False
+        for p in urlpatterns:
+            if p.regex.pattern == '^' + modulename + '/':
+                conflict = True
+        if not conflict:
+            urlpatterns.insert(0, url(r'^' + modulename + '/', include ( modulename + '.urls')))
+        else:
+            logger.warn("Module \'%s\' has a regexp conflict, was not added to the urlpatterns" % modulename)
+
+from pprint import pformat
+logger.debug("urlpatterns list %s", pformat(urlpatterns))