build.py: Operate logfile handling at the task level

Previously, not all task output could end up in the logfile and
sub functions run using exec_func() calls would end up logging to
/dev/null. This is extrememly confusing for users who want to find
the output from a given task as there are several possible log files,
or it might not be there at all.

This patch changes exec_task() to redirect standard output/error
to the logfile along with any python logging output. This change is
applied for all python/shell tasks so it doesn't matter whether output
comes from os.system() calls or wherever, the logs are always correct.

This also removed the ton of log files previous code could generate
in ${T}, again making this simpler for users.

Based upon patches from Poky.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

diff --git a/lib/bb/build.py b/lib/bb/build.py
index 50058faeea5eeae1ae57d89ffdfde78d27598f98..4d8a2688cdb5c988ab6143e7b04e27151caeb917 100644 (file)
--- a/lib/bb/build.py
+++ b/lib/bb/build.py
@@ -122,7 +122,7 @@ class LogTee(object):
         return '<LogTee {0}>'.format(self.name)
 
 
-def exec_func(func, d, dirs = None, logfile = NULL):
+def exec_func(func, d, dirs = None):
     """Execute an BB 'function'"""
 
     body = data.getVar(func, d)
@@ -165,9 +165,9 @@ def exec_func(func, d, dirs = None, logfile = NULL):
 
     with bb.utils.fileslocked(lockfiles):
         if ispython:
-            exec_func_python(func, d, runfile, logfile, cwd=adir)
+            exec_func_python(func, d, runfile, cwd=adir)
         else:
-            exec_func_shell(func, d, runfile, logfile, cwd=adir, fakeroot=fakeroot)
+            exec_func_shell(func, d, runfile, cwd=adir, fakeroot=fakeroot)
 
 _functionfmt = """
 def {function}(d):
@@ -176,7 +176,7 @@ def {function}(d):
 {function}(d)
 """
 logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
-def exec_func_python(func, d, runfile, logfile, cwd=None):
+def exec_func_python(func, d, runfile, cwd=None):
     """Execute a python BB 'function'"""
 
     bbfile = d.getVar('FILE', True)
@@ -189,10 +189,6 @@ def exec_func_python(func, d, runfile, logfile, cwd=None):
     if cwd:
         os.chdir(cwd)
 
-    handler = logging.StreamHandler(logfile)
-    handler.setFormatter(logformatter)
-    bblogger.addHandler(handler)
-
     try:
         comp = utils.better_compile(code, func, bbfile)
         utils.better_exec(comp, {"d": d}, code, bbfile)
@@ -202,10 +198,9 @@ def exec_func_python(func, d, runfile, logfile, cwd=None):
 
         raise FuncFailed(func, None)
     finally:
-        bblogger.removeHandler(handler)
         os.chdir(olddir)
 
-def exec_func_shell(function, d, runfile, logfile, cwd=None, fakeroot=False):
+def exec_func_shell(function, d, runfile, cwd=None, fakeroot=False):
     """Execute a shell function from the metadata
 
     Note on directory behavior.  The 'dirs' varflag should contain a list
@@ -230,12 +225,15 @@ def exec_func_shell(function, d, runfile, logfile, cwd=None, fakeroot=False):
         cmd = runfile
 
     if logger.isEnabledFor(logging.DEBUG):
-        logfile = LogTee(logger, logfile)
+        logfile = LogTee(logger, sys.stdout)
+    else:
+        logfile = sys.stdout
 
     try:
         bb.process.run(cmd, cwd=cwd, shell=False, stdin=NULL, log=logfile)
     except bb.process.CmdError:
-        raise FuncFailed(function, logfile.name)
+        logfn = d.getVar('BB_LOGFILE', True)
+        raise FuncFailed(function, logfn)
 
 def _task_data(fn, task, d):
     localdata = data.createCopy(d)
@@ -279,19 +277,59 @@ def _exec_task(fn, task, d):
     prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True)
     postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True)
 
-    logfile = open(logfn, 'w')
+    # Handle logfiles
+    si = file('/dev/null', 'r')
+    try:
+        logfile = file(logfn, 'w')
+    except OSError:
+        logger.exception("Opening log file '%s'", logfn)
+        pass
+
+    # Dup the existing fds so we dont lose them
+    osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
+    oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
+    ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
+
+    # Replace those fds with our own
+    os.dup2(si.fileno(), osi[1])
+    os.dup2(logfile.fileno(), oso[1])
+    os.dup2(logfile.fileno(), ose[1])
+
+    # Ensure python logging goes to the logfile
+    handler = logging.StreamHandler(logfile)
+    handler.setFormatter(logformatter)
+    bblogger.addHandler(handler)
+
+    localdata.setVar('BB_LOGFILE', logfn)
+
     event.fire(TaskStarted(task, localdata), localdata)
     try:
         for func in (prefuncs or '').split():
-            exec_func(func, localdata, logfile=logfile)
-        exec_func(task, localdata, logfile=logfile)
+            exec_func(func, localdata)
+        exec_func(task, localdata)
         for func in (postfuncs or '').split():
-            exec_func(func, localdata, logfile=logfile)
+            exec_func(func, localdata)
     except FuncFailed as exc:
         logger.error(str(exc))
         event.fire(TaskFailed(task, logfn, localdata), localdata)
         return 1
     finally:
+        sys.stdout.flush()
+        sys.stderr.flush()
+
+        bblogger.removeHandler(handler)
+
+        # Restore the backup fds
+        os.dup2(osi[0], osi[1])
+        os.dup2(oso[0], oso[1])
+        os.dup2(ose[0], ose[1])
+
+        # Close the backup fds
+        os.close(osi[0])
+        os.close(oso[0])
+        os.close(ose[0])
+        si.close()
+
         logfile.close()
         if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
             logger.debug(2, "Zero size logfn %s, removing", logfn)