ast/event/utils: Improve tracebacks to include file and line numbers more correctly

Currently bitbake tracebacks can have places where the line numbers are
inaccurate and filenames may be missing. These changes start to try and
correct this.

The only way I could find to correct line numbers was to compile as a
python ast, tweak the line numbers then compile to bytecode. I'm open
to better ways of doing this if anyone knows of any.

This does mean passing a few more parameters into functions, and putting
more data into the data store about functions (i.e. their filenames
and line numbers) but the improvement in debugging is more than worthwhile).

Before:
----------------
ERROR: Execution of event handler 'run_buildstats' failed
Traceback (most recent call last):
  File "run_buildstats(e)", line 43, in run_buildstats(e=<bb.build.TaskStarted object at 0x7f7b7c57a590>)
NameError: global name 'notexist' is not defined

ERROR: Build of do_patch failed
ERROR: Traceback (most recent call last):
  File "/media/build1/poky/bitbake/lib/bb/build.py", line 560, in exec_task
    return _exec_task(fn, task, d, quieterr)
  File "/media/build1/poky/bitbake/lib/bb/build.py", line 497, in _exec_task
    event.fire(TaskStarted(task, logfn, flags, localdata), localdata)
  File "/media/build1/poky/bitbake/lib/bb/event.py", line 170, in fire
    fire_class_handlers(event, d)
  File "/media/build1/poky/bitbake/lib/bb/event.py", line 109, in fire_class_handlers
    execute_handler(name, handler, event, d)
  File "/media/build1/poky/bitbake/lib/bb/event.py", line 81, in execute_handler
    ret = handler(event)
  File "run_buildstats(e)", line 43, in run_buildstats
NameError: global name 'notexist' is not defined

----------------

After:
----------------
ERROR: Execution of event handler 'run_buildstats' failed
Traceback (most recent call last):
  File "/media/build1/poky/meta/classes/buildstats.bbclass", line 143, in run_buildstats(e=<bb.build.TaskStarted object at 0x7efe89284e10>):
         if isinstance(e, bb.build.TaskStarted):
    >        trigger = notexist
             pn = d.getVar("PN", True)
NameError: global name 'notexist' is not defined

ERROR: Build of do_package failed
ERROR: Traceback (most recent call last):
  File "/media/build1/poky/bitbake/lib/bb/build.py", line 560, in exec_task
    return _exec_task(fn, task, d, quieterr)
  File "/media/build1/poky/bitbake/lib/bb/build.py", line 497, in _exec_task
    event.fire(TaskStarted(task, logfn, flags, localdata), localdata)
  File "/media/build1/poky/bitbake/lib/bb/event.py", line 170, in fire
    fire_class_handlers(event, d)
  File "/media/build1/poky/bitbake/lib/bb/event.py", line 109, in fire_class_handlers
    execute_handler(name, handler, event, d)
  File "/media/build1/poky/bitbake/lib/bb/event.py", line 81, in execute_handler
    ret = handler(event)
  File "/media/build1/poky/meta/classes/buildstats.bbclass", line 143, in run_buildstats
    trigger = notexist
NameError: global name 'notexist' is not defined
----------------

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

diff --git a/lib/bb/cookerdata.py b/lib/bb/cookerdata.py
index 671c0cb0e2df2505c416be960ca3f654552049dd..b47e7f3230c7899b751315b691f0c8bd472c7fdc 100644 (file)
--- a/lib/bb/cookerdata.py
+++ b/lib/bb/cookerdata.py
@@ -317,7 +317,9 @@ class CookerDataBuilder(object):
         # Nomally we only register event handlers at the end of parsing .bb files
         # We register any handlers we've found so far here...
         for var in data.getVar('__BBHANDLERS', False) or []:
-            bb.event.register(var, data.getVar(var, False),  (data.getVarFlag(var, "eventmask", True) or "").split())
+            handlerfn = data.getVarFlag(var, "filename", False)
+            handlerln = int(data.getVarFlag(var, "lineno", False))
+            bb.event.register(var, data.getVar(var, False),  (data.getVarFlag(var, "eventmask", True) or "").split(), handlerfn, handlerln)
 
         if data.getVar("BB_WORKERCONTEXT", False) is None:
             bb.fetch.fetcher_init(data)

diff --git a/lib/bb/event.py b/lib/bb/event.py
index ec25ce77fb4ea21873640ce59881be4fb78ff1e5..bd2b0a4b059eef9bd4174f97532f70ae0f845d6c 100644 (file)
--- a/lib/bb/event.py
+++ b/lib/bb/event.py
@@ -177,7 +177,7 @@ def fire_from_worker(event, d):
     fire_ui_handlers(event, d)
 
 noop = lambda _: None
-def register(name, handler, mask=None):
+def register(name, handler, mask=None, filename=None, lineno=None):
     """Register an Event handler"""
 
     # already registered
@@ -189,7 +189,13 @@ def register(name, handler, mask=None):
         if isinstance(handler, basestring):
             tmp = "def %s(e):\n%s" % (name, handler)
             try:
-                code = compile(tmp, "%s(e)" % name, "exec")
+                import ast
+                if filename is None:
+                    filename = "%s(e)" % name
+                code = compile(tmp, filename, "exec", ast.PyCF_ONLY_AST)
+                if lineno is not None:
+                    ast.increment_lineno(code, lineno-1)
+                code = compile(code, filename, "exec")
             except SyntaxError:
                 logger.error("Unable to register event handler '%s':\n%s", name,
                              ''.join(traceback.format_exc(limit=0)))

diff --git a/lib/bb/methodpool.py b/lib/bb/methodpool.py
index bf2e9f554216b42bb82810a7437bbaef4f7725ce..b2ea1a188704140dd13ee8366f8061dbdf4b5d70 100644 (file)
--- a/lib/bb/methodpool.py
+++ b/lib/bb/methodpool.py
@@ -19,11 +19,11 @@
 
 from bb.utils import better_compile, better_exec
 
-def insert_method(modulename, code, fn):
+def insert_method(modulename, code, fn, lineno):
     """
     Add code of a module should be added. The methods
     will be simply added, no checking will be done
     """
-    comp = better_compile(code, modulename, fn )
+    comp = better_compile(code, modulename, fn, lineno=lineno)
     better_exec(comp, None, code, fn)
 

diff --git a/lib/bb/parse/ast.py b/lib/bb/parse/ast.py
index 11db1801b31709aa17119814f5ad50d7587d0cc0..e1bf82fe90bc60e3679d2288c932c460072aebd6 100644 (file)
--- a/lib/bb/parse/ast.py
+++ b/lib/bb/parse/ast.py
@@ -148,17 +148,19 @@ class MethodNode(AstNode):
 
     def eval(self, data):
         text = '\n'.join(self.body)
+        funcname = self.func_name
         if self.func_name == "__anonymous":
             funcname = ("__anon_%s_%s" % (self.lineno, self.filename.translate(MethodNode.tr_tbl)))
             text = "def %s(d):\n" % (funcname) + text
-            bb.methodpool.insert_method(funcname, text, self.filename)
+            bb.methodpool.insert_method(funcname, text, self.filename, self.lineno - len(self.body))
             anonfuncs = data.getVar('__BBANONFUNCS', False) or []
             anonfuncs.append(funcname)
             data.setVar('__BBANONFUNCS', anonfuncs)
-            data.setVar(funcname, text, parsing=True)
         else:
             data.setVarFlag(self.func_name, "func", 1)
-            data.setVar(self.func_name, text, parsing=True)
+        data.setVar(funcname, text, parsing=True)
+        data.setVarFlag(funcname, 'filename', self.filename)
+        data.setVarFlag(funcname, 'lineno', str(self.lineno - len(self.body)))
 
 class PythonMethodNode(AstNode):
     def __init__(self, filename, lineno, function, modulename, body):
@@ -172,10 +174,12 @@ class PythonMethodNode(AstNode):
         # 'this' file. This means we will not parse methods from
         # bb classes twice
         text = '\n'.join(self.body)
-        bb.methodpool.insert_method(self.modulename, text, self.filename)
+        bb.methodpool.insert_method(self.modulename, text, self.filename, self.lineno - len(self.body) - 1)
         data.setVarFlag(self.function, "func", 1)
         data.setVarFlag(self.function, "python", 1)
         data.setVar(self.function, text, parsing=True)
+        data.setVarFlag(self.function, 'filename', self.filename)
+        data.setVarFlag(self.function, 'lineno', str(self.lineno - len(self.body) - 1))
 
 class MethodFlagsNode(AstNode):
     def __init__(self, filename, lineno, key, m):
@@ -317,7 +321,9 @@ def finalize(fn, d, variant = None):
     all_handlers = {}
     for var in d.getVar('__BBHANDLERS', False) or []:
         # try to add the handler
-        bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask", True) or "").split())
+        handlerfn = d.getVarFlag(var, "filename", False)
+        handlerln = int(d.getVarFlag(var, "lineno", False))
+        bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask", True) or "").split(), handlerfn, handlerln)
 
     bb.event.fire(bb.event.RecipePreFinalise(fn), d)
 

diff --git a/lib/bb/utils.py b/lib/bb/utils.py
index 31ec2b7c9af4b787c844b492245779df2860e760..c5ff903ceffbe6ebe79f5983274fb836372ed5db 100644 (file)
--- a/lib/bb/utils.py
+++ b/lib/bb/utils.py
@@ -33,6 +33,7 @@ import fnmatch
 import traceback
 import errno
 import signal
+import ast
 from commands import getstatusoutput
 from contextlib import contextmanager
 from ctypes import cdll
@@ -291,19 +292,22 @@ def _print_trace(body, line):
             error.append('     %.4d:%s' % (i, body[i-1].rstrip()))
     return error
 
-def better_compile(text, file, realfile, mode = "exec"):
+def better_compile(text, file, realfile, mode = "exec", lineno = None):
     """
     A better compile method. This method
     will print the offending lines.
     """
     try:
-        return compile(text, file, mode)
+        code = compile(text, realfile, mode, ast.PyCF_ONLY_AST)
+        if lineno is not None:
+            ast.increment_lineno(code, lineno)
+        return compile(code, realfile, mode)
     except Exception as e:
         error = []
         # split the text into lines again
         body = text.split('\n')
-        error.append("Error in compiling python function in %s:\n" % realfile)
-        if e.lineno:
+        error.append("Error in compiling python function in %s, line %s:\n" % (realfile, lineno))
+        if hasattr(e, "lineno"):
             error.append("The code lines resulting in this error were:")
             error.extend(_print_trace(body, e.lineno))
         else: