gdb/python: don't use PyObject_IsInstance in py-registers.c

In python/py-registers.c we make use of PyObject_IsInstance.  The
PyObject_IsInstance can return -1 for an error, 0 for false, or 1 for
true.

In py-registers.c we treat the return value from PyObject_IsInstance
as a boolean, which means both -1 and 1 will be treated as true.

If PyObject_IsInstance returns -1 for an error, this will be treated
as true, we will then invoke undefined behaviour as the pyo_reg_id
object will be treated as a gdb.RegisterDescriptor, even though it
might not be.

I noticed that the gdb.RegisterDescriptor class does not have the
Py_TPFLAGS_BASETYPE flag, and therefore cannot be inherited from.  As
such, using PyObject_IsInstance is not necessary, we can use
PyObject_TypeCheck instead.  The PyObject_TypeCheck function only
returns 0 or 1, so we don't need to worry about the error case.

Approved-By: Tom Tromey <tom@tromey.com>

diff --git a/gdb/python/py-registers.c b/gdb/python/py-registers.c
index 78a806c09629b1e2c71ec42282efbf395d524597..9be2e3ceaf526e407407081fbe85aa8637974d77 100644 (file)
--- a/gdb/python/py-registers.c
+++ b/gdb/python/py-registers.c
@@ -403,8 +403,7 @@ gdbpy_parse_register_id (struct gdbarch *gdbarch, PyObject *pyo_reg_id,
        PyErr_SetString (PyExc_ValueError, "Bad register");
     }
   /* The register could be a gdb.RegisterDescriptor object.  */
-  else if (PyObject_IsInstance (pyo_reg_id,
-                          (PyObject *) &register_descriptor_object_type))
+  else if (PyObject_TypeCheck (pyo_reg_id, &register_descriptor_object_type))
     {
       register_descriptor_object *reg
        = (register_descriptor_object *) pyo_reg_id;

diff --git a/gdb/testsuite/gdb.python/py-frame.exp b/gdb/testsuite/gdb.python/py-frame.exp
index 566880760c308268e94a812bd1ffe46775b90281..c1e3e33b1961bdd2df375cb89ad513560e5db2c3 100644 (file)
--- a/gdb/testsuite/gdb.python/py-frame.exp
+++ b/gdb/testsuite/gdb.python/py-frame.exp
@@ -188,6 +188,21 @@ gdb_test "python print(gdb.selected_frame().read_register(list()))" \
     ".*Invalid type for register.*" \
     "test Frame.read_register with list"
 
+gdb_test_multiline "setup a bad object" \
+    "python" "" \
+    "class bad_type:" "" \
+    "  def __init__ (self):" "" \
+    "    pass" "" \
+    "  @property" "" \
+    "  def __class__(self):" "" \
+    "    raise RuntimeError('error from __class in bad_type')" "" \
+    "bad_object = bad_type()" "" \
+    "end" ""
+
+gdb_test "python print(gdb.selected_frame().read_register(bad_object))" \
+    ".*Invalid type for register.*" \
+    "test Frame.read_register with bad_type object"
+
 # Compile again without debug info.
 gdb_exit
 if { [prepare_for_testing "failed to prepare" ${testfile} ${srcfile} {}] } {