fix branch prediction for real

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index c30e314e653e82d99ed00d16072f06d7b9546e6c..d8e987708eba7b807aaab1229287c2b494f1d207 100644 (file)
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -3059,7 +3059,6 @@ dummy_func(
             assert(PyStackRef_BoolCheck(cond));
             int flag = PyStackRef_IsFalse(cond);
             DEAD(cond);
-            RECORD_JUMP_TAKEN();
             JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN);
         }
 
@@ -3067,7 +3066,6 @@ dummy_func(
             assert(PyStackRef_BoolCheck(cond));
             int flag = PyStackRef_IsTrue(cond);
             DEAD(cond);
-            RECORD_JUMP_TAKEN();
             JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN);
         }
 

diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 83f5fde87e6a88d2a450f4e73b130b0a0c3ee9b4..1633559d72ac12ce237e67aff426b67670cb6ccd 100644 (file)
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -24799,7 +24799,6 @@
             cond = stack_pointer[-1];
             assert(PyStackRef_BoolCheck(cond));
             int flag = PyStackRef_IsFalse(cond);
-            RECORD_JUMP_TAKEN();
             JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN);
             stack_pointer += -1;
             assert(WITHIN_STACK_BOUNDS());
@@ -24848,7 +24847,6 @@
                 cond = b;
                 assert(PyStackRef_BoolCheck(cond));
                 int flag = PyStackRef_IsTrue(cond);
-                RECORD_JUMP_TAKEN();
                 JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN);
             }
             stack_pointer += -1;
@@ -24898,7 +24896,6 @@
                 cond = b;
                 assert(PyStackRef_BoolCheck(cond));
                 int flag = PyStackRef_IsFalse(cond);
-                RECORD_JUMP_TAKEN();
                 JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN);
             }
             stack_pointer += -1;
@@ -24928,7 +24925,6 @@
             cond = stack_pointer[-1];
             assert(PyStackRef_BoolCheck(cond));
             int flag = PyStackRef_IsTrue(cond);
-            RECORD_JUMP_TAKEN();
             JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN);
             stack_pointer += -1;
             assert(WITHIN_STACK_BOUNDS());

diff --git a/Python/optimizer.c b/Python/optimizer.c
index b4d3429f7321fc46fd4339e1c569648a0f06033f..c8a0a3cb74ed58d3d1deff3fa5e4e5a9074761a8 100644 (file)
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -617,7 +617,8 @@ _PyJIT_translate_single_bytecode_to_trace(
     const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode];
 
     // Strange control-flow, unsupported opcode, etc.
-    if (jump_taken || opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) {
+    if (jump_taken ||
+        opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) {
     unsupported:
         // Rewind to previous instruction and replace with _EXIT_TRACE.
         _PyUOpInstruction *curr = &trace[trace_length-1];
@@ -671,11 +672,11 @@ _PyJIT_translate_single_bytecode_to_trace(
         case POP_JUMP_IF_TRUE:
         {
             RESERVE(1);
-            int jump_likely = jump_taken;
+            _Py_CODEUNIT *computed_next_instr = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]];
+            _Py_CODEUNIT *computed_jump_instr = computed_next_instr + oparg;
+            int jump_likely = computed_jump_instr == next_instr;
             uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely];
-            _Py_CODEUNIT *next_instr = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]];
-            _Py_CODEUNIT *false_target = next_instr + oparg;
-            ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(false_target, old_code));
+            ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_likely ? computed_next_instr : computed_jump_instr, old_code));
             break;
         }
         case JUMP_BACKWARD_JIT: