Bump table allocations retroactively if they grow later on.

diff --git a/src/lj_jit.h b/src/lj_jit.h
index 4b51baeb614cb0ecb2476f6ff18b5ef61141a52f..db3d89bb1867a7f99ab39173f4e572e6b58eeeb4 100644 (file)
--- a/src/lj_jit.h
+++ b/src/lj_jit.h
@@ -290,6 +290,15 @@ typedef struct ScEvEntry {
   uint8_t dir;         /* Direction. 1: +, 0: -. */
 } ScEvEntry;
 
+/* Reverse bytecode map (IRRef -> PC). Only for selected instructions. */
+typedef struct RBCHashEntry {
+  MRef pc;             /* Bytecode PC. */
+  IRRef ref;           /* IR reference. */
+} RBCHashEntry;
+
+/* Number of slots in the reverse bytecode hash table. Must be a power of 2. */
+#define RBCHASH_SLOTS  8
+
 /* 128 bit SIMD constants. */
 enum {
   LJ_KSIMD_ABS,
@@ -364,8 +373,9 @@ typedef struct jit_State {
 
   PostProc postproc;   /* Required post-processing after execution. */
 #if LJ_SOFTFP || (LJ_32 && LJ_HASFFI)
-  int needsplit;       /* Need SPLIT pass. */
+  uint8_t needsplit;   /* Need SPLIT pass. */
 #endif
+  uint8_t retryrec;    /* Retry recording. */
 
   GCRef *trace;                /* Array of traces. */
   TraceNo freetrace;   /* Start of scan for next free trace. */
@@ -382,6 +392,8 @@ typedef struct jit_State {
   uint32_t penaltyslot;        /* Round-robin index into penalty slots. */
   uint32_t prngstate;  /* PRNG state. */
 
+  RBCHashEntry rbchash[RBCHASH_SLOTS];  /* Reverse bytecode map. */
+
   BPropEntry bpropcache[BPROP_SLOTS];  /* Backpropagation cache slots. */
   uint32_t bpropslot;  /* Round-robin index into bpropcache slots. */
 

diff --git a/src/lj_record.c b/src/lj_record.c
index 560381565ac3c5f092dfcc651e37777430241f1a..583f80aa72130cdafdbebfa7477e150041a872ff 100644 (file)
--- a/src/lj_record.c
+++ b/src/lj_record.c
@@ -235,6 +235,8 @@ static void canonicalize_slots(jit_State *J)
 /* Stop recording. */
 void lj_record_stop(jit_State *J, TraceLink linktype, TraceNo lnk)
 {
+  if (J->retryrec)
+    lj_trace_err(J, LJ_TRERR_RETRY);
   lj_trace_end(J);
   J->cur.linktype = (uint8_t)linktype;
   J->cur.link = (uint16_t)lnk;
@@ -1127,6 +1129,60 @@ static void rec_mm_comp_cdata(jit_State *J, RecordIndex *ix, int op, MMS mm)
 
 /* -- Indexed access ------------------------------------------------------ */
 
+/* Bump table allocations in bytecode when they grow during recording. */
+static void rec_idx_bump(jit_State *J, RecordIndex *ix)
+{
+  RBCHashEntry *rbc = &J->rbchash[(ix->tab & (RBCHASH_SLOTS-1))];
+  if (tref_ref(ix->tab) == rbc->ref) {
+    const BCIns *pc = mref(rbc->pc, const BCIns);
+    GCtab *tb = tabV(&ix->tabv);
+    uint32_t nhbits;
+    IRIns *ir;
+    if (!tvisnil(&ix->keyv))
+      (void)lj_tab_set(J->L, tb, &ix->keyv);  /* Grow table right now. */
+    nhbits = tb->hmask > 0 ? lj_fls(tb->hmask)+1 : 0;
+    ir = IR(tref_ref(ix->tab));
+    if (ir->o == IR_TNEW) {
+      uint32_t ah = bc_d(*pc);
+      uint32_t asize = ah & 0x7ff, hbits = ah >> 11;
+      if (nhbits > hbits) hbits = nhbits;
+      if (tb->asize > asize) {
+       asize = tb->asize <= 0x7ff ? tb->asize : 0x7ff;
+      }
+      if ((asize | (hbits<<11)) != ah) {  /* Has the size changed? */
+       /* Patch bytecode, but continue recording (for more patching). */
+       setbc_d(pc, (asize | (hbits<<11)));
+       /* Patching TNEW operands is only safe if the trace is aborted. */
+       ir->op1 = asize; ir->op2 = hbits;
+       J->retryrec = 1;  /* Abort the trace at the end of recording. */
+      }
+    } else if (ir->o == IR_TDUP) {
+      GCtab *tpl = gco2tab(proto_kgc(J->pt, ~(ptrdiff_t)bc_d(*pc)));
+      /* Grow template table, but preserve keys with nil values. */
+      if (tb->asize > tpl->asize || (1u << nhbits)-1 > tpl->hmask) {
+       Node *node = noderef(tpl->node);
+       uint32_t i, hmask = tpl->hmask;
+       for (i = 0; i <= hmask; i++) {
+         if (!tvisnil(&node[i].key) && tvisnil(&node[i].val))
+           settabV(J->L, &node[i].val, tpl);
+       }
+       if (!tvisnil(&ix->keyv) && tref_isk(ix->key)) {
+         TValue *o = lj_tab_set(J->L, tpl, &ix->keyv);
+         if (tvisnil(o)) settabV(J->L, o, tpl);
+       }
+       lj_tab_resize(J->L, tpl, tb->asize, nhbits);
+       hmask = tpl->hmask;
+       for (i = 0; i <= hmask; i++) {
+         /* This is safe, since template tables only hold immutable values. */
+         if (tvistab(&node[i].val))
+           setnilV(&node[i].val);
+       }
+       J->retryrec = 1;  /* Abort the trace at the end of recording. */
+      }
+    }
+  }
+}
+
 /* Record bounds-check. */
 static void rec_idx_abc(jit_State *J, TRef asizeref, TRef ikey, uint32_t asize)
 {
@@ -1352,6 +1408,8 @@ TRef lj_record_idx(jit_State *J, RecordIndex *ix)
          key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT);
        xref = emitir(IRT(IR_NEWREF, IRT_P32), ix->tab, key);
        keybarrier = 0;  /* NEWREF already takes care of the key barrier. */
+       if ((J->flags & JIT_F_OPT_SINK))  /* Avoid a separate flag. */
+         rec_idx_bump(J, ix);
       }
     } else if (!lj_opt_fwd_wasnonnil(J, loadop, tref_ref(xref))) {
       /* Cannot derive that the previous value was non-nil, must do checks. */
@@ -1390,9 +1448,16 @@ static void rec_tsetm(jit_State *J, BCReg ra, BCReg rn, int32_t i)
 {
   RecordIndex ix;
   cTValue *basev = J->L->base;
-  copyTV(J->L, &ix.tabv, &basev[ra-1]);
+  GCtab *t = tabV(&basev[ra-1]);
+  settabV(J->L, &ix.tabv, t);
   ix.tab = getslot(J, ra-1);
   ix.idxchain = 0;
+  if ((J->flags & JIT_F_OPT_SINK)) {
+    if (t->asize < i+rn-ra)
+      lj_tab_reasize(J->L, t, i+rn-ra);
+    setnilV(&ix.keyv);
+    rec_idx_bump(J, &ix);
+  }
   for (; ra < rn; i++, ra++) {
     setintV(&ix.keyv, i);
     ix.key = lj_ir_kint(J, i);
@@ -1712,8 +1777,12 @@ static TRef rec_tnew(jit_State *J, uint32_t ah)
 {
   uint32_t asize = ah & 0x7ff;
   uint32_t hbits = ah >> 11;
+  TRef tr;
   if (asize == 0x7ff) asize = 0x801;
-  return emitir(IRTG(IR_TNEW, IRT_TAB), asize, hbits);
+  tr = emitir(IRTG(IR_TNEW, IRT_TAB), asize, hbits);
+  J->rbchash[(tr & (RBCHASH_SLOTS-1))].ref = tref_ref(tr);
+  setmref(J->rbchash[(tr & (RBCHASH_SLOTS-1))].pc, J->pc);
+  return tr;
 }
 
 /* -- Concatenation ------------------------------------------------------- */
@@ -2139,6 +2208,8 @@ void lj_record_ins(jit_State *J)
   case BC_TDUP:
     rc = emitir(IRTG(IR_TDUP, IRT_TAB),
                lj_ir_ktab(J, gco2tab(proto_kgc(J->pt, ~(ptrdiff_t)rc))), 0);
+    J->rbchash[(rc & (RBCHASH_SLOTS-1))].ref = tref_ref(rc);
+    setmref(J->rbchash[(rc & (RBCHASH_SLOTS-1))].pc, pc);
     break;
 
   /* -- Calls and vararg handling ----------------------------------------- */
@@ -2352,6 +2423,7 @@ void lj_record_setup(jit_State *J)
   /* Initialize state related to current trace. */
   memset(J->slot, 0, sizeof(J->slot));
   memset(J->chain, 0, sizeof(J->chain));
+  memset(J->rbchash, 0, sizeof(J->rbchash));
   memset(J->bpropcache, 0, sizeof(J->bpropcache));
   J->scev.idx = REF_NIL;
   setmref(J->scev.pc, NULL);

diff --git a/src/lj_tab.c b/src/lj_tab.c
index a9f43835c175a9af5fcdfe75259da7331efc3fca..88bf1089b5bf9354ddcb18f07f0590b6fc12392a 100644 (file)
--- a/src/lj_tab.c
+++ b/src/lj_tab.c
@@ -246,7 +246,7 @@ void LJ_FASTCALL lj_tab_free(global_State *g, GCtab *t)
 /* -- Table resizing ------------------------------------------------------ */
 
 /* Resize a table to fit the new array/hash part sizes. */
-static void resizetab(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits)
+void lj_tab_resize(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits)
 {
   Node *oldnode = noderef(t->node);
   uint32_t oldasize = t->asize;
@@ -383,7 +383,7 @@ static void rehashtab(lua_State *L, GCtab *t, cTValue *ek)
   asize += countint(ek, bins);
   na = bestasize(bins, &asize);
   total -= na;
-  resizetab(L, t, asize, hsize2hbits(total));
+  lj_tab_resize(L, t, asize, hsize2hbits(total));
 }
 
 #if LJ_HASFFI
@@ -395,7 +395,7 @@ void lj_tab_rehash(lua_State *L, GCtab *t)
 
 void lj_tab_reasize(lua_State *L, GCtab *t, uint32_t nasize)
 {
-  resizetab(L, t, nasize+1, t->hmask > 0 ? lj_fls(t->hmask)+1 : 0);
+  lj_tab_resize(L, t, nasize+1, t->hmask > 0 ? lj_fls(t->hmask)+1 : 0);
 }
 
 /* -- Table getters ------------------------------------------------------- */

diff --git a/src/lj_tab.h b/src/lj_tab.h
index 1da28bd97f652f30a269f3f98e16130ec0afffb3..7cf031be9afb137a81b2dea3704af85bb306204b 100644 (file)
--- a/src/lj_tab.h
+++ b/src/lj_tab.h
@@ -44,6 +44,7 @@ LJ_FUNC void LJ_FASTCALL lj_tab_free(global_State *g, GCtab *t);
 #if LJ_HASFFI
 LJ_FUNC void lj_tab_rehash(lua_State *L, GCtab *t);
 #endif
+LJ_FUNC void lj_tab_resize(lua_State *L, GCtab *t, uint32_t asize, uint32_t hbits);
 LJ_FUNCA void lj_tab_reasize(lua_State *L, GCtab *t, uint32_t nasize);
 
 /* Caveat: all getters except lj_tab_get() can return NULL! */

diff --git a/src/lj_trace.c b/src/lj_trace.c
index 39ff0461601fdcda0353182b34c2e4e365ce441b..79c50b0abb0c528dc23b3f02696a78b7dea6ba19 100644 (file)
--- a/src/lj_trace.c
+++ b/src/lj_trace.c
@@ -394,6 +394,7 @@ static void trace_start(jit_State *J)
   J->guardemit.irt = 0;
   J->postproc = LJ_POST_NONE;
   lj_resetsplit(J);
+  J->retryrec = 0;
   setgcref(J->cur.startpt, obj2gco(J->pt));
 
   L = J->L;
@@ -510,10 +511,15 @@ static int trace_abort(jit_State *J)
   }
   /* Penalize or blacklist starting bytecode instruction. */
   if (J->parent == 0 && !bc_isret(bc_op(J->cur.startins))) {
-    if (J->exitno == 0)
-      penalty_pc(J, &gcref(J->cur.startpt)->pt, mref(J->cur.startpc, BCIns), e);
-    else
+    if (J->exitno == 0) {
+      BCIns *startpc = mref(J->cur.startpc, BCIns);
+      if (e == LJ_TRERR_RETRY)
+       hotcount_set(J2GG(J), startpc+1, 1);  /* Immediate retry. */
+      else
+       penalty_pc(J, &gcref(J->cur.startpt)->pt, startpc, e);
+    } else {
       traceref(J, J->exitno)->link = J->exitno;  /* Self-link is blacklisted. */
+    }
   }
 
   /* Is there anything to abort? */

diff --git a/src/lj_traceerr.h b/src/lj_traceerr.h
index 5a2fe8bb73eecfb48f190f909a16d90acaa8e445..0f38562b6b3b5b480b9025a27e5895e21856c691 100644 (file)
--- a/src/lj_traceerr.h
+++ b/src/lj_traceerr.h
@@ -12,6 +12,7 @@ TREDEF(TRACEOV,       "trace too long")
 TREDEF(STACKOV,        "trace too deep")
 TREDEF(SNAPOV, "too many snapshots")
 TREDEF(BLACKL, "blacklisted")
+TREDEF(RETRY,  "retry recording")
 TREDEF(NYIBC,  "NYI: bytecode %d")
 
 /* Recording loop ops. */