Drop range limit for sunk stores relative to sunk allocation.

diff --git a/src/jit/dump.lua b/src/jit/dump.lua
index 989339711e3e2f2f83ef3f60d3309f49cc2b0286..e5871b8da0f924339e090cb8b84968ddeb9284f2 100644 (file)
--- a/src/jit/dump.lua
+++ b/src/jit/dump.lua
@@ -378,7 +378,7 @@ local function ridsp_name(ridsp, ins)
   if not disass then disass = require("jit.dis_"..jit.arch) end
   local rid, slot = band(ridsp, 0xff), shr(ridsp, 8)
   if rid == 253 or rid == 254 then
-    return slot == 0 and " {sink" or format(" {%04d", ins-slot)
+    return (slot == 0 or slot == 255) and " {sink" or format(" {%04d", ins-slot)
   end
   if ridsp > 255 then return format("[%x]", slot*4) end
   if rid < 128 then return disass.regname(rid) end

diff --git a/src/lj_asm.c b/src/lj_asm.c
index 15685d858fd1dcd9af916613522bc918c1195ef8..6ea5bc93c4fec43d644d8fb6a9fc348a04350898 100644 (file)
--- a/src/lj_asm.c
+++ b/src/lj_asm.c
@@ -778,6 +778,23 @@ static int asm_snap_canremat(ASMState *as)
   return 0;
 }
 
+/* Check whether a sunk store corresponds to an allocation. */
+static int asm_sunk_store(ASMState *as, IRIns *ira, IRIns *irs)
+{
+  if (irs->s == 255) {
+    if (irs->o == IR_ASTORE || irs->o == IR_HSTORE ||
+       irs->o == IR_FSTORE || irs->o == IR_XSTORE) {
+      IRIns *irk = IR(irs->op1);
+      if (irk->o == IR_AREF || irk->o == IR_HREFK)
+       irk = IR(irk->op1);
+      return (IR(irk->op1) == ira);
+    }
+    return 0;
+  } else {
+    return (ira + irs->s == irs);  /* Quick check. */
+  }
+}
+
 /* Allocate register or spill slot for a ref that escapes to a snapshot. */
 static void asm_snap_alloc1(ASMState *as, IRRef ref)
 {
@@ -795,8 +812,8 @@ static void asm_snap_alloc1(ASMState *as, IRRef ref)
       else {  /* Allocate stored values for TNEW, TDUP and CNEW. */
        IRIns *irs;
        lua_assert(ir->o == IR_TNEW || ir->o == IR_TDUP || ir->o == IR_CNEW);
-       for (irs = IR(as->curins); irs > ir; irs--)
-         if (irs->r == RID_SINK && ir + irs->s == irs) {
+       for (irs = IR(as->snapref-1); irs > ir; irs--)
+         if (irs->r == RID_SINK && asm_sunk_store(as, ir, irs)) {
            lua_assert(irs->o == IR_ASTORE || irs->o == IR_HSTORE ||
                       irs->o == IR_FSTORE || irs->o == IR_XSTORE);
            asm_snap_alloc1(as, irs->op2);

diff --git a/src/lj_opt_sink.c b/src/lj_opt_sink.c
index 937cccc7a040f12fa6b0cac99ea9886017e85b7a..b7a3af2d1954744ae107a48bed17b56341b04230 100644 (file)
--- a/src/lj_opt_sink.c
+++ b/src/lj_opt_sink.c
@@ -32,8 +32,6 @@ static IRIns *sink_checkalloc(jit_State *J, IRIns *irs)
   ir = IR(ir->op1);
   if (!(ir->o == IR_TNEW || ir->o == IR_TDUP || ir->o == IR_CNEW))
     return NULL;  /* Not an allocation. */
-  if (ir + 255 < irs)
-    return NULL;  /* Out of range. */
   return ir;  /* Return allocation. */
 }
 
@@ -173,10 +171,12 @@ static void sink_sweep_ins(jit_State *J)
     switch (ir->o) {
     case IR_ASTORE: case IR_HSTORE: case IR_FSTORE: case IR_XSTORE: {
       IRIns *ira = sink_checkalloc(J, ir);
-      if (ira && !irt_ismarked(ira->t))
-       ir->prev = REGSP(RID_SINK, (int)(ir - ira));
-      else
+      if (ira && !irt_ismarked(ira->t)) {
+       int delta = (int)(ir - ira);
+       ir->prev = REGSP(RID_SINK, delta > 255 ? 255 : delta);
+      } else {
        ir->prev = REGSP_INIT;
+      }
       break;
       }
     case IR_NEWREF:

diff --git a/src/lj_snap.c b/src/lj_snap.c
index 9fae57d8935d88d0cc0a6cb4c0707546262fae81..b9a82008e33c1f433a07915def0ee604d442bba0 100644 (file)
--- a/src/lj_snap.c
+++ b/src/lj_snap.c
@@ -403,6 +403,27 @@ static TRef snap_pref(jit_State *J, GCtrace *T, SnapEntry *map, MSize nmax,
   return tr;
 }
 
+/* Check whether a sunk store corresponds to an allocation. Slow path. */
+static int snap_sunk_store2(jit_State *J, IRIns *ira, IRIns *irs)
+{
+  if (irs->o == IR_ASTORE || irs->o == IR_HSTORE ||
+      irs->o == IR_FSTORE || irs->o == IR_XSTORE) {
+    IRIns *irk = IR(irs->op1);
+    if (irk->o == IR_AREF || irk->o == IR_HREFK)
+      irk = IR(irk->op1);
+    return (IR(irk->op1) == ira);
+  }
+  return 0;
+}
+
+/* Check whether a sunk store corresponds to an allocation. Fast path. */
+static LJ_AINLINE int snap_sunk_store(jit_State *J, IRIns *ira, IRIns *irs)
+{
+  if (irs->s != 255)
+    return (ira + irs->s == irs);  /* Fast check. */
+  return snap_sunk_store2(J, ira, irs);
+}
+
 /* Replay snapshot state to setup side trace. */
 void lj_snap_replay(jit_State *J, GCtrace *T)
 {
@@ -464,7 +485,7 @@ void lj_snap_replay(jit_State *J, GCtrace *T)
        } else {
          IRIns *irs;
          for (irs = ir+1; irs < irlast; irs++)
-           if (irs->r == RID_SINK && ir + irs->s == irs) {
+           if (irs->r == RID_SINK && snap_sunk_store(J, ir, irs)) {
              if (snap_pref(J, T, map, nent, seen, irs->op2) == 0)
                snap_pref(J, T, map, nent, seen, T->ir[irs->op2].op1);
              else if ((LJ_SOFTFP || (LJ_32 && LJ_HASFFI)) &&
@@ -504,7 +525,7 @@ void lj_snap_replay(jit_State *J, GCtrace *T)
          TRef tr = emitir(ir->ot, op1, op2);
          J->slot[snap_slot(sn)] = tr;
          for (irs = ir+1; irs < irlast; irs++)
-           if (irs->r == RID_SINK && ir + irs->s == irs) {
+           if (irs->r == RID_SINK && snap_sunk_store(J, ir, irs)) {
              IRIns *irr = &T->ir[irs->op1];
              TRef val, key = irr->op2, tmp = tr;
              if (irr->o != IR_FREF) {
@@ -700,7 +721,7 @@ static void snap_unsink(jit_State *J, GCtrace *T, ExitState *ex,
     } else {
       IRIns *irs, *irlast = &T->ir[T->snap[snapno].ref];
       for (irs = ir+1; irs < irlast; irs++)
-       if (irs->r == RID_SINK && ir + irs->s == irs) {
+       if (irs->r == RID_SINK && snap_sunk_store(J, ir, irs)) {
          IRIns *iro = &T->ir[T->ir[irs->op1].op2];
          uint8_t *p = (uint8_t *)cd;
          CTSize szs;
@@ -733,7 +754,7 @@ static void snap_unsink(jit_State *J, GCtrace *T, ExitState *ex,
     settabV(J->L, o, t);
     irlast = &T->ir[T->snap[snapno].ref];
     for (irs = ir+1; irs < irlast; irs++)
-      if (irs->r == RID_SINK && ir + irs->s == irs) {
+      if (irs->r == RID_SINK && snap_sunk_store(J, ir, irs)) {
        IRIns *irk = &T->ir[irs->op1];
        TValue tmp, *val;
        lua_assert(irs->o == IR_ASTORE || irs->o == IR_HSTORE ||