debugging purposes.
</p>
-<h3 id="jit_flush_tr"><tt>status = jit.flush(tr)</tt></h3>
+<h3 id="jit_flush_tr"><tt>jit.flush(tr)</tt></h3>
<p>
-Tries to flush the code for the specified trace and all of its
-side traces from the cache. Returns <tt>true</tt> on success.
-Returns <tt>false</tt> if there are still links to this trace.
+Flushes the specified root trace and all of its side traces from the cache.
+The code for the trace will be retained as long as there are any other
+traces which link to it.
</p>
<h3 id="jit_status"><tt>status, ... = jit.status()</tt></h3>
<h3 id="mode_engine"><tt>luaJIT_setmode(L, trace,<br>
LUAJIT_MODE_TRACE|LUAJIT_MODE_FLUSH)</tt></h3>
<p>
-Tries to flush the code for the specified trace and all of its
-side traces from the cache.
+Flushes the specified root trace and all of its side traces from the cache.
+The code for the trace will be retained as long as there are any other
+traces which link to it.
</p>
<h3 id="mode_engine"><tt>luaJIT_setmode(L, idx, LUAJIT_MODE_WRAPCFUNC|flag)</tt></h3>
255,248,3,102,15,46,193,252,233,244,1,255,141,12,202,139,105,4,129,252,253,
239,15,132,244,247,255,137,105,252,252,139,41,137,105,252,248,252,233,245,
255,141,156,253,131,233,139,1,137,105,252,252,137,65,252,248,255,65,139,142,
- 233,139,4,193,72,139,128,233,139,108,36,24,65,137,150,233,65,137,174,233,
+ 233,139,4,129,72,139,128,233,139,108,36,24,65,137,150,233,65,137,174,233,
76,137,36,36,76,137,108,36,8,72,131,252,236,16,252,255,224,255,141,156,253,
131,233,139,3,15,182,204,15,182,232,131,195,4,193,232,16,65,252,255,36,252,
238,255,137,221,209,252,237,129,229,239,102,65,131,172,253,46,233,1,15,132,
#define DtA(_V) (int)(ptrdiff_t)&(((GCupval *)0)_V)
#define DtB(_V) (int)(ptrdiff_t)&(((Node *)0)_V)
#define DtC(_V) (int)(ptrdiff_t)&(((int *)0)_V)
-#define DtD(_V) (int)(ptrdiff_t)&(((Trace *)0)_V)
+#define DtD(_V) (int)(ptrdiff_t)&(((GCtrace *)0)_V)
#define DtE(_V) (int)(ptrdiff_t)&(((ExitInfo *)0)_V)
#define DISPATCH_GL(field) (GG_DISP2G + (int)offsetof(global_State, field))
#define DISPATCH_J(field) (GG_DISP2J + (int)offsetof(jit_State, field))
180,253,134,233,255,248,3,102,15,46,193,252,233,244,1,255,141,12,202,139,
105,4,129,252,253,239,15,132,244,247,255,137,105,252,252,139,41,137,105,252,
248,252,233,245,255,141,180,253,134,233,139,1,137,105,252,252,137,65,252,
- 248,255,139,139,233,139,4,193,72,139,128,233,139,108,36,96,137,147,233,137,
+ 248,255,139,139,233,139,4,129,72,139,128,233,139,108,36,96,137,147,233,137,
171,233,76,137,100,36,80,76,137,108,36,32,76,137,116,36,24,76,137,124,36,
16,72,137,225,72,129,252,236,239,102,15,127,49,102,15,127,185,233,102,68,
15,127,129,233,102,68,15,127,137,233,102,68,15,127,145,233,102,68,15,127,
#define DtA(_V) (int)(ptrdiff_t)&(((GCupval *)0)_V)
#define DtB(_V) (int)(ptrdiff_t)&(((Node *)0)_V)
#define DtC(_V) (int)(ptrdiff_t)&(((int *)0)_V)
-#define DtD(_V) (int)(ptrdiff_t)&(((Trace *)0)_V)
+#define DtD(_V) (int)(ptrdiff_t)&(((GCtrace *)0)_V)
#define DtE(_V) (int)(ptrdiff_t)&(((ExitInfo *)0)_V)
#define DISPATCH_GL(field) (GG_DISP2G + (int)offsetof(global_State, field))
#define DISPATCH_J(field) (GG_DISP2J + (int)offsetof(jit_State, field))
|.type UPVAL, GCupval
|.type NODE, Node
|.type NARGS, int
-|.type TRACE, Trace
+|.type TRACE, GCtrace
|.type EXITINFO, ExitInfo
|
|// Stack layout while in interpreter. Must match with lj_frame.h.
#if LJ_HASJIT
| ins_AD // RA = base (ignored), RD = traceno
| mov RA, [DISPATCH+DISPATCH_J(trace)]
- |.if X64
- | mov TRACE:RD, [RA+RD*8]
- |.else
| mov TRACE:RD, [RA+RD*4]
- |.endif
| mov RDa, TRACE:RD->mcode
| mov L:RB, SAVE_L
| mov [DISPATCH+DISPATCH_GL(jit_base)], BASE
#define DtA(_V) (int)(ptrdiff_t)&(((GCupval *)0)_V)
#define DtB(_V) (int)(ptrdiff_t)&(((Node *)0)_V)
#define DtC(_V) (int)(ptrdiff_t)&(((int *)0)_V)
-#define DtD(_V) (int)(ptrdiff_t)&(((Trace *)0)_V)
+#define DtD(_V) (int)(ptrdiff_t)&(((GCtrace *)0)_V)
#define DtE(_V) (int)(ptrdiff_t)&(((ExitInfo *)0)_V)
#define DISPATCH_GL(field) (GG_DISP2G + (int)offsetof(global_State, field))
#define DISPATCH_J(field) (GG_DISP2J + (int)offsetof(jit_State, field))
LJLIB_PUSH("thread")
LJLIB_PUSH("proto")
LJLIB_PUSH("function")
-LJLIB_PUSH("") /* Unused. */
+LJLIB_PUSH("trace")
LJLIB_PUSH("table")
LJLIB_PUSH(top-8) /* userdata */
LJLIB_PUSH("number")
#if LJ_HASJIT
if (L->base < L->top && (tvisnum(L->base) || tvisstr(L->base))) {
int traceno = lj_lib_checkint(L, 1);
- setboolV(L->top-1,
- luaJIT_setmode(L, traceno, LUAJIT_MODE_FLUSH|LUAJIT_MODE_TRACE));
- return 1;
+ luaJIT_setmode(L, traceno, LUAJIT_MODE_FLUSH|LUAJIT_MODE_TRACE);
+ return 0;
}
#endif
return setjitmode(L, LUAJIT_MODE_FLUSH);
#if LJ_HASJIT
/* Check trace argument. Must not throw for non-existent trace numbers. */
-static Trace *jit_checktrace(lua_State *L)
+static GCtrace *jit_checktrace(lua_State *L)
{
TraceNo tr = (TraceNo)lj_lib_checkint(L, 1);
jit_State *J = L2J(L);
if (tr > 0 && tr < J->sizetrace)
- return J->trace[tr];
+ return traceref(J, tr);
return NULL;
}
/* local info = jit.util.traceinfo(tr) */
LJLIB_CF(jit_util_traceinfo)
{
- Trace *T = jit_checktrace(L);
+ GCtrace *T = jit_checktrace(L);
if (T) {
GCtab *t;
lua_createtable(L, 0, 4); /* Increment hash size if fields are added. */
/* local m, ot, op1, op2, prev = jit.util.traceir(tr, idx) */
LJLIB_CF(jit_util_traceir)
{
- Trace *T = jit_checktrace(L);
+ GCtrace *T = jit_checktrace(L);
IRRef ref = (IRRef)lj_lib_checkint(L, 2) + REF_BIAS;
if (T && ref >= REF_BIAS && ref < T->nins) {
IRIns *ir = &T->ir[ref];
/* local k, t [, slot] = jit.util.tracek(tr, idx) */
LJLIB_CF(jit_util_tracek)
{
- Trace *T = jit_checktrace(L);
+ GCtrace *T = jit_checktrace(L);
IRRef ref = (IRRef)lj_lib_checkint(L, 2) + REF_BIAS;
if (T && ref >= T->nk && ref < REF_BIAS) {
IRIns *ir = &T->ir[ref];
/* local snap = jit.util.tracesnap(tr, sn) */
LJLIB_CF(jit_util_tracesnap)
{
- Trace *T = jit_checktrace(L);
+ GCtrace *T = jit_checktrace(L);
SnapNo sn = (SnapNo)lj_lib_checkint(L, 2);
if (T && sn < T->nsnap) {
SnapShot *snap = &T->snap[sn];
/* local mcode, addr, loop = jit.util.tracemc(tr) */
LJLIB_CF(jit_util_tracemc)
{
- Trace *T = jit_checktrace(L);
+ GCtrace *T = jit_checktrace(L);
if (T && T->mcode != NULL) {
setstrV(L, L->top-1, lj_str_new(L, (const char *)T->mcode, T->szmcode));
setnumV(L->top++, cast_num((intptr_t)T->mcode));
BCReg topslot; /* Number of slots for stack check (unless 0). */
MSize gcsteps; /* Accumulated number of GC steps (per section). */
- Trace *T; /* Trace to assemble. */
- Trace *parent; /* Parent trace (or NULL). */
+ GCtrace *T; /* Trace to assemble. */
+ GCtrace *parent; /* Parent trace (or NULL). */
MCode *mcbot; /* Bottom of reserved MCode. */
MCode *mctop; /* Top of generated MCode. */
const BCIns *pc = snap_pc(as->T->snapmap[snap->mapofs + snap->nent]);
int32_t mres;
if (bc_op(*pc) == BC_JLOOP) { /* NYI: find a better way to do this. */
- BCIns *retpc = &as->J->trace[bc_d(*pc)]->startins;
+ BCIns *retpc = &traceref(as->J, bc_d(*pc))->startins;
if (bc_isret(bc_op(*retpc)))
pc = retpc;
}
}
/* Patch exit branch. */
target = lnk == TRACE_INTERP ? (MCode *)lj_vm_exit_interp :
- as->J->trace[lnk]->mcode;
+ traceref(as->J, lnk)->mcode;
*(int32_t *)(p-4) = jmprel(p, target);
p[-5] = XI_JMP;
/* Drop unused mcode tail. Fill with NOPs to make the prefetcher happy. */
/* -- Trace setup --------------------------------------------------------- */
/* Clear reg/sp for all instructions and add register hints. */
-static void asm_setup_regsp(ASMState *as, Trace *T)
+static void asm_setup_regsp(ASMState *as, GCtrace *T)
{
IRRef i, nins;
int inloop;
#endif
/* Assemble a trace. */
-void lj_asm_trace(jit_State *J, Trace *T)
+void lj_asm_trace(jit_State *J, GCtrace *T)
{
ASMState as_;
ASMState *as = &as_;
as->realign = NULL;
as->loopinv = 0;
if (J->parent) {
- as->parent = J->trace[J->parent];
+ as->parent = traceref(J, J->parent);
lj_snap_regspmap(as->parentmap, as->parent, J->exitno);
} else {
as->parent = NULL;
}
/* Patch exit jumps of existing machine code to a new target. */
-void lj_asm_patchexit(jit_State *J, Trace *T, ExitNo exitno, MCode *target)
+void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno, MCode *target)
{
MCode *p = T->mcode;
MCode *mcarea = lj_mcode_patch(J, p, 0);
#include "lj_jit.h"
#if LJ_HASJIT
-LJ_FUNC void lj_asm_trace(jit_State *J, Trace *T);
-LJ_FUNC void lj_asm_patchexit(jit_State *J, Trace *T, ExitNo exitno,
+LJ_FUNC void lj_asm_trace(jit_State *J, GCtrace *T);
+LJ_FUNC void lj_asm_patchexit(jit_State *J, GCtrace *T, ExitNo exitno,
MCode *target);
#endif
case LUAJIT_MODE_TRACE:
if (!(mode & LUAJIT_MODE_FLUSH))
return 0; /* Failed. */
- return lj_trace_flush(G2J(g), idx);
+ lj_trace_flush(G2J(g), idx);
+ break;
#else
case LUAJIT_MODE_ENGINE:
case LUAJIT_MODE_FUNC:
void LJ_FASTCALL lj_func_freeproto(global_State *g, GCproto *pt)
{
- lj_trace_freeproto(g, pt);
lj_mem_free(g, pt, pt->sizept);
}
}
#if LJ_HASJIT
+/* Mark a trace. */
+static void gc_marktrace(global_State *g, TraceNo traceno)
+{
+ if (traceno && traceno != G2J(g)->curtrace) {
+ GCobj *o = obj2gco(traceref(G2J(g), traceno));
+ if (iswhite(o)) {
+ white2gray(o);
+ setgcrefr(o->gch.gclist, g->gc.gray);
+ setgcref(g->gc.gray, o);
+ }
+ }
+}
+
/* Traverse a trace. */
-static void gc_traverse_trace(global_State *g, Trace *T)
+static void gc_traverse_trace(global_State *g, GCtrace *T)
{
IRRef ref;
for (ref = T->nk; ref < REF_TRUE; ref++) {
if (ir->o == IR_KGC)
gc_markobj(g, ir_kgc(ir));
}
+ gc_marktrace(g, T->link);
+ gc_marktrace(g, T->nextroot);
+ gc_marktrace(g, T->nextside);
+ gc_markobj(g, gcref(T->startpt));
}
/* The current trace is a GC root while not anchored in the prototype (yet). */
-#define gc_mark_curtrace(g) \
+#define gc_traverse_curtrace(g) \
{ if (G2J(g)->curtrace != 0) gc_traverse_trace(g, &G2J(g)->cur); }
#else
-#define gc_mark_curtrace(g) UNUSED(g)
+#define gc_traverse_curtrace(g) UNUSED(g)
#endif
/* Traverse a prototype. */
static void gc_traverse_proto(global_State *g, GCproto *pt)
{
ptrdiff_t i;
-#if LJ_HASJIT
- jit_State *J = G2J(g);
- TraceNo root, side;
- /* Mark all root traces and attached side traces. */
- for (root = pt->trace; root != 0; root = J->trace[root]->nextroot) {
- for (side = J->trace[root]->nextside; side != 0;
- side = J->trace[side]->nextside)
- gc_traverse_trace(g, J->trace[side]);
- gc_traverse_trace(g, J->trace[root]);
- }
-#endif
- /* GC during prototype creation could cause NULL fields. */
gc_mark_str(proto_chunkname(pt));
for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) /* Mark collectable consts. */
gc_markobj(g, proto_kgc(pt, i));
gc_mark_str(proto_uvname(pt, i));
for (i = 0; i < (ptrdiff_t)pt->sizevarinfo; i++) /* Mark names of locals. */
gc_mark_str(gco2str(gcref(proto_varinfo(pt)[i].name)));
+#if LJ_HASJIT
+ gc_marktrace(g, pt->trace);
+#endif
}
/* Traverse the frame structure of a stack. */
GCproto *pt = gco2pt(o);
gc_traverse_proto(g, pt);
return pt->sizept;
- } else {
+ } else if (LJ_LIKELY(o->gch.gct == ~LJ_TTHREAD)) {
lua_State *th = gco2th(o);
setgcrefr(th->gclist, g->gc.grayagain);
setgcref(g->gc.grayagain, o);
black2gray(o); /* Threads are never black. */
gc_traverse_thread(g, th);
return sizeof(lua_State) + sizeof(TValue) * th->stacksize;
+ } else {
+#if LJ_HASJIT
+ GCtrace *T = gco2trace(o);
+ gc_traverse_trace(g, T);
+ return ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) +
+ T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry);
+#else
+ lua_assert(0);
+ return 0;
+#endif
}
}
(GCFreeFunc)lj_state_free,
(GCFreeFunc)lj_func_freeproto,
(GCFreeFunc)lj_func_free,
+#if LJ_HASJIT
+ (GCFreeFunc)lj_trace_free,
+#else
(GCFreeFunc)0,
+#endif
(GCFreeFunc)lj_tab_free,
(GCFreeFunc)lj_udata_free
};
setgcrefnull(g->gc.weak);
lua_assert(!iswhite(obj2gco(mainthread(g))));
gc_markobj(g, L); /* Mark running thread. */
- gc_mark_curtrace(g); /* Mark current trace. */
+ gc_traverse_curtrace(g); /* Traverse current trace. */
gc_mark_gcroot(g); /* Mark GC roots (again). */
gc_propagate_gray(g); /* Propagate all of the above. */
lua_assert(g->gc.state != GCSfinalize && g->gc.state != GCSpause);
lua_assert(o->gch.gct != ~LJ_TTAB);
/* Preserve invariant during propagation. Otherwise it doesn't matter. */
- if (g->gc.state == GCSpropagate)
+ if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic)
gc_mark(g, v); /* Move frontier forward. */
else
makewhite(g, o); /* Make it white to avoid the following barrier. */
{
#define TV2MARKED(x) \
(*((uint8_t *)(x) - offsetof(GCupval, tv) + offsetof(GCupval, marked)))
- if (g->gc.state == GCSpropagate)
+ if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic)
gc_mark(g, gcV(tv));
else
TV2MARKED(tv) = (TV2MARKED(tv) & cast_byte(~LJ_GC_COLORS)) | curwhite(g);
setgcrefr(o->gch.nextgc, g->gc.root);
setgcref(g->gc.root, o);
if (isgray(o)) { /* A closed upvalue is never gray, so fix this. */
- if (g->gc.state == GCSpropagate) {
+ if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic) {
gray2black(o); /* Make it black and preserve invariant. */
if (tviswhite(&uv->tv))
lj_gc_barrierf(g, o, gcV(&uv->tv));
#if LJ_HASJIT
/* Mark a trace if it's saved during the propagation phase. */
-void lj_gc_barriertrace(global_State *g, void *T)
+void lj_gc_barriertrace(global_State *g, uint32_t traceno)
{
- if (g->gc.state == GCSpropagate)
- gc_traverse_trace(g, (Trace *)T);
+ if (g->gc.state == GCSpropagate || g->gc.state == GCSatomic)
+ gc_marktrace(g, traceno);
}
#endif
LJ_FUNCA void LJ_FASTCALL lj_gc_barrieruv(global_State *g, TValue *tv);
LJ_FUNC void lj_gc_closeuv(global_State *g, GCupval *uv);
#if LJ_HASJIT
-LJ_FUNC void lj_gc_barriertrace(global_State *g, void *T);
+LJ_FUNC void lj_gc_barriertrace(global_State *g, uint32_t traceno);
#endif
/* Barrier for stores to table objects. TValue and GCobj variant. */
typedef struct GDBJITctx {
uint8_t *p; /* Pointer to next address in obj.space. */
uint8_t *startp; /* Pointer to start address in obj.space. */
- Trace *T; /* Generate symbols for this trace. */
+ GCtrace *T; /* Generate symbols for this trace. */
uintptr_t mcaddr; /* Machine code address. */
MSize szmcode; /* Size of machine code. */
MSize spadjp; /* Stack adjustment for parent trace or interpreter. */
}
/* Add debug info for newly compiled trace and notify GDB. */
-void lj_gdbjit_addtrace(jit_State *J, Trace *T, TraceNo traceno)
+void lj_gdbjit_addtrace(jit_State *J, GCtrace *T, TraceNo traceno)
{
GDBJITctx ctx;
lua_State *L = J->L;
ctx.T = T;
ctx.mcaddr = (uintptr_t)T->mcode;
ctx.szmcode = T->szmcode;
- ctx.spadjp = CFRAME_SIZE_JIT + (MSize)(parent?J->trace[parent]->spadjust:0);
+ ctx.spadjp = CFRAME_SIZE_JIT +
+ (MSize)(parent ? traceref(J, parent)->spadjust : 0);
ctx.spadj = CFRAME_SIZE_JIT + T->spadjust;
if (startpc >= proto_bc(pt) && startpc < proto_bc(pt) + pt->sizebc)
ctx.lineno = proto_line(pt, proto_bcpos(pt, startpc));
}
/* Delete debug info for trace and notify GDB. */
-void lj_gdbjit_deltrace(jit_State *J, Trace *T)
+void lj_gdbjit_deltrace(jit_State *J, GCtrace *T)
{
GDBJITentryobj *eo = (GDBJITentryobj *)T->gdbjit_entry;
if (eo) {
#if LJ_HASJIT && defined(LUAJIT_USE_GDBJIT)
-LJ_FUNC void lj_gdbjit_addtrace(jit_State *J, Trace *T, TraceNo traceno);
-LJ_FUNC void lj_gdbjit_deltrace(jit_State *J, Trace *T);
+LJ_FUNC void lj_gdbjit_addtrace(jit_State *J, GCtrace *T, TraceNo traceno);
+LJ_FUNC void lj_gdbjit_deltrace(jit_State *J, GCtrace *T);
#else
#define lj_gdbjit_addtrace(J, T, tn) UNUSED(T)
IRT_THREAD,
IRT_PROTO,
IRT_FUNC,
- IRT_9, /* Never used in the IR. */
+ IRT_9, /* Unused (map of LJ_TTRACE). */
IRT_TAB,
IRT_UDATA,
/* ... until here. */
#define TRACE_INTERP 0 /* Fallback to interpreter. */
-/* Trace anchor. */
-typedef struct Trace {
- IRIns *ir; /* IR instructions/constants. Biased with REF_BIAS. */
+/* Trace object. */
+typedef struct GCtrace {
+ GCHeader;
+ uint8_t topslot; /* Top stack slot already checked to be allocated. */
+ uint8_t unused1;
IRRef nins; /* Next IR instruction. Biased with REF_BIAS. */
+ GCRef gclist;
+ IRIns *ir; /* IR instructions/constants. Biased with REF_BIAS. */
IRRef nk; /* Lowest IR constant. Biased with REF_BIAS. */
- SnapShot *snap; /* Snapshot array. */
- SnapEntry *snapmap; /* Snapshot map. */
uint16_t nsnap; /* Number of snapshots. */
uint16_t nsnapmap; /* Number of snapshot map elements. */
+ SnapShot *snap; /* Snapshot array. */
+ SnapEntry *snapmap; /* Snapshot map. */
GCRef startpt; /* Starting prototype. */
BCIns startins; /* Original bytecode of starting instruction. */
MCode *mcode; /* Start of machine code. */
MSize szmcode; /* Size of machine code. */
MSize mcloop; /* Offset of loop start in machine code. */
+ uint16_t nchild; /* Number of child traces (root trace only). */
+ uint16_t spadjust; /* Stack pointer adjustment (offset in bytes). */
+ TraceNo1 traceno; /* Trace number. */
TraceNo1 link; /* Linked trace (or self for loops). */
TraceNo1 root; /* Root trace of side trace (or 0 for root traces). */
TraceNo1 nextroot; /* Next root trace for same prototype. */
TraceNo1 nextside; /* Next side trace of same root trace. */
- uint16_t nchild; /* Number of child traces (root trace only). */
- uint16_t spadjust; /* Stack pointer adjustment (offset in bytes). */
- uint8_t topslot; /* Top stack slot already checked to be allocated. */
- uint8_t unused1;
- uint8_t unused2;
- uint8_t unused3;
+ uint16_t unused2;
#ifdef LUAJIT_USE_GDBJIT
void *gdbjit_entry; /* GDB JIT entry. */
#endif
-} Trace;
+} GCtrace;
+
+#define gco2trace(o) check_exp((o)->gch.gct == ~LJ_TTRACE, (GCtrace *)(o))
+#define traceref(J, n) \
+ check_exp((n)>0 && (MSize)(n)<J->sizetrace, (GCtrace *)gcref(J->trace[(n)]))
+
+LJ_STATIC_ASSERT(offsetof(GChead, gclist) == offsetof(GCtrace, gclist));
/* Round-robin penalty cache for bytecodes leading to aborted traces. */
typedef struct HotPenalty {
/* JIT compiler state. */
typedef struct jit_State {
- Trace cur; /* Current trace. */
+ GCtrace cur; /* Current trace. */
lua_State *L; /* Current Lua state. */
const BCIns *pc; /* Current PC. */
MSize sizesnapmap; /* Size of temp. snapshot map buffer. */
TraceNo curtrace; /* Current trace number (if not 0). Kept in J->cur. */
- Trace **trace; /* Array of traces. */
+ GCRef *trace; /* Array of traces. */
TraceNo freetrace; /* Start of scan for next free trace. */
MSize sizetrace; /* Size of trace array. */
LJ_DATADEF const char *const lj_obj_itypename[] = { /* ORDER LJ_T */
"nil", "boolean", "boolean", "userdata", "string", "upval", "thread",
- "proto", "function", "" /* Unused */, "table", "userdata", "number"
+ "proto", "function", "trace", "table", "userdata", "number"
};
/* Compare two objects without calling metamethods. */
#define LJ_TTHREAD (-7)
#define LJ_TPROTO (-8)
#define LJ_TFUNC (-9)
-/* Unused (-10) */
+#define LJ_TTRACE (-10)
#define LJ_TTAB (-11)
#define LJ_TUDATA (-12)
/* This is just the canonical number type used in some places. */
if (op == BC_KSHORT || op == BC_KNUM) { /* Found const. initializer. */
/* Now try to verify there's no forward jump across it. */
const BCIns *kpc = pc;
- for ( ; pc > startpc; pc--)
+ for (; pc > startpc; pc--)
if (bc_op(*pc) == BC_JMP) {
const BCIns *target = pc+bc_j(*pc)+1;
if (target > kpc && target <= endpc)
break;
case BC_JFORL:
- rec_loop_jit(J, rc, rec_for(J, pc+bc_j(J->trace[rc]->startins), 1));
+ rec_loop_jit(J, rc, rec_for(J, pc+bc_j(traceref(J, rc)->startins), 1));
break;
case BC_JITERL:
- rec_loop_jit(J, rc, rec_iterl(J, J->trace[rc]->startins));
+ rec_loop_jit(J, rc, rec_iterl(J, traceref(J, rc)->startins));
break;
case BC_JLOOP:
rec_loop_jit(J, rc, rec_loop(J, ra));
}
/* Setup recording for a side trace. */
-static void rec_setup_side(jit_State *J, Trace *T)
+static void rec_setup_side(jit_State *J, GCtrace *T)
{
SnapShot *snap = &T->snap[J->exitno];
SnapEntry *map = &T->snapmap[snap->mapofs];
}
J->cur.nk = REF_TRUE;
- setgcref(J->cur.startpt, obj2gco(J->pt));
J->startpc = J->pc;
if (J->parent) { /* Side trace. */
- Trace *T = J->trace[J->parent];
+ GCtrace *T = traceref(J, J->parent);
TraceNo root = T->root ? T->root : J->parent;
J->cur.root = (uint16_t)root;
J->cur.startins = BCINS_AD(BC_JMP, 0, 0);
}
rec_setup_side(J, T);
sidecheck:
- if (J->trace[J->cur.root]->nchild >= J->param[JIT_P_maxside] ||
+ if (traceref(J, J->cur.root)->nchild >= J->param[JIT_P_maxside] ||
T->snap[J->exitno].count >= J->param[JIT_P_hotexit] +
J->param[JIT_P_tryside])
rec_stop(J, TRACE_INTERP);
** There are very few renames (often none), so the filter has
** very few bits set. This makes it suitable for negative filtering.
*/
-static BloomFilter snap_renamefilter(Trace *T, SnapNo lim)
+static BloomFilter snap_renamefilter(GCtrace *T, SnapNo lim)
{
BloomFilter rfilt = 0;
IRIns *ir;
}
/* Process matching renames to find the original RegSP. */
-static RegSP snap_renameref(Trace *T, SnapNo lim, IRRef ref, RegSP rs)
+static RegSP snap_renameref(GCtrace *T, SnapNo lim, IRRef ref, RegSP rs)
{
IRIns *ir;
for (ir = &T->ir[T->nins-1]; ir->o == IR_RENAME; ir--)
/* Convert a snapshot into a linear slot -> RegSP map.
** Note: unused slots are not initialized!
*/
-void lj_snap_regspmap(uint16_t *rsmap, Trace *T, SnapNo snapno)
+void lj_snap_regspmap(uint16_t *rsmap, GCtrace *T, SnapNo snapno)
{
SnapShot *snap = &T->snap[snapno];
MSize n, nent = snap->nent;
{
ExitState *ex = (ExitState *)exptr;
SnapNo snapno = J->exitno; /* For now, snapno == exitno. */
- Trace *T = J->trace[J->parent];
+ GCtrace *T = traceref(J, J->parent);
SnapShot *snap = &T->snap[snapno];
MSize n, nent = snap->nent;
SnapEntry *map = &T->snapmap[snap->mapofs];
#if LJ_HASJIT
LJ_FUNC void lj_snap_add(jit_State *J);
LJ_FUNC void lj_snap_shrink(jit_State *J);
-LJ_FUNC void lj_snap_regspmap(uint16_t *rsmap, Trace *T, SnapNo snapno);
+LJ_FUNC void lj_snap_regspmap(uint16_t *rsmap, GCtrace *T, SnapNo snapno);
LJ_FUNC const BCIns *lj_snap_restore(jit_State *J, void *exptr);
LJ_FUNC void lj_snap_grow_buf_(jit_State *J, MSize need);
LJ_FUNC void lj_snap_grow_map_(jit_State *J, MSize need);
/* The current trace is first assembled in J->cur. The variable length
** arrays point to shared, growable buffers (J->irbuf etc.). The trace is
** kept in this state until a new trace needs to be created. Then the current
-** trace and its data structures are copied to a new (compact) Trace object.
+** trace and its data structures are copied to a new (compact) GCtrace object.
*/
/* Find a free trace number. */
if (J->freetrace == 0)
J->freetrace = 1;
for (; J->freetrace < J->sizetrace; J->freetrace++)
- if (J->trace[J->freetrace] == NULL)
+ if (traceref(J, J->freetrace) == NULL)
return J->freetrace++;
/* Need to grow trace array. */
lim = (MSize)J->param[JIT_P_maxtrace] + 1;
osz = J->sizetrace;
if (osz >= lim)
return 0; /* Too many traces. */
- lj_mem_growvec(J->L, J->trace, J->sizetrace, lim, Trace *);
- while (osz < J->sizetrace)
- J->trace[osz++] = NULL;
+ lj_mem_growvec(J->L, J->trace, J->sizetrace, lim, GCRef);
+ for (; osz < J->sizetrace; osz++)
+ setgcrefnull(J->trace[osz]);
return J->freetrace;
}
p += T->szfield*sizeof(tp);
/* Save a trace by copying and compacting it. */
-static Trace *trace_save(jit_State *J, Trace *T)
+static GCtrace *trace_save(jit_State *J, GCtrace *T)
{
- size_t sztr = ((sizeof(Trace)+7)&~7);
+ size_t sztr = ((sizeof(GCtrace)+7)&~7);
size_t szins = (T->nins-T->nk)*sizeof(IRIns);
size_t sz = sztr + szins +
T->nsnap*sizeof(SnapShot) +
T->nsnapmap*sizeof(SnapEntry);
- Trace *T2 = lj_mem_newt(J->L, (MSize)sz, Trace);
+ GCtrace *T2 = lj_mem_newt(J->L, (MSize)sz, GCtrace);
char *p = (char *)T2 + sztr;
- memcpy(T2, T, sizeof(Trace));
+ memcpy(T2, T, sizeof(GCtrace));
+ setgcrefr(T2->nextgc, J2G(J)->gc.root);
+ setgcrefp(J2G(J)->gc.root, T2);
+ newwhite(J2G(J), T2);
+ T2->gct = ~LJ_TTRACE;
T2->ir = (IRIns *)p - T->nk;
memcpy(p, T->ir+T->nk, szins);
p += szins;
TRACE_COPYELEM(snap, nsnap, SnapShot)
TRACE_COPYELEM(snapmap, nsnapmap, SnapEntry)
- lj_gc_barriertrace(J2G(J), T);
return T2;
}
-/* Free a trace. */
-static void trace_free(jit_State *J, TraceNo traceno)
-{
- lua_assert(traceno != 0);
- if (traceno < J->freetrace)
- J->freetrace = traceno;
- lj_gdbjit_deltrace(J, J->trace[traceno]);
- if (traceno == J->curtrace) {
- lua_assert(J->trace[traceno] == &J->cur);
- J->trace[traceno] = NULL;
- J->curtrace = 0;
- } else {
- Trace *T = J->trace[traceno];
- lua_assert(T != NULL && T != &J->cur);
- J->trace[traceno] = NULL;
- lj_mem_free(J2G(J), T,
- ((sizeof(Trace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) +
- T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry));
- }
-}
-
-/* Free all traces associated with a prototype. No unpatching needed. */
-void lj_trace_freeproto(global_State *g, GCproto *pt)
+void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T)
{
jit_State *J = G2J(g);
- TraceNo traceno;
- /* Free all root traces. */
- for (traceno = pt->trace; traceno != 0; ) {
- TraceNo side, nextroot = J->trace[traceno]->nextroot;
- /* Free all side traces. */
- for (side = J->trace[traceno]->nextside; side != 0; ) {
- TraceNo next = J->trace[side]->nextside;
- trace_free(J, side);
- side = next;
- }
- /* Now free the trace itself. */
- trace_free(J, traceno);
- traceno = nextroot;
+ if (T->traceno) {
+ lj_gdbjit_deltrace(J, T);
+ if (T->traceno < J->freetrace)
+ J->freetrace = T->traceno;
+ setgcrefnull(J->trace[T->traceno]);
}
+ lj_mem_free(g, T,
+ ((sizeof(GCtrace)+7)&~7) + (T->nins-T->nk)*sizeof(IRIns) +
+ T->nsnap*sizeof(SnapShot) + T->nsnapmap*sizeof(SnapEntry));
}
/* Re-enable compiling a prototype by unpatching any modified bytecode. */
}
/* Unpatch the bytecode modified by a root trace. */
-static void trace_unpatch(jit_State *J, Trace *T)
+static void trace_unpatch(jit_State *J, GCtrace *T)
{
BCOp op = bc_op(T->startins);
MSize pcofs = T->snap[0].mapofs + T->snap[0].nent;
lua_assert(bc_op(*pc) == BC_JFORI);
setbc_op(pc, BC_FORI); /* Unpatch JFORI, too. */
pc += bc_j(*pc);
- lua_assert(bc_op(*pc) == BC_JFORL && J->trace[bc_d(*pc)] == T);
+ lua_assert(bc_op(*pc) == BC_JFORL && traceref(J, bc_d(*pc)) == T);
*pc = T->startins;
break;
case BC_LOOP:
- lua_assert(bc_op(*pc) == BC_JLOOP && J->trace[bc_d(*pc)] == T);
+ lua_assert(bc_op(*pc) == BC_JLOOP && traceref(J, bc_d(*pc)) == T);
*pc = T->startins;
break;
case BC_ITERL:
lua_assert(bc_op(*pc) == BC_JMP);
pc += bc_j(*pc)+2;
- lua_assert(bc_op(*pc) == BC_JITERL && J->trace[bc_d(*pc)] == T);
+ lua_assert(bc_op(*pc) == BC_JITERL && traceref(J, bc_d(*pc)) == T);
*pc = T->startins;
break;
case BC_FUNCF:
- lua_assert(bc_op(*pc) == BC_JFUNCF && J->trace[bc_d(*pc)] == T);
+ lua_assert(bc_op(*pc) == BC_JFUNCF && traceref(J, bc_d(*pc)) == T);
*pc = T->startins;
break;
case BC_JMP: /* No need to unpatch branches in parent traces (yet). */
}
}
-/* Free a root trace and any attached side traces. */
-static void trace_freeroot(jit_State *J, Trace *T, TraceNo traceno)
+/* Flush a root trace. */
+static void trace_flushroot(jit_State *J, GCtrace *T)
{
GCproto *pt = &gcref(T->startpt)->pt;
- TraceNo side;
lua_assert(T->root == 0 && pt != NULL);
/* First unpatch any modified bytecode. */
trace_unpatch(J, T);
/* Unlink root trace from chain anchored in prototype. */
- if (pt->trace == traceno) { /* Trace is first in chain. Easy. */
+ if (pt->trace == T->traceno) { /* Trace is first in chain. Easy. */
pt->trace = T->nextroot;
} else { /* Otherwise search in chain of root traces. */
- Trace *T2 = J->trace[pt->trace];
- while (T2->nextroot != traceno) {
+ GCtrace *T2 = traceref(J, pt->trace);
+ while (T2->nextroot != T->traceno) {
lua_assert(T2->nextroot != 0);
- T2 = J->trace[T2->nextroot];
+ T2 = traceref(J, T2->nextroot);
}
T2->nextroot = T->nextroot; /* Unlink from chain. */
}
- /* Free all side traces. */
- for (side = T->nextside; side != 0; ) {
- TraceNo next = J->trace[side]->nextside;
- trace_free(J, side);
- side = next;
- }
- /* Now free the trace itself. */
- trace_free(J, traceno);
}
-/* Flush a root trace + side traces, if there are no links to it. */
-int lj_trace_flush(jit_State *J, TraceNo traceno)
+/* Flush a trace. Only root traces are considered. */
+void lj_trace_flush(jit_State *J, TraceNo traceno)
{
if (traceno > 0 && traceno < J->sizetrace) {
- Trace *T = J->trace[traceno];
- if (T && T->root == 0) {
- ptrdiff_t i;
- for (i = (ptrdiff_t)J->sizetrace-1; i > 0; i--)
- if (i != (ptrdiff_t)traceno && J->trace[i] &&
- J->trace[i]->root != traceno && J->trace[i]->link == traceno)
- return 0; /* Failed: existing link to trace. */
- trace_freeroot(J, T, traceno);
- return 1; /* Ok. */
- }
+ GCtrace *T = traceref(J, traceno);
+ if (T && T->root == 0)
+ trace_flushroot(J, T);
}
- return 0; /* Failed. */
}
/* Flush all traces associated with a prototype. */
void lj_trace_flushproto(global_State *g, GCproto *pt)
{
while (pt->trace != 0)
- trace_freeroot(G2J(g), G2J(g)->trace[pt->trace], pt->trace);
+ trace_flushroot(G2J(g), traceref(G2J(g), pt->trace));
}
/* Flush all traces. */
if ((J2G(J)->hookmask & HOOK_GC))
return 1;
for (i = (ptrdiff_t)J->sizetrace-1; i > 0; i--) {
- Trace *T = J->trace[i];
- if (T && T->root == 0)
- trace_freeroot(J, T, (TraceNo)i);
+ GCtrace *T = traceref(J, i);
+ if (T) {
+ if (T->root == 0)
+ trace_flushroot(J, T);
+ lj_gdbjit_deltrace(J, T);
+ T->traceno = 0;
+ setgcrefnull(J->trace[i]);
+ }
}
-#ifdef LUA_USE_ASSERT
- for (i = 0; i < (ptrdiff_t)J->sizetrace; i++)
- lua_assert(J->trace[i] == NULL);
-#endif
+ J->curtrace = 0;
J->freetrace = 0;
/* Free the whole machine code and invalidate all exit stub groups. */
lj_mcode_free(J);
void lj_trace_freestate(global_State *g)
{
jit_State *J = G2J(g);
+ if (J->curtrace)
+ lj_gdbjit_deltrace(J, &J->cur);
#ifdef LUA_USE_ASSERT
{ /* This assumes all traces have already been freed. */
ptrdiff_t i;
- for (i = 0; i < (ptrdiff_t)J->sizetrace; i++)
- lua_assert(J->trace[i] == NULL);
+ for (i = 1; i < (ptrdiff_t)J->sizetrace; i++)
+ lua_assert(i == (ptrdiff_t)J->curtrace || traceref(J, i) == NULL);
}
#endif
lj_mcode_free(J);
lj_mem_freevec(g, J->snapmapbuf, J->sizesnapmap, SnapEntry);
lj_mem_freevec(g, J->snapbuf, J->sizesnap, SnapShot);
lj_mem_freevec(g, J->irbuf + J->irbotlim, J->irtoplim - J->irbotlim, IRIns);
- lj_mem_freevec(g, J->trace, J->sizetrace, Trace *);
+ lj_mem_freevec(g, J->trace, J->sizetrace, GCRef);
}
/* -- Penalties and blacklisting ------------------------------------------ */
{
lua_State *L;
- if (J->curtrace != 0 && J->trace[J->curtrace] == &J->cur) {
- J->trace[J->curtrace] = trace_save(J, &J->cur); /* Save current trace. */
+ if (J->curtrace != 0 && traceref(J, J->curtrace) == &J->cur) {
+ TraceNo tr = J->curtrace; /* Save current trace. */
+ setgcrefp(J->trace[tr], trace_save(J, &J->cur));
J->curtrace = 0;
+ lj_gc_barriertrace(J2G(J), tr);
}
if ((J->pt->flags & PROTO_NO_JIT)) { /* JIT disabled for this proto? */
J->state = LJ_TRACE_IDLE; /* Silently ignored. */
return;
}
- J->trace[J->curtrace] = &J->cur;
+ setgcrefp(J->trace[J->curtrace], &J->cur);
/* Setup enough of the current trace to be able to send the vmevent. */
- memset(&J->cur, 0, sizeof(Trace));
+ memset(&J->cur, 0, sizeof(GCtrace));
+ J->cur.traceno = J->curtrace;
J->cur.nins = J->cur.nk = REF_BASE;
J->cur.ir = J->irbuf;
J->cur.snap = J->snapbuf;
J->mergesnap = 0;
J->needsnap = 0;
J->guardemit.irt = 0;
+ setgcref(J->cur.startpt, obj2gco(J->pt));
L = J->L;
lj_vmevent_send(L, TRACE,
case BC_JMP:
/* Patch exit branch in parent to side trace entry. */
lua_assert(J->parent != 0 && J->cur.root != 0);
- lj_asm_patchexit(J, J->trace[J->parent], J->exitno, J->cur.mcode);
+ lj_asm_patchexit(J, traceref(J, J->parent), J->exitno, J->cur.mcode);
/* Avoid compiling a side trace twice (stack resizing uses parent exit). */
- J->trace[J->parent]->snap[J->exitno].count = SNAPCOUNT_DONE;
+ traceref(J, J->parent)->snap[J->exitno].count = SNAPCOUNT_DONE;
/* Add to side trace chain in root trace. */
{
- Trace *root = J->trace[J->cur.root];
+ GCtrace *root = traceref(J, J->cur.root);
root->nchild++;
J->cur.nextside = root->nextside;
root->nextside = (TraceNo1)J->curtrace;
copyTV(L, L->top++, &J->errinfo);
);
/* Drop aborted trace after the vmevent (which may still access it). */
- J->trace[J->curtrace] = NULL;
+ setgcrefnull(J->trace[J->curtrace]);
if (J->curtrace < J->freetrace)
J->freetrace = J->curtrace;
J->curtrace = 0;
/* Check for a hot side exit. If yes, start recording a side trace. */
static void trace_hotside(jit_State *J, const BCIns *pc)
{
- SnapShot *snap = &J->trace[J->parent]->snap[J->exitno];
+ SnapShot *snap = &traceref(J, J->parent)->snap[J->exitno];
if (!(J2G(J)->hookmask & (HOOK_GC|HOOK_VMEVENT)) &&
snap->count != SNAPCOUNT_DONE &&
++snap->count >= J->param[JIT_P_hotexit]) {
else
trace_hotside(J, pc);
if (bc_op(*pc) == BC_JLOOP) {
- BCIns *retpc = &J->trace[bc_d(*pc)]->startins;
+ BCIns *retpc = &traceref(J, bc_d(*pc))->startins;
if (bc_isret(bc_op(*retpc))) {
if (J->state == LJ_TRACE_RECORD) {
J->patchins = *pc;
LJ_FUNC_NORET void lj_trace_err_info(jit_State *J, TraceError e);
/* Trace management. */
-LJ_FUNC void lj_trace_freeproto(global_State *g, GCproto *pt);
+LJ_FUNC void LJ_FASTCALL lj_trace_free(global_State *g, GCtrace *T);
LJ_FUNC void lj_trace_reenableproto(GCproto *pt);
LJ_FUNC void lj_trace_flushproto(global_State *g, GCproto *pt);
-LJ_FUNC int lj_trace_flush(jit_State *J, TraceNo traceno);
+LJ_FUNC void lj_trace_flush(jit_State *J, TraceNo traceno);
LJ_FUNC int lj_trace_flushall(lua_State *L);
LJ_FUNC void lj_trace_initstate(global_State *g);
LJ_FUNC void lj_trace_freestate(global_State *g);
#define lj_trace_flushall(L) (UNUSED(L), 0)
#define lj_trace_initstate(g) UNUSED(g)
#define lj_trace_freestate(g) UNUSED(g)
-#define lj_trace_freeproto(g, pt) (UNUSED(g), UNUSED(pt), (void)0)
#define lj_trace_abort(g) UNUSED(g)
#define lj_trace_end(J) UNUSED(J)
projects / thirdparty / LuaJIT.git / commitdiff
