Actually NaN tagging with 47 bit pointers and 13+4 bit tags.
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
-#if LJ_64
+#if LJ_64 && !LJ_GC64
/* Undocumented, but hey, that's what we all love so much about Windows. */
typedef long (*PNTAVM)(HANDLE handle, void **addr, ULONG zbits,
#endif
#define MMAP_FLAGS (MAP_PRIVATE|MAP_ANONYMOUS)
-#if LJ_64
-/* 64 bit mode needs special support for allocating memory in the lower 2GB. */
+#if LJ_64 && !LJ_GC64
+/* 64 bit mode with 32 bit pointers needs special support for allocating
+** memory in the lower 2GB.
+*/
#if defined(MAP_32BIT)
#else
-/* 32 bit mode is easy. */
+/* 32 bit mode and GC64 mode is easy. */
static LJ_AINLINE void *CALL_MMAP(size_t size)
{
int olderr = errno;
#define CALL_MREMAP(addr, osz, nsz, mv) CALL_MREMAP_((addr), (osz), (nsz), (mv))
#define CALL_MREMAP_NOMOVE 0
#define CALL_MREMAP_MAYMOVE 1
-#if LJ_64
+#if LJ_64 && !LJ_GC64
#define CALL_MREMAP_MV CALL_MREMAP_NOMOVE
#else
#define CALL_MREMAP_MV CALL_MREMAP_MAYMOVE
cTValue *o = index2adr(L, idx);
if (tvisnumber(o)) {
return LUA_TNUMBER;
-#if LJ_64
+#if LJ_64 && !LJ_GC64
} else if (tvislightud(o)) {
return LUA_TLIGHTUSERDATA;
#endif
return 0;
} else if (tvispri(o1)) {
return o1 != niltv(L) && o2 != niltv(L);
-#if LJ_64
+#if LJ_64 && !LJ_GC64
} else if (tvislightud(o1)) {
return o1->u64 == o2->u64;
#endif
#endif
#endif
+/* 64 bit GC references. */
+#if LJ_TARGET_GC64
+#define LJ_GC64 1
+#else
+#define LJ_GC64 0
+#endif
+
/* 2-slot frame info. */
+#if LJ_GC64
+#define LJ_FR2 1
+#else
#define LJ_FR2 0
+#endif
/* Disable or enable the JIT compiler. */
-#if defined(LUAJIT_DISABLE_JIT) || defined(LJ_ARCH_NOJIT) || defined(LJ_OS_NOJIT) || LJ_FR2
+#if defined(LUAJIT_DISABLE_JIT) || defined(LJ_ARCH_NOJIT) || defined(LJ_OS_NOJIT) || LJ_FR2 || LJ_GC64
#define LJ_HASJIT 0
#else
#define LJ_HASJIT 1
/* Various VM limits. */
#define LJ_MAX_MEM32 0x7fffff00 /* Max. 32 bit memory allocation. */
-#define LJ_MAX_MEM LJ_MAX_MEM32 /* Max. total memory allocation. */
+#define LJ_MAX_MEM64 ((uint64_t)1<<47) /* Max. 64 bit memory allocation. */
+/* Max. total memory allocation. */
+#define LJ_MAX_MEM (LJ_GC64 ? LJ_MAX_MEM64 : LJ_MAX_MEM32)
#define LJ_MAX_ALLOC LJ_MAX_MEM /* Max. individual allocation length. */
#define LJ_MAX_STR LJ_MAX_MEM32 /* Max. string length. */
#define LJ_MAX_BUF LJ_MAX_MEM32 /* Max. buffer length. */
#define LJ_MAX_UPVAL 60 /* Max. # of upvalues. */
#define LJ_MAX_IDXCHAIN 100 /* __index/__newindex chain limit. */
-#define LJ_STACK_EXTRA 5 /* Extra stack space (metamethods). */
+#define LJ_STACK_EXTRA (5+2*LJ_FR2) /* Extra stack space (metamethods). */
#define LJ_NUM_CBPAGE 1 /* Number of FFI callback pages. */
#define checki32(x) ((x) == (int32_t)(x))
#define checku32(x) ((x) == (uint32_t)(x))
#define checkptr32(x) ((uintptr_t)(x) == (uint32_t)(uintptr_t)(x))
-#define checkptrGC(x) (checkptr32(x))
+#define checkptr47(x) (((uint64_t)(x) >> 47) == 0)
+#if LJ_GC64
+#define checkptrGC(x) (checkptr47((x)))
+#elif LJ_64
+#define checkptrGC(x) (checkptr32((x)))
+#else
+#define checkptrGC(x) 1
+#endif
/* Every half-decent C compiler transforms this into a rotate instruction. */
#define lj_rol(x, n) (((x)<<(n)) | ((x)>>(-(int)(n)&(8*sizeof(x)-1))))
uint32_t t;
if (tvisnumber(&rd->argv[0]))
t = ~LJ_TNUMX;
- else if (LJ_64 && tvislightud(&rd->argv[0]))
+ else if (LJ_64 && !LJ_GC64 && tvislightud(&rd->argv[0]))
t = ~LJ_TLIGHTUD;
else
t = ~itype(&rd->argv[0]);
{
IRIns *ir, *cir = J->cur.ir;
IRRef ref;
+ lua_assert(!LJ_GC64); /* TODO_GC64: major changes required. */
lua_assert(!isdead(J2G(J), o));
for (ref = J->chain[IR_KGC]; ref; ref = cir[ref].prev)
if (ir_kgc(&cir[ref]) == o)
IRT_PTR = LJ_64 ? IRT_P64 : IRT_P32,
IRT_INTP = LJ_64 ? IRT_I64 : IRT_INT,
IRT_UINTP = LJ_64 ? IRT_U64 : IRT_U32,
+ /* TODO_GC64: major changes required for all uses of IRT_P32. */
/* Additional flags. */
IRT_MARK = 0x20, /* Marker for misc. purposes. */
#define irt_isaddr(t) (irt_typerange((t), IRT_LIGHTUD, IRT_UDATA))
#define irt_isint64(t) (irt_typerange((t), IRT_I64, IRT_U64))
-#if LJ_64
+#if LJ_GC64
+#define IRT_IS64 \
+ ((1u<<IRT_NUM)|(1u<<IRT_I64)|(1u<<IRT_U64)|(1u<<IRT_P64)|\
+ (1u<<IRT_LIGHTUD)|(1u<<IRT_STR)|(1u<<IRT_THREAD)|(1u<<IRT_PROTO)|\
+ (1u<<IRT_FUNC)|(1u<<IRT_CDATA)|(1u<<IRT_TAB)|(1u<<IRT_UDATA))
+#elif LJ_64
#define IRT_IS64 \
((1u<<IRT_NUM)|(1u<<IRT_I64)|(1u<<IRT_U64)|(1u<<IRT_P64)|(1u<<IRT_LIGHTUD))
#else
return IRT_INT;
else if (tvisnum(tv))
return IRT_NUM;
-#if LJ_64
+#if LJ_64 && !LJ_GC64
else if (tvislightud(tv))
return IRT_LIGHTUD;
#endif
MRef ptr; /* Pointer constant (overlaps op12). */
} IRIns;
+/* TODO_GC64: major changes required. */
#define ir_kgc(ir) check_exp((ir)->o == IR_KGC, gcref((ir)->gcr))
#define ir_kstr(ir) (gco2str(ir_kgc((ir))))
#define ir_ktab(ir) (gco2tab(ir_kgc((ir))))
uint8_t topslot; /* Top stack slot already checked to be allocated. */
uint8_t linktype; /* Type of link. */
IRRef nins; /* Next IR instruction. Biased with REF_BIAS. */
+#if LJ_GC64
+ uint32_t unused_gc64;
+#endif
GCRef gclist;
IRIns *ir; /* IR instructions/constants. Biased with REF_BIAS. */
IRRef nk; /* Lowest IR constant. Biased with REF_BIAS. */
LJ_FUNC int lj_lib_checkopt(lua_State *L, int narg, int def, const char *lst);
/* Avoid including lj_frame.h. */
-#if LJ_FR2
+#if LJ_GC64
+#define lj_lib_upvalue(L, n) \
+ (&gcval(L->base-2)->fn.c.upvalue[(n)-1])
+#elif LJ_FR2
#define lj_lib_upvalue(L, n) \
(&gcref((L->base-2)->gcr)->fn.c.upvalue[(n)-1])
#else
/* Memory and GC object sizes. */
typedef uint32_t MSize;
+#if LJ_GC64
+typedef uint64_t GCSize;
+#else
typedef uint32_t GCSize;
+#endif
/* Memory reference */
typedef struct MRef {
+#if LJ_GC64
+ uint64_t ptr64; /* True 64 bit pointer. */
+#else
uint32_t ptr32; /* Pseudo 32 bit pointer. */
+#endif
} MRef;
+#if LJ_GC64
+#define mref(r, t) ((t *)(void *)(r).ptr64)
+
+#define setmref(r, p) ((r).ptr64 = (uint64_t)(void *)(p))
+#define setmrefr(r, v) ((r).ptr64 = (v).ptr64)
+#else
#define mref(r, t) ((t *)(void *)(uintptr_t)(r).ptr32)
#define setmref(r, p) ((r).ptr32 = (uint32_t)(uintptr_t)(void *)(p))
#define setmrefr(r, v) ((r).ptr32 = (v).ptr32)
+#endif
/* -- GC object references (32 bit address space) ------------------------- */
/* GCobj reference */
typedef struct GCRef {
+#if LJ_GC64
+ uint64_t gcptr64; /* True 64 bit pointer. */
+#else
uint32_t gcptr32; /* Pseudo 32 bit pointer. */
+#endif
} GCRef;
/* Common GC header for all collectable objects. */
#define GCHeader GCRef nextgc; uint8_t marked; uint8_t gct
/* This occupies 6 bytes, so use the next 2 bytes for non-32 bit fields. */
+#if LJ_GC64
+#define gcref(r) ((GCobj *)(r).gcptr64)
+#define gcrefp(r, t) ((t *)(void *)(r).gcptr64)
+#define gcrefu(r) ((r).gcptr64)
+#define gcrefeq(r1, r2) ((r1).gcptr64 == (r2).gcptr64)
+
+#define setgcref(r, gc) ((r).gcptr64 = (uint64_t)&(gc)->gch)
+#define setgcreft(r, gc, it) \
+ (r).gcptr64 = (uint64_t)&(gc)->gch | (((uint64_t)(it)) << 47)
+#define setgcrefp(r, p) ((r).gcptr64 = (uint64_t)(p))
+#define setgcrefnull(r) ((r).gcptr64 = 0)
+#define setgcrefr(r, v) ((r).gcptr64 = (v).gcptr64)
+#else
#define gcref(r) ((GCobj *)(uintptr_t)(r).gcptr32)
#define gcrefp(r, t) ((t *)(void *)(uintptr_t)(r).gcptr32)
#define gcrefu(r) ((r).gcptr32)
#define gcrefeq(r1, r2) ((r1).gcptr32 == (r2).gcptr32)
-#define gcnext(gc) (gcref((gc)->gch.nextgc))
#define setgcref(r, gc) ((r).gcptr32 = (uint32_t)(uintptr_t)&(gc)->gch)
#define setgcrefp(r, p) ((r).gcptr32 = (uint32_t)(uintptr_t)(p))
#define setgcrefnull(r) ((r).gcptr32 = 0)
#define setgcrefr(r, v) ((r).gcptr32 = (v).gcptr32)
+#endif
+
+#define gcnext(gc) (gcref((gc)->gch.nextgc))
/* IMPORTANT NOTE:
**
typedef LJ_ALIGN(8) union TValue {
uint64_t u64; /* 64 bit pattern overlaps number. */
lua_Number n; /* Number object overlaps split tag/value object. */
+#if LJ_GC64
+ GCRef gcr; /* GCobj reference with tag. */
+ int64_t it64;
+#endif
struct {
LJ_ENDIAN_LOHI(
union {
+#if !LJ_GC64
GCRef gcr; /* GCobj reference (if any). */
+#endif
int32_t i; /* Integer value. */
};
, uint32_t it; /* Internal object tag. Must overlap MSW of number. */
/* Internal object tags.
**
+** Format for 32 bit GC references (!LJ_GC64):
+**
** Internal tags overlap the MSW of a number object (must be a double).
** Interpreted as a double these are special NaNs. The FPU only generates
** one type of NaN (0xfff8_0000_0000_0000). So MSWs > 0xfff80000 are available
** int (LJ_DUALNUM)| itype | int |
** number -------double------
**
+** Format for 64 bit GC references (LJ_GC64):
+**
+** The upper 13 bits must be 1 (0xfff8...) for a special NaN. The next
+** 4 bits hold the internal tag. The lowest 47 bits either hold a pointer,
+** a zero-extended 32 bit integer or all bits set to 1 for primitive types.
+**
+** ------MSW------.------LSW------
+** primitive types |1..1|itype|1..................1|
+** GC objects/lightud |1..1|itype|-------GCRef--------|
+** int (LJ_DUALNUM) |1..1|itype|0..0|-----int-------|
+** number ------------double-------------
+**
** ORDER LJ_T
** Primitive types nil/false/true must be first, lightuserdata next.
** GC objects are at the end, table/userdata must be lowest.
#define LJ_TNUMX (~13u)
/* Integers have itype == LJ_TISNUM doubles have itype < LJ_TISNUM */
-#if LJ_64
+#if LJ_64 && !LJ_GC64
#define LJ_TISNUM 0xfffeffffu
#else
#define LJ_TISNUM LJ_TNUMX
#define LJ_TISGCV (LJ_TSTR+1)
#define LJ_TISTABUD LJ_TTAB
+#if LJ_GC64
+#define LJ_GCVMASK (((uint64_t)1 << 47) - 1)
+#endif
+
/* -- String object ------------------------------------------------------- */
/* String object header. String payload follows. */
uint8_t numparams; /* Number of parameters. */
uint8_t framesize; /* Fixed frame size. */
MSize sizebc; /* Number of bytecode instructions. */
+#if LJ_GC64
+ uint32_t unused_gc64;
+#endif
GCRef gclist;
MRef k; /* Split constant array (points to the middle). */
MRef uv; /* Upvalue list. local slot|0x8000 or parent uv idx. */
TValue val; /* Value object. Must be first field. */
TValue key; /* Key object. */
MRef next; /* Hash chain. */
+#if !LJ_GC64
MRef freetop; /* Top of free elements (stored in t->node[0]). */
+#endif
} Node;
LJ_STATIC_ASSERT(offsetof(Node, val) == 0);
MRef node; /* Hash part. */
uint32_t asize; /* Size of array part (keys [0, asize-1]). */
uint32_t hmask; /* Hash part mask (size of hash part - 1). */
+#if LJ_GC64
+ MRef freetop; /* Top of free elements. */
+#endif
} GCtab;
#define sizetabcolo(n) ((n)*sizeof(TValue) + sizeof(GCtab))
#define tabref(r) (&gcref((r))->tab)
#define noderef(r) (mref((r), Node))
#define nextnode(n) (mref((n)->next, Node))
+#if LJ_GC64
+#define getfreetop(t, n) (noderef((t)->freetop))
+#define setfreetop(t, n, v) (setmref((t)->freetop, (v)))
+#else
+#define getfreetop(t, n) (noderef((n)->freetop))
+#define setfreetop(t, n, v) (setmref((n)->freetop, (v)))
+#endif
/* -- State objects ------------------------------------------------------- */
#define registry(L) (&G(L)->registrytv)
/* Macros to access the currently executing (Lua) function. */
-#if LJ_FR2
+#if LJ_GC64
+#define curr_func(L) (&gcval(L->base-2)->fn)
+#elif LJ_FR2
#define curr_func(L) (&gcref((L->base-2)->gcr)->fn)
#else
#define curr_func(L) (&gcref((L->base-1)->fr.func)->fn)
#endif
/* Macros to test types. */
+#if LJ_GC64
+#define itype(o) ((uint32_t)((o)->it64 >> 47))
+#define tvisnil(o) ((o)->it64 == -1)
+#else
#define itype(o) ((o)->it)
#define tvisnil(o) (itype(o) == LJ_TNIL)
+#endif
#define tvisfalse(o) (itype(o) == LJ_TFALSE)
#define tvistrue(o) (itype(o) == LJ_TTRUE)
#define tvisbool(o) (tvisfalse(o) || tvistrue(o))
-#if LJ_64
+#if LJ_64 && !LJ_GC64
#define tvislightud(o) (((int32_t)itype(o) >> 15) == -2)
#else
#define tvislightud(o) (itype(o) == LJ_TLIGHTUD)
#define rawnumequal(o1, o2) ((o1)->u64 == (o2)->u64)
/* Macros to convert type ids. */
-#if LJ_64
+#if LJ_64 && !LJ_GC64
#define itypemap(o) \
(tvisnumber(o) ? ~LJ_TNUMX : tvislightud(o) ? ~LJ_TLIGHTUD : ~itype(o))
#else
#endif
/* Macros to get tagged values. */
+#if LJ_GC64
+#define gcval(o) ((GCobj *)(gcrefu((o)->gcr) & LJ_GCVMASK))
+#else
#define gcval(o) (gcref((o)->gcr))
-#define boolV(o) check_exp(tvisbool(o), (LJ_TFALSE - (o)->it))
+#endif
+#define boolV(o) check_exp(tvisbool(o), (LJ_TFALSE - itype(o)))
#if LJ_64
#define lightudV(o) \
check_exp(tvislightud(o), (void *)((o)->u64 & U64x(00007fff,ffffffff)))
#define intV(o) check_exp(tvisint(o), (int32_t)(o)->i)
/* Macros to set tagged values. */
+#if LJ_GC64
+#define setitype(o, i) ((o)->it = ((i) << 15))
+#define setnilV(o) ((o)->it64 = -1)
+#define setpriV(o, x) ((o)->it64 = (int64_t)~((uint64_t)~(x)<<47))
+#define setboolV(o, x) ((o)->it64 = (int64_t)~((uint64_t)((x)+1)<<47))
+#else
#define setitype(o, i) ((o)->it = (i))
#define setnilV(o) ((o)->it = LJ_TNIL)
#define setboolV(o, x) ((o)->it = LJ_TFALSE-(uint32_t)(x))
#define setpriV(o, i) (setitype((o), (i)))
+#endif
static LJ_AINLINE void setlightudV(TValue *o, void *p)
{
-#if LJ_64
+#if LJ_GC64
+ o->u64 = (uint64_t)p | (((uint64_t)LJ_TLIGHTUD) << 47);
+#elif LJ_64
o->u64 = (uint64_t)p | (((uint64_t)0xffff) << 48);
#else
setgcrefp(o->gcr, p); setitype(o, LJ_TLIGHTUD);
static LJ_AINLINE void setgcVraw(TValue *o, GCobj *v, uint32_t itype)
{
+#if LJ_GC64
+ setgcreft(o->gcr, v, itype);
+#else
setgcref(o->gcr, v); setitype(o, itype);
+#endif
}
static LJ_AINLINE void setgcV(lua_State *L, TValue *o, GCobj *v, uint32_t it)
}
if (LJ_UNLIKELY(bloomtest(rfilt, ref)))
rs = snap_renameref(T, snapno, ref, rs);
+ lua_assert(!LJ_GC64); /* TODO_GC64: handle 64 bit references. */
if (ra_hasspill(regsp_spill(rs))) { /* Restore from spill slot. */
int32_t *sps = &ex->spill[regsp_spill(rs)];
if (irt_isinteger(t)) {
setnilV(registry(L));
setnilV(&g->nilnode.val);
setnilV(&g->nilnode.key);
+#if !LJ_GC64
setmref(g->nilnode.freetop, &g->nilnode);
+#endif
lj_buf_init(NULL, &g->tmpbuf);
g->gc.state = GCSpause;
setgcref(g->gc.root, obj2gco(L));
#define hashlohi(t, lo, hi) hashmask((t), hashrot((lo), (hi)))
#define hashnum(t, o) hashlohi((t), (o)->u32.lo, ((o)->u32.hi << 1))
#define hashptr(t, p) hashlohi((t), u32ptr(p), u32ptr(p) + HASH_BIAS)
+#if LJ_GC64
+#define hashgcref(t, r) \
+ hashlohi((t), (uint32_t)gcrefu(r), (uint32_t)(gcrefu(r) >> 32))
+#else
#define hashgcref(t, r) hashlohi((t), gcrefu(r), gcrefu(r) + HASH_BIAS)
+#endif
/* Hash an arbitrary key and return its anchor position in the hash table. */
static Node *hashkey(const GCtab *t, cTValue *key)
lj_err_msg(L, LJ_ERR_TABOV);
hsize = 1u << hbits;
node = lj_mem_newvec(L, hsize, Node);
- setmref(node->freetop, &node[hsize]);
setmref(t->node, node);
+ setfreetop(t, node, &node[hsize]);
t->hmask = hsize-1;
}
GCtab *t;
/* First try to colocate the array part. */
if (LJ_MAX_COLOSIZE != 0 && asize > 0 && asize <= LJ_MAX_COLOSIZE) {
+ Node *nilnode;
lua_assert((sizeof(GCtab) & 7) == 0);
t = (GCtab *)lj_mem_newgco(L, sizetabcolo(asize));
t->gct = ~LJ_TTAB;
setgcrefnull(t->metatable);
t->asize = asize;
t->hmask = 0;
- setmref(t->node, &G(L)->nilnode);
+ nilnode = &G(L)->nilnode;
+ setmref(t->node, nilnode);
+#if LJ_GC64
+ setmref(t->freetop, nilnode);
+#endif
} else { /* Otherwise separately allocate the array part. */
+ Node *nilnode;
t = lj_mem_newobj(L, GCtab);
t->gct = ~LJ_TTAB;
t->nomm = (uint8_t)~0;
setgcrefnull(t->metatable);
t->asize = 0; /* In case the array allocation fails. */
t->hmask = 0;
- setmref(t->node, &G(L)->nilnode);
+ nilnode = &G(L)->nilnode;
+ setmref(t->node, nilnode);
+#if LJ_GC64
+ setmref(t->freetop, nilnode);
+#endif
if (asize > 0) {
if (asize > LJ_MAX_ASIZE)
lj_err_msg(L, LJ_ERR_TABOV);
Node *node = noderef(t->node);
Node *knode = noderef(kt->node);
ptrdiff_t d = (char *)node - (char *)knode;
- setmref(node->freetop, (Node *)((char *)noderef(knode->freetop) + d));
+ setfreetop(t, node, (Node *)((char *)getfreetop(kt, knode) + d));
for (i = 0; i <= hmask; i++) {
Node *kn = &knode[i];
Node *n = &node[i];
clearapart(t);
if (t->hmask > 0) {
Node *node = noderef(t->node);
- setmref(node->freetop, &node[t->hmask+1]);
+ setfreetop(t, node, &node[t->hmask+1]);
clearhpart(t);
}
}
} else {
global_State *g = G(L);
setmref(t->node, &g->nilnode);
+#if LJ_GC64
+ setmref(t->freetop, &g->nilnode);
+#endif
t->hmask = 0;
}
if (asize < oldasize) { /* Array part shrinks? */
Node *n = hashkey(t, key);
if (!tvisnil(&n->val) || t->hmask == 0) {
Node *nodebase = noderef(t->node);
- Node *collide, *freenode = noderef(nodebase->freetop);
+ Node *collide, *freenode = getfreetop(t, nodebase);
lua_assert(freenode >= nodebase && freenode <= nodebase+t->hmask+1);
do {
if (freenode == nodebase) { /* No free node found? */
return lj_tab_set(L, t, key); /* Retry key insertion. */
}
} while (!tvisnil(&(--freenode)->key));
- setmref(nodebase->freetop, freenode);
+ setfreetop(t, nodebase, freenode);
lua_assert(freenode != &G(L)->nilnode);
collide = hashkey(t, &n->key);
if (collide != n) { /* Colliding node not the main node? */
projects / thirdparty / LuaJIT.git / commitdiff
