Add stack check to pcall/xpcall.

Analyzed by Peter Cawley. #1048

diff --git a/src/vm_arm.dasc b/src/vm_arm.dasc
index 7dae1a53dd327a07dc991c56d76cfb66da058d1e..872de45a91c29917d302310fd8f2abc0a76620d2 100644 (file)
--- a/src/vm_arm.dasc
+++ b/src/vm_arm.dasc
@@ -1155,8 +1155,11 @@ static void build_subroutines(BuildCtx *ctx)
   |//-- Base library: catch errors ----------------------------------------
   |
   |.ffunc pcall
+  |   ldr RB, L->maxstack
+  |   add INS, BASE, NARGS8:RC
   |  ldrb RA, [DISPATCH, #DISPATCH_GL(hookmask)]
   |   cmp NARGS8:RC, #8
+  |   cmphs RB, INS
   |   blo ->fff_fallback
   |  tst RA, #HOOK_ACTIVE              // Remember active hook before pcall.
   |   mov RB, BASE
@@ -1167,7 +1170,11 @@ static void build_subroutines(BuildCtx *ctx)
   |  b ->vm_call_dispatch
   |
   |.ffunc_2 xpcall
+  |   ldr RB, L->maxstack
+  |   add INS, BASE, NARGS8:RC
   |  ldrb RA, [DISPATCH, #DISPATCH_GL(hookmask)]
+  |   cmp RB, INS
+  |   blo ->fff_fallback
   |  checkfunc CARG4, ->fff_fallback   // Traceback must be a function.
   |   mov RB, BASE
   |  strd CARG12, [BASE, #8]           // Swap function and traceback.

diff --git a/src/vm_mips.dasc b/src/vm_mips.dasc
index f6f801f206bc48e34ccdaf33f795cbba02d44457..c4c0a416f24f6d8a0631d25445d6102dec27243b 100644 (file)
--- a/src/vm_mips.dasc
+++ b/src/vm_mips.dasc
@@ -1244,9 +1244,13 @@ static void build_subroutines(BuildCtx *ctx)
   |//-- Base library: catch errors ----------------------------------------
   |
   |.ffunc pcall
+  |   lw TMP1, L->maxstack
+  |   addu TMP2, BASE, NARGS8:RC
   |  lbu TMP3, DISPATCH_GL(hookmask)(DISPATCH)
   |  beqz NARGS8:RC, ->fff_fallback
-  |   move TMP2, BASE
+  |.  sltu AT, TMP1, TMP2
+  |   bnez AT, ->fff_fallback
+  |.   move TMP2, BASE
   |   addiu BASE, BASE, 8
   |  // Remember active hook before pcall.
   |  srl TMP3, TMP3, HOOK_ACTIVE_SHIFT
@@ -1256,8 +1260,12 @@ static void build_subroutines(BuildCtx *ctx)
   |.  addiu NARGS8:RC, NARGS8:RC, -8
   |
   |.ffunc xpcall
+  |   lw TMP1, L->maxstack
+  |   addu TMP2, BASE, NARGS8:RC
   |    sltiu AT, NARGS8:RC, 16
   |  lw CARG4, 8+HI(BASE)
+  |   sltu TMP1, TMP1, TMP2
+  |    or AT, AT, TMP1
   |    bnez AT, ->fff_fallback
   |.  ldc1 FARG2, 8(BASE)
   |   ldc1 FARG1, 0(BASE)

diff --git a/src/vm_ppc.dasc b/src/vm_ppc.dasc
index 61ebbb04121f2465d72c915d931e2ddcde39cf85..d6792f2cc59b8a86f768034b611e50bb3fcb1dab 100644 (file)
--- a/src/vm_ppc.dasc
+++ b/src/vm_ppc.dasc
@@ -1537,8 +1537,12 @@ static void build_subroutines(BuildCtx *ctx)
   |//-- Base library: catch errors ----------------------------------------
   |
   |.ffunc pcall
+  |    lwz TMP1, L->maxstack
+  |    add TMP2, BASE, NARGS8:RC
   |  cmplwi NARGS8:RC, 8
   |   lbz TMP3, DISPATCH_GL(hookmask)(DISPATCH)
+  |    cmplw cr1, TMP1, TMP2
+  |  cror 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
   |  blt ->fff_fallback
   |   mr TMP2, BASE
   |   la BASE, 8(BASE)
@@ -1549,9 +1553,13 @@ static void build_subroutines(BuildCtx *ctx)
   |  b ->vm_call_dispatch
   |
   |.ffunc xpcall
+  |     lwz TMP1, L->maxstack
+  |     add TMP2, BASE, NARGS8:RC
   |  cmplwi NARGS8:RC, 16
   |   lwz CARG4, 8(BASE)
+  |     cmplw cr1, TMP1, TMP2
   |    lfd FARG2, 8(BASE)
+  |  cror 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
   |    lfd FARG1, 0(BASE)
   |  blt ->fff_fallback
   |  lbz TMP1, DISPATCH_GL(hookmask)(DISPATCH)

diff --git a/src/vm_ppcspe.dasc b/src/vm_ppcspe.dasc
index c4a4419170b0571c09e95b561aa446e8e27e36c9..ea33c08b7dc7926aa427af8ac6577fbc2228ccfa 100644 (file)
--- a/src/vm_ppcspe.dasc
+++ b/src/vm_ppcspe.dasc
@@ -1184,8 +1184,12 @@ static void build_subroutines(BuildCtx *ctx)
   |//-- Base library: catch errors ----------------------------------------
   |
   |.ffunc pcall
+  |    lwz TMP1, L->maxstack
+  |    add TMP2, BASE, NARGS8:RC
   |  cmplwi NARGS8:RC, 8
   |   lbz TMP3, DISPATCH_GL(hookmask)(DISPATCH)
+  |    cmplw cr1, TMP1, TMP2
+  |  cror 4*cr0+lt, 4*cr0+lt, 4*cr1+lt
   |  blt ->fff_fallback
   |   mr TMP2, BASE
   |   la BASE, 8(BASE)
@@ -1196,8 +1200,12 @@ static void build_subroutines(BuildCtx *ctx)
   |  b ->vm_call_dispatch
   |
   |.ffunc_2 xpcall
+  |    lwz TMP1, L->maxstack
+  |    add TMP2, BASE, NARGS8:RC
   |  lbz TMP3, DISPATCH_GL(hookmask)(DISPATCH)
   |   mr TMP2, BASE
+  |    cmplw TMP1, TMP2
+  |    blt ->fff_fallback
   |  checkfunc CARG2                   // Traceback must be a function.
   |  checkfail ->fff_fallback
   |   la BASE, 16(BASE)

diff --git a/src/vm_x86.dasc b/src/vm_x86.dasc
index 56712f9033a0b4d55053230fbcdca9fb8274cd06..811d5e750a524f3fb80ab09945a858532d256d54 100644 (file)
--- a/src/vm_x86.dasc
+++ b/src/vm_x86.dasc
@@ -1720,6 +1720,9 @@ static void build_subroutines(BuildCtx *ctx)
   |//-- Base library: catch errors ----------------------------------------
   |
   |.ffunc_1 pcall
+  |  mov L:RB, SAVE_L
+  |  lea RA, [BASE+NARGS:RD*8]
+  |  cmp RA, L:RB->maxstack; ja ->fff_fallback
   |  lea RA, [BASE+8]
   |  sub NARGS:RD, 1
   |  mov PC, 8+FRAME_PCALL
@@ -1731,6 +1734,9 @@ static void build_subroutines(BuildCtx *ctx)
   |  jmp ->vm_call_dispatch
   |
   |.ffunc_2 xpcall
+  |  mov L:RB, SAVE_L
+  |  lea RA, [BASE+NARGS:RD*8]
+  |  cmp RA, L:RB->maxstack; ja ->fff_fallback
   |  cmp dword [BASE+12], LJ_TFUNC;  jne ->fff_fallback
   |  mov RB, [BASE+4]                  // Swap function and traceback.
   |  mov [BASE+12], RB