]> git.ipfire.org Git - thirdparty/Python/cpython.git/commitdiff
GH-115802: JIT "small" code for Windows (GH-115964)
authorBrandt Bucher <brandtbucher@microsoft.com>
Thu, 29 Feb 2024 16:11:28 +0000 (08:11 -0800)
committerGitHub <noreply@github.com>
Thu, 29 Feb 2024 16:11:28 +0000 (08:11 -0800)
26 files changed:
Include/cpython/optimizer.h
Include/internal/pycore_ceval.h
Include/internal/pycore_dict.h
Include/internal/pycore_floatobject.h
Include/internal/pycore_function.h
Include/internal/pycore_genobject.h
Include/internal/pycore_intrinsics.h
Include/internal/pycore_list.h
Include/internal/pycore_long.h
Include/internal/pycore_object.h
Include/internal/pycore_optimizer.h
Include/internal/pycore_pyerrors.h
Include/internal/pycore_sliceobject.h
Include/internal/pycore_tuple.h
Include/internal/pycore_typeobject.h
Include/internal/pycore_unicodeobject.h
Python/bytecodes.c
Python/ceval.c
Python/ceval_macros.h
Python/executor_cases.c.h
Python/generated_cases.c.h
Python/jit.c
Tools/jit/_schema.py
Tools/jit/_stencils.py
Tools/jit/_targets.py
Tools/jit/template.c

index 8fc9fb62aebdb415ea4fd1bd77dfbd0ece2f1b83..6d7b8bc3c1433aa9116150bff0099d3de14e98b2 100644 (file)
@@ -92,9 +92,6 @@ PyAPI_FUNC(_PyOptimizerObject *) PyUnstable_GetOptimizer(void);
 
 PyAPI_FUNC(_PyExecutorObject *) PyUnstable_GetExecutor(PyCodeObject *code, int offset);
 
-int
-_PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer, _PyExecutorObject **exec_ptr);
-
 void _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *);
 void _Py_ExecutorClear(_PyExecutorObject *);
 void _Py_BloomFilter_Init(_PyBloomFilter *);
index bf77526cf75cc14fdf7322ab2e5a642dae8702ec..6eab2ba1daedf8f2a497939162f9dea31eec524b 100644 (file)
@@ -181,22 +181,26 @@ extern PyObject* _Py_MakeCoro(PyFunctionObject *func);
 
 /* Handle signals, pending calls, GIL drop request
    and asynchronous exception */
-extern int _Py_HandlePending(PyThreadState *tstate);
+PyAPI_FUNC(int) _Py_HandlePending(PyThreadState *tstate);
 
 extern PyObject * _PyEval_GetFrameLocals(void);
 
-extern const binaryfunc _PyEval_BinaryOps[];
-int _PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right);
-int _PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right);
-int _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, PyObject **match, PyObject **rest);
-void _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg);
-void _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj);
-void _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg);
-void _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs);
-PyObject *_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs);
-PyObject *_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys);
-int _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, PyObject **sp);
-void _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame);
+typedef PyObject *(*conversion_func)(PyObject *);
+
+PyAPI_DATA(const binaryfunc) _PyEval_BinaryOps[];
+PyAPI_DATA(const conversion_func) _PyEval_ConversionFuncs[];
+
+PyAPI_FUNC(int) _PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right);
+PyAPI_FUNC(int) _PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right);
+PyAPI_FUNC(int) _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, PyObject **match, PyObject **rest);
+PyAPI_FUNC(void) _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg);
+PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj);
+PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg);
+PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs);
+PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs);
+PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys);
+PyAPI_FUNC(int) _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, PyObject **sp);
+PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame);
 
 
 /* Bits that can be set in PyThreadState.eval_breaker */
index d1a0010b9a81dd7d2451eb77a26c1dfe55dc7496..cd171a4384db5d920e1bb7041f1116048a9378a7 100644 (file)
@@ -52,7 +52,7 @@ PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *);
    of a key wins, if override is 2, a KeyError with conflicting key as
    argument is raised.
 */
-extern int _PyDict_MergeEx(PyObject *mp, PyObject *other, int override);
+PyAPI_FUNC(int) _PyDict_MergeEx(PyObject *mp, PyObject *other, int override);
 
 extern void _PyDict_DebugMallocStats(FILE *out);
 
@@ -100,10 +100,10 @@ extern Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t has
 
 extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *);
 extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key);
-extern PyObject *_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *);
+PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *);
 
 /* Consumes references to key and value */
-extern int _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value);
+PyAPI_FUNC(int) _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value);
 extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value);
 
 extern int _PyDict_Pop_KnownHash(
@@ -247,8 +247,8 @@ _PyDict_NotifyEvent(PyInterpreterState *interp,
 }
 
 extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values);
-extern bool _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv);
-extern PyObject *_PyDict_FromItems(
+PyAPI_FUNC(bool) _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv);
+PyAPI_FUNC(PyObject *)_PyDict_FromItems(
         PyObject *const *keys, Py_ssize_t keys_offset,
         PyObject *const *values, Py_ssize_t values_offset,
         Py_ssize_t length);
index 3767df5506d43fbdb4346222a4c17b3f2fc51f6f..f984df695696c3e6ffdf75a39fa11376499d3cca 100644 (file)
@@ -34,7 +34,7 @@ struct _Py_float_runtime_state {
 
 
 
-void _PyFloat_ExactDealloc(PyObject *op);
+PyAPI_FUNC(void) _PyFloat_ExactDealloc(PyObject *op);
 
 
 extern void _PyFloat_DebugMallocStats(FILE* out);
index 3f3da8a44b77e4c1bb031378c8af413ff1ee0d41..dad6a89af77dec92e16507ed75885e95505193c9 100644 (file)
@@ -29,7 +29,7 @@ struct _py_func_state {
 extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr);
 
 extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func);
-extern void _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version);
+PyAPI_FUNC(void) _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version);
 PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version);
 
 extern PyObject *_Py_set_function_type_params(
index b2aa017598409f7c857184d2b8ef7af7cb41de65..9463c822ad86698e64aced58adfc1088cb14ab29 100644 (file)
@@ -10,7 +10,7 @@ extern "C" {
 
 #include "pycore_freelist.h"
 
-extern PyObject *_PyGen_yf(PyGenObject *);
+PyAPI_FUNC(PyObject *)_PyGen_yf(PyGenObject *);
 extern void _PyGen_Finalize(PyObject *self);
 
 // Export for '_asyncio' shared extension
@@ -19,7 +19,7 @@ PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *);
 // Export for '_asyncio' shared extension
 PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **);
 
-extern PyObject *_PyCoro_GetAwaitableIter(PyObject *o);
+PyAPI_FUNC(PyObject *)_PyCoro_GetAwaitableIter(PyObject *o);
 extern PyObject *_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *);
 
 extern PyTypeObject _PyCoroWrapper_Type;
index 3a8dd95cff8e5db677d9be16c5dfbcbdd9cfc821..8fa88ea3f74caa8c2c9d0318557147242c7909e6 100644 (file)
@@ -44,7 +44,7 @@ typedef struct {
     const char *name;
 } intrinsic_func2_info;
 
-extern const intrinsic_func1_info _PyIntrinsics_UnaryFunctions[];
-extern const intrinsic_func2_info _PyIntrinsics_BinaryFunctions[];
+PyAPI_DATA(const intrinsic_func1_info) _PyIntrinsics_UnaryFunctions[];
+PyAPI_DATA(const intrinsic_func2_info) _PyIntrinsics_BinaryFunctions[];
 
 #endif  // !Py_INTERNAL_INTRINSIC_H
index 50dc13c4da4487caa41e5550187824cd2a2161d5..2a82912e41d55715cabb5149402e2f9f075899fb 100644 (file)
@@ -10,12 +10,12 @@ extern "C" {
 
 #include "pycore_freelist.h"  // _PyFreeListState
 
-extern PyObject* _PyList_Extend(PyListObject *, PyObject *);
+PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *);
 extern void _PyList_DebugMallocStats(FILE *out);
 
 #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item)
 
-extern int
+PyAPI_FUNC(int)
 _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem);
 
 // In free-threaded build: self should be locked by the caller, if it should be thread-safe.
@@ -54,7 +54,7 @@ typedef struct {
     PyListObject *it_seq; /* Set to NULL when iterator is exhausted */
 } _PyListIterObject;
 
-extern PyObject *_PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n);
+PyAPI_FUNC(PyObject *)_PyList_FromArraySteal(PyObject *const *src, Py_ssize_t n);
 
 #ifdef __cplusplus
 }
index ec27df9e416c58c295cae351bce338fcb4ff7951..f04f66d053bab9daeb66c89bdcc07790eac2e8c2 100644 (file)
@@ -121,9 +121,9 @@ PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, size_t);
 // Export for 'math' shared extension
 PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, size_t);
 
-extern PyObject* _PyLong_Add(PyLongObject *left, PyLongObject *right);
-extern PyObject* _PyLong_Multiply(PyLongObject *left, PyLongObject *right);
-extern PyObject* _PyLong_Subtract(PyLongObject *left, PyLongObject *right);
+PyAPI_FUNC(PyObject*) _PyLong_Add(PyLongObject *left, PyLongObject *right);
+PyAPI_FUNC(PyObject*) _PyLong_Multiply(PyLongObject *left, PyLongObject *right);
+PyAPI_FUNC(PyObject*) _PyLong_Subtract(PyLongObject *left, PyLongObject *right);
 
 // Export for 'binascii' shared extension.
 PyAPI_DATA(unsigned char) _PyLong_DigitValue[256];
index 34a83ea228e8b10e0f74a1362a39cfe02625ad1d..9809f5f2e0271ae70acdef06dfe77b68b97aa5aa 100644 (file)
@@ -73,7 +73,7 @@ PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *);
         .ob_size = size                       \
     }
 
-extern void _Py_NO_RETURN _Py_FatalRefcountErrorFunc(
+PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc(
     const char *func,
     const char *message);
 
@@ -684,7 +684,7 @@ PyAPI_FUNC(PyObject*) _PyObject_LookupSpecial(PyObject *, PyObject *);
 
 extern int _PyObject_IsAbstract(PyObject *);
 
-extern int _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method);
+PyAPI_FUNC(int) _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method);
 extern PyObject* _PyObject_NextNotImplemented(PyObject *);
 
 // Pickle support.
index 614850468ec1d38c958056502128b5190a1144a4..4894f613b5fb9188820ac3423922b6adfbc80e2b 100644 (file)
@@ -111,6 +111,8 @@ extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx);
 
 PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored);
 
+PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer, _PyExecutorObject **exec_ptr);
+
 #ifdef __cplusplus
 }
 #endif
index 0f16fb894d17e12791c08bb897157ff75835206b..910335fd2cf33b7ed7fe5c3b0efc6e3a9ad8f267 100644 (file)
@@ -95,7 +95,7 @@ extern void _PyErr_Fetch(
 
 extern PyObject* _PyErr_GetRaisedException(PyThreadState *tstate);
 
-extern int _PyErr_ExceptionMatches(
+PyAPI_FUNC(int) _PyErr_ExceptionMatches(
     PyThreadState *tstate,
     PyObject *exc);
 
@@ -114,18 +114,18 @@ extern void _PyErr_SetObject(
 
 extern void _PyErr_ChainStackItem(void);
 
-extern void _PyErr_Clear(PyThreadState *tstate);
+PyAPI_FUNC(void) _PyErr_Clear(PyThreadState *tstate);
 
 extern void _PyErr_SetNone(PyThreadState *tstate, PyObject *exception);
 
 extern PyObject* _PyErr_NoMemory(PyThreadState *tstate);
 
-extern void _PyErr_SetString(
+PyAPI_FUNC(void) _PyErr_SetString(
     PyThreadState *tstate,
     PyObject *exception,
     const char *string);
 
-extern PyObject* _PyErr_Format(
+PyAPI_FUNC(PyObject*) _PyErr_Format(
     PyThreadState *tstate,
     PyObject *exception,
     const char *format,
index 89086f67683a2f5ca1da8d099218904221b3a6b6..ba8b1f1cb27dee3219436416586b2bb3e3b8d4e3 100644 (file)
@@ -11,7 +11,7 @@ extern "C" {
 
 /* runtime lifecycle */
 
-extern PyObject *
+PyAPI_FUNC(PyObject *)
 _PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop);
 
 #ifdef __cplusplus
index 4605f355ccbc38ffb8c28ea1fb48c387500725f3..14a9e42c3a324cd73bd0527fe5ba805f7a91d7b0 100644 (file)
@@ -21,7 +21,7 @@ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *);
 #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item)
 
 extern PyObject *_PyTuple_FromArray(PyObject *const *, Py_ssize_t);
-extern PyObject *_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t);
+PyAPI_FUNC(PyObject *)_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t);
 
 typedef struct {
     PyObject_HEAD
index 9134ab45cd003951f7a3627422768ec5f4ddb6a4..c214111fed6f97aa7271f9bac40bdbe24cc3c5ed 100644 (file)
@@ -147,7 +147,7 @@ extern PyObject* _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name);
 
 extern PyTypeObject _PyBufferWrapper_Type;
 
-extern PyObject* _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj,
+PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj,
                                  PyObject *name, int *meth_found);
 
 
index 7ee540154b23d88d0076f84d7e37882fe2c6d219..fea5ceea0954f4287ad69f13ee4b1cfef711d3d7 100644 (file)
@@ -31,7 +31,7 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
     PyObject *op,
     int check_content);
 
-extern void _PyUnicode_ExactDealloc(PyObject *op);
+PyAPI_FUNC(void) _PyUnicode_ExactDealloc(PyObject *op);
 extern Py_ssize_t _PyUnicode_InternedSize(void);
 
 // Get a copy of a Unicode string.
@@ -202,7 +202,7 @@ PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII(
 
 /* --- Methods & Slots ---------------------------------------------------- */
 
-extern PyObject* _PyUnicode_JoinArray(
+PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray(
     PyObject *separator,
     PyObject *const *items,
     Py_ssize_t seqlen
index 565379afc4b5a7057ed96e48709bf5d5c9e6f926..1d515098f6c7e920313254a3027904ad5ba96395 100644 (file)
@@ -2755,7 +2755,7 @@ dummy_func(
                 GOTO_ERROR(error);
             }
             DECREF_INPUTS();
-            res = _PyObject_CallNoArgsTstate(tstate, enter);
+            res = PyObject_CallNoArgs(enter);
             Py_DECREF(enter);
             if (res == NULL) {
                 Py_DECREF(exit);
@@ -2790,7 +2790,7 @@ dummy_func(
                 GOTO_ERROR(error);
             }
             DECREF_INPUTS();
-            res = _PyObject_CallNoArgsTstate(tstate, enter);
+            res = PyObject_CallNoArgs(enter);
             Py_DECREF(enter);
             if (res == NULL) {
                 Py_DECREF(exit);
@@ -3822,9 +3822,9 @@ dummy_func(
         }
 
         inst(CONVERT_VALUE, (value -- result)) {
-            convertion_func_ptr  conv_fn;
+            conversion_func conv_fn;
             assert(oparg >= FVC_STR && oparg <= FVC_ASCII);
-            conv_fn = CONVERSION_FUNCTIONS[oparg];
+            conv_fn = _PyEval_ConversionFuncs[oparg];
             result = conv_fn(value);
             Py_DECREF(value);
             ERROR_IF(result == NULL, error);
index 41e9310938d826e61e9f07055b32bf247b64531f..34f286e4e17eb8c9c568f7c9c7953fb28c0397b7 100644 (file)
@@ -337,6 +337,12 @@ const binaryfunc _PyEval_BinaryOps[] = {
     [NB_INPLACE_XOR] = PyNumber_InPlaceXor,
 };
 
+const conversion_func _PyEval_ConversionFuncs[4] = {
+    [FVC_STR] = PyObject_Str,
+    [FVC_REPR] = PyObject_Repr,
+    [FVC_ASCII] = PyObject_ASCII
+};
+
 
 // PEP 634: Structural Pattern Matching
 
index 6ad41950ea3b7e9efd98374985ce1687117c35a0..9674a7824a46901da1785cdb26f1aed6fdf1e8ad 100644 (file)
@@ -352,13 +352,6 @@ do { \
     } \
 } while (0);
 
-typedef PyObject *(*convertion_func_ptr)(PyObject *);
-
-static const convertion_func_ptr CONVERSION_FUNCTIONS[4] = {
-    [FVC_STR] = PyObject_Str,
-    [FVC_REPR] = PyObject_Repr,
-    [FVC_ASCII] = PyObject_ASCII
-};
 
 // GH-89279: Force inlining by using a macro.
 #if defined(_MSC_VER) && SIZEOF_INT == 4
index 20fab8f4c61eb5cd6652eadff027574870bc12de..9ec1be9076a5a0c5337270ed245f76c83f6c5c02 100644 (file)
                 GOTO_ERROR(error);
             }
             Py_DECREF(mgr);
-            res = _PyObject_CallNoArgsTstate(tstate, enter);
+            res = PyObject_CallNoArgs(enter);
             Py_DECREF(enter);
             if (res == NULL) {
                 Py_DECREF(exit);
                 GOTO_ERROR(error);
             }
             Py_DECREF(mgr);
-            res = _PyObject_CallNoArgsTstate(tstate, enter);
+            res = PyObject_CallNoArgs(enter);
             Py_DECREF(enter);
             if (res == NULL) {
                 Py_DECREF(exit);
             PyObject *result;
             oparg = CURRENT_OPARG();
             value = stack_pointer[-1];
-            convertion_func_ptr  conv_fn;
+            conversion_func conv_fn;
             assert(oparg >= FVC_STR && oparg <= FVC_ASCII);
-            conv_fn = CONVERSION_FUNCTIONS[oparg];
+            conv_fn = _PyEval_ConversionFuncs[oparg];
             result = conv_fn(value);
             Py_DECREF(value);
             if (result == NULL) goto pop_1_error_tier_two;
index bb26dac0e2be2039ed52c85b1e158eaa4f9f4323..3312078e9a2d4de1188465497762284ab2e6878f 100644 (file)
@@ -40,7 +40,7 @@
                 GOTO_ERROR(error);
             }
             Py_DECREF(mgr);
-            res = _PyObject_CallNoArgsTstate(tstate, enter);
+            res = PyObject_CallNoArgs(enter);
             Py_DECREF(enter);
             if (res == NULL) {
                 Py_DECREF(exit);
@@ -86,7 +86,7 @@
                 GOTO_ERROR(error);
             }
             Py_DECREF(mgr);
-            res = _PyObject_CallNoArgsTstate(tstate, enter);
+            res = PyObject_CallNoArgs(enter);
             Py_DECREF(enter);
             if (res == NULL) {
                 Py_DECREF(exit);
             PyObject *value;
             PyObject *result;
             value = stack_pointer[-1];
-            convertion_func_ptr  conv_fn;
+            conversion_func conv_fn;
             assert(oparg >= FVC_STR && oparg <= FVC_ASCII);
-            conv_fn = CONVERSION_FUNCTIONS[oparg];
+            conv_fn = _PyEval_ConversionFuncs[oparg];
             result = conv_fn(value);
             Py_DECREF(value);
             if (result == NULL) goto pop_1_error;
index ac2c60ed925a2641b962eeb4bfa27d6f1f6cac36..9f9e123ab91fef4b3630022bb55344ccad4b5601 100644 (file)
@@ -203,13 +203,14 @@ patch(unsigned char *base, const Stencil *stencil, uint64_t *patches)
                 *loc32 = (uint32_t)value;
                 continue;
             case HoleKind_ARM64_RELOC_UNSIGNED:
-            case HoleKind_IMAGE_REL_AMD64_ADDR64:
             case HoleKind_R_AARCH64_ABS64:
             case HoleKind_X86_64_RELOC_UNSIGNED:
             case HoleKind_R_X86_64_64:
                 // 64-bit absolute address.
                 *loc64 = value;
                 continue;
+            case HoleKind_IMAGE_REL_AMD64_REL32:
+            case HoleKind_IMAGE_REL_I386_REL32:
             case HoleKind_R_X86_64_GOTPCRELX:
             case HoleKind_R_X86_64_REX_GOTPCRELX:
             case HoleKind_X86_64_RELOC_GOT:
@@ -249,7 +250,7 @@ patch(unsigned char *base, const Stencil *stencil, uint64_t *patches)
                 // Check that we're not out of range of 32 signed bits:
                 assert((int64_t)value >= -(1LL << 31));
                 assert((int64_t)value < (1LL << 31));
-                loc32[0] = (uint32_t)value;
+                *loc32 = (uint32_t)value;
                 continue;
             case HoleKind_R_AARCH64_CALL26:
             case HoleKind_R_AARCH64_JUMP26:
@@ -307,23 +308,23 @@ patch(unsigned char *base, const Stencil *stencil, uint64_t *patches)
                     next_hole->addend == hole->addend &&
                     next_hole->value == hole->value)
                 {
-                    unsigned char rd = get_bits(loc32[0], 0, 5);
+                    unsigned char reg = get_bits(loc32[0], 0, 5);
                     assert(IS_AARCH64_LDR_OR_STR(loc32[1]));
-                    unsigned char rt = get_bits(loc32[1], 0, 5);
-                    unsigned char rn = get_bits(loc32[1], 5, 5);
-                    assert(rd == rn && rn == rt);
+                    // There should be only one register involved:
+                    assert(reg == get_bits(loc32[1], 0, 5));  // ldr's output register.
+                    assert(reg == get_bits(loc32[1], 5, 5));  // ldr's input register.
                     uint64_t relaxed = *(uint64_t *)value;
                     if (relaxed < (1UL << 16)) {
                         // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; nop
-                        loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | rd;
+                        loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg;
                         loc32[1] = 0xD503201F;
                         i++;
                         continue;
                     }
                     if (relaxed < (1ULL << 32)) {
                         // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; movk reg, YYY
-                        loc32[0] = 0xD2800000 | (get_bits(relaxed,  0, 16) << 5) | rd;
-                        loc32[1] = 0xF2A00000 | (get_bits(relaxed, 16, 16) << 5) | rd;
+                        loc32[0] = 0xD2800000 | (get_bits(relaxed,  0, 16) << 5) | reg;
+                        loc32[1] = 0xF2A00000 | (get_bits(relaxed, 16, 16) << 5) | reg;
                         i++;
                         continue;
                     }
@@ -332,13 +333,15 @@ patch(unsigned char *base, const Stencil *stencil, uint64_t *patches)
                         (int64_t)relaxed >= -(1L << 19) &&
                         (int64_t)relaxed < (1L << 19))
                     {
-                        // adrp reg, AAA; ldr reg, [reg + BBB] -> ldr x0, XXX; nop
-                        loc32[0] = 0x58000000 | (get_bits(relaxed, 2, 19) << 5) | rd;
+                        // adrp reg, AAA; ldr reg, [reg + BBB] -> ldr reg, XXX; nop
+                        loc32[0] = 0x58000000 | (get_bits(relaxed, 2, 19) << 5) | reg;
                         loc32[1] = 0xD503201F;
                         i++;
                         continue;
                     }
                 }
+                // Fall through...
+            case HoleKind_ARM64_RELOC_PAGE21:
                 // Number of pages between this page and the value's page:
                 value = (value >> 12) - ((uint64_t)location >> 12);
                 // Check that we're not out of range of 21 signed bits:
@@ -350,6 +353,7 @@ patch(unsigned char *base, const Stencil *stencil, uint64_t *patches)
                 set_bits(loc32, 5, value, 2, 19);
                 continue;
             case HoleKind_ARM64_RELOC_GOT_LOAD_PAGEOFF12:
+            case HoleKind_ARM64_RELOC_PAGEOFF12:
             case HoleKind_R_AARCH64_LD64_GOT_LO12_NC:
                 // 12-bit low part of an absolute address. Pairs nicely with
                 // ARM64_RELOC_GOT_LOAD_PAGE21 (above).
index 975ca650a13c1a8e9aec60e7dd29c627e2df88b4..14e5fc2aae80ef824e7662cf75ef23d73fedd6db 100644 (file)
@@ -4,9 +4,12 @@ import typing
 HoleKind: typing.TypeAlias = typing.Literal[
     "ARM64_RELOC_GOT_LOAD_PAGE21",
     "ARM64_RELOC_GOT_LOAD_PAGEOFF12",
+    "ARM64_RELOC_PAGE21",
+    "ARM64_RELOC_PAGEOFF12",
     "ARM64_RELOC_UNSIGNED",
-    "IMAGE_REL_AMD64_ADDR64",
+    "IMAGE_REL_AMD64_REL32",
     "IMAGE_REL_I386_DIR32",
+    "IMAGE_REL_I386_REL32",
     "R_AARCH64_ABS64",
     "R_AARCH64_ADR_GOT_PAGE",
     "R_AARCH64_CALL26",
index 71c678e04fbfd5ac1627d3087cbd89bc11180682..eddec731984c82ad00f7035a6156a451cdb0bb85 100644 (file)
@@ -96,7 +96,7 @@ class Stencil:
         instruction |= ((base - hole.offset) >> 2) & 0x03FFFFFF
         self.body[where] = instruction.to_bytes(4, sys.byteorder)
         self.disassembly += [
-            f"{base + 4 * 0: x}: d2800008      mov     x8, #0x0",
+            f"{base + 4 * 0:x}: d2800008      mov     x8, #0x0",
             f"{base + 4 * 0:016x}:  R_AARCH64_MOVW_UABS_G0_NC    {hole.symbol}",
             f"{base + 4 * 1:x}: f2a00008      movk    x8, #0x0, lsl #16",
             f"{base + 4 * 1:016x}:  R_AARCH64_MOVW_UABS_G1_NC    {hole.symbol}",
@@ -162,6 +162,13 @@ class StencilGroup:
                 ):
                     self.code.emit_aarch64_trampoline(hole)
                     continue
+                elif (
+                    hole.kind in {"IMAGE_REL_AMD64_REL32"}
+                    and hole.value is HoleValue.ZERO
+                ):
+                    raise ValueError(
+                        f"Add PyAPI_FUNC(...) or PyAPI_DATA(...) to declaration of {hole.symbol}!"
+                    )
                 holes.append(hole)
             stencil.holes[:] = holes
         self.code.pad(alignment)
index 06dc4e7acc6c91c4fcc94d3b459595605f18dd9e..07959b15b6c4b9f8b0e70a58a306a0d76074a66e 100644 (file)
@@ -106,7 +106,7 @@ class _Target(typing.Generic[_S, _R]):
         o = tempdir / f"{opname}.o"
         args = [
             f"--target={self.triple}",
-            "-DPy_BUILD_CORE",
+            "-DPy_BUILD_CORE_MODULE",
             "-D_DEBUG" if self.debug else "-DNDEBUG",
             f"-D_JIT_OPCODE={opname}",
             "-D_PyJIT_ACTIVE",
@@ -118,12 +118,17 @@ class _Target(typing.Generic[_S, _R]):
             f"-I{CPYTHON / 'Python'}",
             "-O3",
             "-c",
+            # This debug info isn't necessary, and bloats out the JIT'ed code.
+            # We *may* be able to re-enable this, process it, and JIT it for a
+            # nicer debugging experience... but that needs a lot more research:
             "-fno-asynchronous-unwind-tables",
+            # Don't call built-in functions that we can't find or patch:
             "-fno-builtin",
-            # SET_FUNCTION_ATTRIBUTE on 32-bit Windows debug builds:
-            "-fno-jump-tables",
+            # Emit relaxable 64-bit calls/jumps, so we don't have to worry about
+            # about emitting in-range trampolines for out-of-range targets.
+            # We can probably remove this and emit trampolines in the future:
             "-fno-plt",
-            # Don't make calls to weird stack-smashing canaries:
+            # Don't call stack-smashing canaries that we can't find or patch:
             "-fno-stack-protector",
             "-o",
             f"{o}",
@@ -194,12 +199,21 @@ class _COFF(
             offset = base + symbol["Value"]
             name = symbol["Name"]
             name = name.removeprefix(self.prefix)
-            group.symbols[name] = value, offset
+            if name not in group.symbols:
+                group.symbols[name] = value, offset
         for wrapped_relocation in section["Relocations"]:
             relocation = wrapped_relocation["Relocation"]
             hole = self._handle_relocation(base, relocation, stencil.body)
             stencil.holes.append(hole)
 
+    def _unwrap_dllimport(self, name: str) -> tuple[_stencils.HoleValue, str | None]:
+        if name.startswith("__imp_"):
+            name = name.removeprefix("__imp_")
+            name = name.removeprefix(self.prefix)
+            return _stencils.HoleValue.GOT, name
+        name = name.removeprefix(self.prefix)
+        return _stencils.symbol_to_value(name)
+
     def _handle_relocation(
         self, base: int, relocation: _schema.COFFRelocation, raw: bytes
     ) -> _stencils.Hole:
@@ -207,21 +221,23 @@ class _COFF(
             case {
                 "Offset": offset,
                 "Symbol": s,
-                "Type": {"Value": "IMAGE_REL_AMD64_ADDR64" as kind},
+                "Type": {"Value": "IMAGE_REL_I386_DIR32" as kind},
             }:
                 offset += base
-                s = s.removeprefix(self.prefix)
-                value, symbol = _stencils.symbol_to_value(s)
-                addend = int.from_bytes(raw[offset : offset + 8], "little")
+                value, symbol = self._unwrap_dllimport(s)
+                addend = int.from_bytes(raw[offset : offset + 4], "little")
             case {
                 "Offset": offset,
                 "Symbol": s,
-                "Type": {"Value": "IMAGE_REL_I386_DIR32" as kind},
+                "Type": {
+                    "Value": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind
+                },
             }:
                 offset += base
-                s = s.removeprefix(self.prefix)
-                value, symbol = _stencils.symbol_to_value(s)
-                addend = int.from_bytes(raw[offset : offset + 4], "little")
+                value, symbol = self._unwrap_dllimport(s)
+                addend = (
+                    int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4
+                )
             case _:
                 raise NotImplementedError(relocation)
         return _stencils.Hole(offset, kind, value, symbol, addend)
@@ -423,12 +439,12 @@ def get_target(host: str) -> _COFF | _ELF | _MachO:
         args = ["-mcmodel=large"]
         return _ELF(host, alignment=8, args=args)
     if re.fullmatch(r"i686-pc-windows-msvc", host):
-        args = ["-mcmodel=large"]
+        args = ["-DPy_NO_ENABLE_SHARED"]
         return _COFF(host, args=args, prefix="_")
     if re.fullmatch(r"x86_64-apple-darwin.*", host):
         return _MachO(host, prefix="_")
     if re.fullmatch(r"x86_64-pc-windows-msvc", host):
-        args = ["-mcmodel=large"]
+        args = ["-fms-runtime-lib=dll"]
         return _COFF(host, args=args)
     if re.fullmatch(r"x86_64-.*-linux-gnu", host):
         return _ELF(host)
index d79c6efb8f6de4e4a74505dcc2164ce7e97835ac..8aaf4581de362d3a1fbff9158af6297a00ea2397 100644 (file)
@@ -9,6 +9,7 @@
 #include "pycore_long.h"
 #include "pycore_opcode_metadata.h"
 #include "pycore_opcode_utils.h"
+#include "pycore_optimizer.h"
 #include "pycore_range.h"
 #include "pycore_setobject.h"
 #include "pycore_sliceobject.h"
@@ -58,11 +59,11 @@ do {  \
     } while (0)
 
 #define PATCH_VALUE(TYPE, NAME, ALIAS)  \
-    extern void ALIAS;                  \
+    PyAPI_DATA(void) ALIAS;             \
     TYPE NAME = (TYPE)(uint64_t)&ALIAS;
 
 #define PATCH_JUMP(ALIAS)                                    \
-    extern void ALIAS;                                       \
+    PyAPI_DATA(void) ALIAS;                                  \
     __attribute__((musttail))                                \
     return ((jit_func)&ALIAS)(frame, stack_pointer, tstate);