gh-128563: A new tail-calling interpreter

Include/internal/pycore_frame.h

@@ -75,7 +75,8 @@ typedef struct _PyInterpreterFrame {
     _PyStackRef *stackpointer;
     uint16_t return_offset;  /* Only relevant during a function call */
     char owner;
-    char visited;
+    char visited:4;
+    char is_entry_frame:4;
     /* Locals and stack */
     _PyStackRef localsplus[1];
 } _PyInterpreterFrame;
@@ -214,6 +215,8 @@ _PyFrame_Initialize(
         frame->localsplus[i] = PyStackRef_NULL;
     }
 
+    frame->is_entry_frame = 0;
+
 #ifdef Py_GIL_DISABLED
     // On GIL disabled, we walk the entire stack in GC. Since stacktop
     // is not always in sync with the real stack pointer, we have
@@ -394,6 +397,8 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int
     frame->visited = 0;
     frame->return_offset = 0;
 
+    frame->is_entry_frame = 0;
+
 #ifdef Py_GIL_DISABLED
     assert(code->co_nlocalsplus == 0);
     for (int i = 0; i < code->co_stacksize; i++) {

Lib/test/test_generated_cases.py

@@ -4,6 +4,7 @@
 import sys
 import tempfile
 import unittest
+import textwrap
 
 from io import StringIO
 from test import support
@@ -36,6 +37,7 @@ def skip_if_different_mount_drives():
     import parser
     from stack import Local, Stack
     import tier1_generator
+    import tier1_tail_call_generator
     import opcode_metadata_generator
     import optimizer_generator
 
@@ -538,7 +540,7 @@ def test_error_if_plain(self):
             frame->instr_ptr = next_instr;
             next_instr += 1;
             INSTRUCTION_STATS(OP);
-            if (cond) goto label;
+            if (cond) CEVAL_GOTO(label);
             DISPATCH();
         }
     """
@@ -555,7 +557,7 @@ def test_error_if_plain_with_comment(self):
             frame->instr_ptr = next_instr;
             next_instr += 1;
             INSTRUCTION_STATS(OP);
-            if (cond) goto label;
+            if (cond) CEVAL_GOTO(label);
             // Comment is ok
             DISPATCH();
         }
@@ -582,7 +584,7 @@ def test_error_if_pop(self):
             right = stack_pointer[-1];
             left = stack_pointer[-2];
             SPAM(left, right);
-            if (cond) goto pop_2_label;
+            if (cond) CEVAL_GOTO(pop_2_label);
             res = 0;
             stack_pointer[-2] = res;
             stack_pointer += -1;
@@ -611,7 +613,7 @@ def test_error_if_pop_with_result(self):
             right = stack_pointer[-1];
             left = stack_pointer[-2];
             res = SPAM(left, right);
-            if (cond) goto pop_2_label;
+            if (cond) CEVAL_GOTO(pop_2_label);
             stack_pointer[-2] = res;
             stack_pointer += -1;
             assert(WITHIN_STACK_BOUNDS());
@@ -929,7 +931,7 @@ def test_array_error_if(self):
             if (oparg == 0) {
                 stack_pointer += -1 - oparg;
                 assert(WITHIN_STACK_BOUNDS());
-                goto somewhere;
+                CEVAL_GOTO(somewhere);
             }
             stack_pointer += -1 - oparg;
             assert(WITHIN_STACK_BOUNDS());
@@ -1392,7 +1394,7 @@ def test_pop_on_error_peeks(self):
             // THIRD
             {
                 // Mark j and k as used
-                if (cond) goto pop_2_error;
+                if (cond) CEVAL_GOTO(pop_2_error);
             }
             stack_pointer += -2;
             assert(WITHIN_STACK_BOUNDS());
@@ -1435,7 +1437,7 @@ def test_push_then_error(self):
                     stack_pointer[1] = b;
                     stack_pointer += 2;
                     assert(WITHIN_STACK_BOUNDS());
-                    goto error;
+                    CEVAL_GOTO(error);
                 }
             }
             stack_pointer[0] = a;
@@ -1462,14 +1464,14 @@ def test_error_if_true(self):
             frame->instr_ptr = next_instr;
             next_instr += 1;
             INSTRUCTION_STATS(OP1);
-            goto here;
+            CEVAL_GOTO(here);
         }
 
         TARGET(OP2) {
             frame->instr_ptr = next_instr;
             next_instr += 1;
             INSTRUCTION_STATS(OP2);
-            goto there;
+            CEVAL_GOTO(there);
         }
         """
         self.run_cases_test(input, output)
@@ -1739,6 +1741,156 @@ def test_no_escaping_calls_in_branching_macros(self):
         with self.assertRaises(SyntaxError):
             self.run_cases_test(input, "")
 
+class TestGeneratedTailCallErorHandlers(unittest.TestCase):
+    def setUp(self) -> None:
+        super().setUp()
+        self.maxDiff = None
+
+        self.temp_dir = tempfile.gettempdir()
+        self.temp_input_filename = os.path.join(self.temp_dir, "input.txt")
+        self.temp_output_filename = os.path.join(self.temp_dir, "output.txt")
+        self.temp_metadata_filename = os.path.join(self.temp_dir, "metadata.txt")
+        self.temp_pymetadata_filename = os.path.join(self.temp_dir, "pymetadata.txt")
+        self.temp_executor_filename = os.path.join(self.temp_dir, "executor.txt")
+
+    def tearDown(self) -> None:
+        for filename in [
+            self.temp_input_filename,
+            self.temp_output_filename,
+            self.temp_metadata_filename,
+            self.temp_pymetadata_filename,
+            self.temp_executor_filename,
+        ]:
+            try:
+                os.remove(filename)
+            except:
+                pass
+        super().tearDown()
+
+    def run_cases_test(self, input: str, expected: str):
+        with open(self.temp_input_filename, "w+") as temp_input:
+            temp_input.write(textwrap.dedent(input))
+            temp_input.flush()
+
+        with handle_stderr():
+            tier1_tail_call_generator.generate_label_handlers_from_files(
+                self.temp_input_filename, self.temp_output_filename
+            )
+
+        with open(self.temp_output_filename) as temp_output:
+            lines = temp_output.readlines()
+            while lines and lines[0].startswith(("// ", "#", "    #", "\n")):
+                lines.pop(0)
+            while lines and lines[-1].startswith(("#", "\n")):
+                lines.pop(-1)
+        actual = "".join(lines)
+
+        self.assertEqual(actual.strip(), textwrap.dedent(expected).strip())
+
+    def test_correctly_finds_pyeval_framedefault(self):
+        input = """
+        PyObject* _Py_HOT_FUNCTION
+        _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
+        {
+
+        /* END_BASE_INTERPRETER */
+        }
+        """
+        output = """
+        """
+        self.run_cases_test(input, output)
+
+    def test_simple_label(self):
+        input = """
+        PyObject* _Py_HOT_FUNCTION
+        _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
+        {
+
+        TAIL_CALL_TARGET(error):
+            DO_THING();
+        /* END_BASE_INTERPRETER */
+        }
+        """
+        output = """
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS);
+
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS)
+        {
+            DO_THING();
+        /* END_BASE_INTERPRETER */
+        }
+        """
+        self.run_cases_test(input, output)
+
+    def test_fallthrough_label(self):
+        input = """
+        PyObject* _Py_HOT_FUNCTION
+        _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
+        {
+
+        TAIL_CALL_TARGET(error):
+            DO_THING();
+        TAIL_CALL_TARGET(fallthrough):
+            DO_THING2();
+        /* END_BASE_INTERPRETER */
+        }
+        """
+        output = """
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS);
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS);
+
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS)
+        {
+            DO_THING();
+            CEVAL_GOTO(fallthrough);
+        }
+
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS)
+        {
+            DO_THING2();
+        /* END_BASE_INTERPRETER */
+        }
+        """
+        self.run_cases_test(input, output)
+
+    def test_transform_gotos(self):
+        input = """
+        PyObject* _Py_HOT_FUNCTION
+        _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
+        {
+
+        TAIL_CALL_TARGET(error):
+            if (thing) {
+                goto fallthrough;
+            }
+            DO_THING();
+        TAIL_CALL_TARGET(fallthrough):
+            DO_THING2();
+        /* END_BASE_INTERPRETER */
+        }
+        """
+        output = """
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS);
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS);
+
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS)
+        {
+            if (thing) {
+            CEVAL_GOTO(fallthrough);
+            }
+            DO_THING();
+            CEVAL_GOTO(fallthrough);
+        }
+
+        Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_fallthrough(TAIL_CALL_PARAMS)
+        {
+            DO_THING2();
+        /* END_BASE_INTERPRETER */
+        }
+        """
+        self.run_cases_test(input, output)
+
+
 class TestGeneratedAbstractCases(unittest.TestCase):
     def setUp(self) -> None:
         super().setUp()

Makefile.pre.in

@@ -1985,7 +1985,7 @@ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c
 .PHONY: regen-cases
 regen-cases: \
         regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \
-		regen-generated-cases regen-executor-cases regen-optimizer-cases \
+		regen-generated-cases regen-generated-tail-call-handlers regen-executor-cases regen-optimizer-cases \
 		regen-opcode-metadata regen-uop-metadata
 
 .PHONY: regen-opcode-ids
@@ -2018,6 +2018,12 @@ regen-generated-cases:
 	    -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c
 	$(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new
 
+.PHONY: regen-generated-tail-call-handlers
+regen-generated-tail-call-handlers:
+	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier1_tail_call_generator.py \
+	    -o $(srcdir)/Python/generated_tail_call_handlers.c.h.new $(srcdir)/Python/bytecodes.c
+	$(UPDATE_FILE) $(srcdir)/Python/generated_tail_call_handlers.c.h $(srcdir)/Python/generated_tail_call_handlers.c.h.new
+
 .PHONY: regen-executor-cases
 regen-executor-cases:
 	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tier2_generator.py \
@@ -2055,6 +2061,7 @@ Python/ceval.o: \
 		$(srcdir)/Python/ceval_macros.h \
 		$(srcdir)/Python/condvar.h \
 		$(srcdir)/Python/generated_cases.c.h \
+		$(srcdir)/Python/generated_tail_call_handlers.c.h \
 		$(srcdir)/Python/executor_cases.c.h \
 		$(srcdir)/Python/opcode_targets.h
 

Misc/NEWS.d/next/Core_and_Builtins/2025-01-10-18-56-20.gh-issue-128563.baDvls.rst

@@ -0,0 +1 @@
+A new type of interpreter has been added to CPython. This interpreter uses tail calls for its instruction handlers. Preliminary benchmark results suggest 7.4-14.7% geometric mean faster on pyperformance (depending on platform), and up to 45% faster on Python-intensive workloads. Patch by Ken Jin, with ideas by Garret Gu, Haoran Xu, and Josh Haberman.