Skip to content

Commit fd1c59a

Browse files
diegorussodiegorusso
committed
GH-126910: Add GNU backtrace support for unwinding JIT frames
TEST_TEMPLATE: projects/python/templates/default PYTHON_OPTIONS: --configure-flags "--enable-experimental-jit" Jira: ENTLLT-9342 Change-Id: I2527751b6ba2dfe2a089d7818517a2ef5f8437c2
1 parent b97b769 commit fd1c59a

14 files changed

Lines changed: 376 additions & 53 deletions

‎Include/internal/pycore_jit_publish.h‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#ifndef Py_INTERNAL_JIT_PUBLISH_H
2+
#define Py_INTERNAL_JIT_PUBLISH_H
3+
4+
#ifndef Py_BUILD_CORE
5+
# error "this header requires Py_BUILD_CORE define"
6+
#endif
7+
8+
#include <stddef.h>
9+
10+
typedef struct _PyJitCodeRegistration _PyJitCodeRegistration;
11+
12+
#ifdef _Py_JIT
13+
14+
_PyJitCodeRegistration *_PyJit_RegisterCode(const void *code_addr,
15+
size_t code_size,
16+
const char *entry,
17+
const char *filename);
18+
19+
void _PyJit_UnregisterCode(_PyJitCodeRegistration *registration);
20+
21+
#endif // _Py_JIT
22+
23+
#endif // Py_INTERNAL_JIT_PUBLISH_H

‎Include/internal/pycore_jit_unwind.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,11 @@ void *_PyJitUnwind_GdbRegisterCode(const void *code_addr,
5555

5656
void _PyJitUnwind_GdbUnregisterCode(void *handle);
5757

58+
void *_PyJitUnwind_GnuBacktraceRegisterCode(const void *code_addr,
59+
size_t code_size);
60+
61+
void _PyJitUnwind_GnuBacktraceUnregisterCode(void *handle);
62+
5863
#endif // defined(PY_HAVE_PERF_TRAMPOLINE) || (defined(__linux__) && defined(__ELF__))
5964

6065
#endif // Py_INTERNAL_JIT_UNWIND_H

‎Include/internal/pycore_optimizer.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ extern "C" {
99
#endif
1010

1111
#include "pycore_typedefs.h" // _PyInterpreterFrame
12+
#include "pycore_jit_publish.h"
1213
#include "pycore_uop.h" // _PyUOpInstruction
1314
#include "pycore_uop_ids.h"
1415
#include "pycore_stackref.h" // _PyStackRef
@@ -198,7 +199,7 @@ typedef struct _PyExecutorObject {
198199
uint32_t code_size;
199200
size_t jit_size;
200201
void *jit_code;
201-
void *jit_gdb_handle;
202+
_PyJitCodeRegistration *jit_registration;
202203
_PyExitData exits[1];
203204
} _PyExecutorObject;
204205

‎Lib/test/test_frame_pointer_unwind.py‎

Lines changed: 55 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ def _frame_pointers_expected(machine):
7171
return None
7272

7373

74-
def _build_stack_and_unwind():
74+
def _build_stack_and_unwind(unwinder):
7575
import operator
7676

7777
def build_stack(n, unwinder, warming_up_caller=False):
@@ -90,7 +90,7 @@ def build_stack(n, unwinder, warming_up_caller=False):
9090
result = operator.call(build_stack, n - 1, unwinder, warming_up)
9191
return result
9292

93-
stack = build_stack(10, _testinternalcapi.manual_frame_pointer_unwind)
93+
stack = build_stack(10, unwinder)
9494
return stack
9595

9696

@@ -113,8 +113,9 @@ def _classify_stack(stack, jit_enabled):
113113
return annotated, python_frames, jit_frames, other_frames
114114

115115

116-
def _annotate_unwind():
117-
stack = _build_stack_and_unwind()
116+
def _annotate_unwind(unwinder_name="manual_frame_pointer_unwind"):
117+
unwinder = getattr(_testinternalcapi, unwinder_name)
118+
stack = _build_stack_and_unwind(unwinder)
118119
jit_enabled = hasattr(sys, "_jit") and sys._jit.is_enabled()
119120
jit_backend = _testinternalcapi.get_jit_backend()
120121
ranges = _testinternalcapi.get_jit_code_ranges() if jit_enabled else []
@@ -133,13 +134,14 @@ def _annotate_unwind():
133134
"jit_frames": jit_frames,
134135
"other_frames": other_frames,
135136
"jit_backend": jit_backend,
137+
"unwinder": unwinder_name,
136138
})
137139

138140

139-
def _manual_unwind_length(**env):
141+
def _unwind_result(unwinder_name, **env):
140142
code = (
141143
"from test.test_frame_pointer_unwind import _annotate_unwind; "
142-
"print(_annotate_unwind());"
144+
f"print(_annotate_unwind({unwinder_name!r}));"
143145
)
144146
run_env = os.environ.copy()
145147
run_env.update(env)
@@ -198,7 +200,7 @@ def test_manual_unwind_respects_frame_pointers(self):
198200

199201
for env, using_jit in envs:
200202
with self.subTest(env=env):
201-
result = _manual_unwind_length(**env)
203+
result = _unwind_result("manual_frame_pointer_unwind", **env)
202204
jit_frames = result["jit_frames"]
203205
python_frames = result.get("python_frames", 0)
204206
jit_backend = result.get("jit_backend")
@@ -241,5 +243,51 @@ def test_manual_unwind_respects_frame_pointers(self):
241243
)
242244

243245

246+
@support.requires_gil_enabled("test requires the GIL enabled")
247+
@unittest.skipIf(support.is_wasi, "test not supported on WASI")
248+
@unittest.skipUnless(sys.platform == "linux", "GNU backtrace unwinding test requires Linux")
249+
class GnuBacktraceUnwindTests(unittest.TestCase):
250+
251+
def setUp(self):
252+
super().setUp()
253+
try:
254+
_testinternalcapi.gnu_backtrace_unwind()
255+
except RuntimeError as exc:
256+
if "not supported" in str(exc):
257+
self.skipTest("gnu backtrace unwinding not supported on this platform")
258+
raise
259+
260+
def test_gnu_backtrace_unwinds_through_jit_frames(self):
261+
jit_available = hasattr(sys, "_jit") and sys._jit.is_available()
262+
envs = [({"PYTHON_JIT": "0"}, False)]
263+
if jit_available:
264+
envs.append(({"PYTHON_JIT": "1"}, True))
265+
266+
for env, using_jit in envs:
267+
with self.subTest(env=env):
268+
result = _unwind_result("gnu_backtrace_unwind", **env)
269+
python_frames = result.get("python_frames", 0)
270+
jit_frames = result.get("jit_frames", 0)
271+
jit_backend = result.get("jit_backend")
272+
273+
self.assertGreater(
274+
python_frames,
275+
0,
276+
f"expected to find Python frames in GNU backtrace with env {env}",
277+
)
278+
if using_jit and jit_backend == "jit":
279+
self.assertGreater(
280+
jit_frames,
281+
0,
282+
f"expected GNU backtrace to include JIT frames with env {env}",
283+
)
284+
else:
285+
self.assertEqual(
286+
jit_frames,
287+
0,
288+
f"unexpected JIT frames counted in GNU backtrace with env {env}",
289+
)
290+
291+
244292
if __name__ == "__main__":
245293
unittest.main()

‎Makefile.pre.in‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -470,6 +470,7 @@ PYTHON_OBJS= \
470470
Python/instruction_sequence.o \
471471
Python/intrinsics.o \
472472
Python/jit.o \
473+
Python/jit_publish.o \
473474
$(JIT_OBJS) \
474475
Python/legacy_tracing.o \
475476
Python/lock.o \

‎Modules/_testinternalcapi.c‎

Lines changed: 59 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@
4747
#if defined(HAVE_DLADDR) && !defined(__wasi__)
4848
# include <dlfcn.h>
4949
#endif
50+
#if defined(HAVE_EXECINFO_H)
51+
# include <execinfo.h>
52+
#endif
5053
#ifdef MS_WINDOWS
5154
# include <windows.h>
5255
# include <intrin.h>
@@ -58,6 +61,7 @@
5861

5962

6063
static const uintptr_t min_frame_pointer_addr = 0x1000;
64+
#define MAX_UNWIND_FRAMES 200
6165

6266

6367
static PyObject *
@@ -328,7 +332,6 @@ get_jit_backend(PyObject *self, PyObject *Py_UNUSED(args))
328332
static PyObject *
329333
manual_unwind_from_fp(uintptr_t *frame_pointer)
330334
{
331-
Py_ssize_t max_depth = 200;
332335
int stack_grows_down = _Py_STACK_GROWS_DOWN;
333336

334337
if (frame_pointer == NULL) {
@@ -340,14 +343,20 @@ manual_unwind_from_fp(uintptr_t *frame_pointer)
340343
return NULL;
341344
}
342345

343-
for (Py_ssize_t depth = 0;
344-
depth < max_depth && frame_pointer != NULL;
345-
depth++)
346-
{
346+
Py_ssize_t depth = 0;
347+
while (frame_pointer != NULL) {
347348
uintptr_t fp_addr = (uintptr_t)frame_pointer;
348349
if ((fp_addr % sizeof(uintptr_t)) != 0) {
349350
break;
350351
}
352+
if (depth >= MAX_UNWIND_FRAMES) {
353+
Py_DECREF(result);
354+
PyErr_Format(
355+
PyExc_RuntimeError,
356+
"manual frame pointer unwind returned more than %d frames",
357+
MAX_UNWIND_FRAMES);
358+
return NULL;
359+
}
351360
uintptr_t return_addr = frame_pointer[1];
352361

353362
PyObject *addr_obj = PyLong_FromUnsignedLongLong(return_addr);
@@ -361,6 +370,7 @@ manual_unwind_from_fp(uintptr_t *frame_pointer)
361370
return NULL;
362371
}
363372
Py_DECREF(addr_obj);
373+
depth++;
364374

365375
uintptr_t *next_fp = (uintptr_t *)frame_pointer[0];
366376
// Stop if the frame pointer is extremely low.
@@ -383,6 +393,49 @@ manual_unwind_from_fp(uintptr_t *frame_pointer)
383393

384394
return result;
385395
}
396+
397+
#if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE)
398+
static PyObject *
399+
gnu_backtrace_unwind(PyObject *self, PyObject *Py_UNUSED(args))
400+
{
401+
void *addresses[MAX_UNWIND_FRAMES + 1];
402+
int frame_count = backtrace(addresses, Py_ARRAY_LENGTH(addresses));
403+
if (frame_count < 0) {
404+
PyErr_SetString(PyExc_RuntimeError, "backtrace() failed");
405+
return NULL;
406+
}
407+
if (frame_count > MAX_UNWIND_FRAMES) {
408+
PyErr_Format(
409+
PyExc_RuntimeError,
410+
"backtrace() returned more than %d frames",
411+
MAX_UNWIND_FRAMES);
412+
return NULL;
413+
}
414+
415+
PyObject *result = PyList_New(frame_count);
416+
if (result == NULL) {
417+
return NULL;
418+
}
419+
for (int i = 0; i < frame_count; i++) {
420+
PyObject *addr_obj = PyLong_FromUnsignedLongLong((uintptr_t)addresses[i]);
421+
if (addr_obj == NULL) {
422+
Py_DECREF(result);
423+
return NULL;
424+
}
425+
PyList_SET_ITEM(result, i, addr_obj);
426+
}
427+
return result;
428+
}
429+
#else
430+
static PyObject *
431+
gnu_backtrace_unwind(PyObject *self, PyObject *Py_UNUSED(args))
432+
{
433+
PyErr_SetString(PyExc_RuntimeError,
434+
"gnu_backtrace_unwind is not supported on this platform");
435+
return NULL;
436+
}
437+
#endif
438+
386439
#if defined(__GNUC__) || defined(__clang__)
387440
static PyObject *
388441
manual_frame_pointer_unwind(PyObject *self, PyObject *args)
@@ -2914,6 +2967,7 @@ static PyMethodDef module_functions[] = {
29142967
{"classify_stack_addresses", classify_stack_addresses, METH_VARARGS},
29152968
{"get_jit_code_ranges", get_jit_code_ranges, METH_NOARGS},
29162969
{"get_jit_backend", get_jit_backend, METH_NOARGS},
2970+
{"gnu_backtrace_unwind", gnu_backtrace_unwind, METH_NOARGS},
29172971
{"manual_frame_pointer_unwind", manual_frame_pointer_unwind, METH_NOARGS},
29182972
{"test_bswap", test_bswap, METH_NOARGS},
29192973
{"test_popcount", test_popcount, METH_NOARGS},

‎PCbuild/_freeze_module.vcxproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,7 @@
237237
<ClCompile Include="..\Python\intrinsics.c" />
238238
<ClCompile Include="..\Python\instrumentation.c" />
239239
<ClCompile Include="..\Python\jit.c" />
240+
<ClCompile Include="..\Python\jit_publish.c" />
240241
<ClCompile Include="..\Python\legacy_tracing.c" />
241242
<ClCompile Include="..\Python\lock.c" />
242243
<ClCompile Include="..\Python\marshal.c" />

‎PCbuild/_freeze_module.vcxproj.filters‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,9 @@
265265
<ClCompile Include="..\Python\jit.c">
266266
<Filter>Source Files</Filter>
267267
</ClCompile>
268+
<ClCompile Include="..\Python\jit_publish.c">
269+
<Filter>Source Files</Filter>
270+
</ClCompile>
268271
<ClCompile Include="..\Objects\lazyimportobject.c">
269272
<Filter>Source Files</Filter>
270273
</ClCompile>

‎PCbuild/pythoncore.vcxproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -648,6 +648,7 @@
648648
<ClCompile Include="..\Python\instruction_sequence.c" />
649649
<ClCompile Include="..\Python\instrumentation.c" />
650650
<ClCompile Include="..\Python\jit.c" />
651+
<ClCompile Include="..\Python\jit_publish.c" />
651652
<ClCompile Include="..\Python\legacy_tracing.c" />
652653
<ClCompile Include="..\Python\lock.c" />
653654
<ClCompile Include="..\Python\marshal.c" />

‎PCbuild/pythoncore.vcxproj.filters‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1481,6 +1481,9 @@
14811481
<ClCompile Include="..\Python\jit.c">
14821482
<Filter>Python</Filter>
14831483
</ClCompile>
1484+
<ClCompile Include="..\Python\jit_publish.c">
1485+
<Filter>Python</Filter>
1486+
</ClCompile>
14841487
<ClCompile Include="..\Python\legacy_tracing.c">
14851488
<Filter>Source Files</Filter>
14861489
</ClCompile>

0 commit comments

Comments
 (0)