From: Victor Stinner Date: Wed, 18 Sep 2019 12:10:16 +0000 (+0200) Subject: [3.8] bpo-38070: Py_FatalError() logs runtime state (GH-16258) X-Git-Tag: v3.8.0rc1~70 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=47bbab9f76735acc1991e541d12fd18be6b13b16;p=python [3.8] bpo-38070: Py_FatalError() logs runtime state (GH-16258) * bpo-38070: _Py_DumpTraceback() writes (GH-16244) When a Python thread has no frame, _Py_DumpTraceback() and _Py_DumpTracebackThreads() now write "", rather than writing nothing. (cherry picked from commit 8fa3e1740b3f03ea65ddb68411c2238c5f98eec2) * bpo-38070: Enhance _PyObject_Dump() (GH-16243) _PyObject_Dump() now dumps the object address for freed objects and objects with ob_type=NULL. (cherry picked from commit b39afb78768418d9405c4b528c80fa968ccc974d) * bpo-38070: Add _PyRuntimeState.preinitializing (GH-16245) Add _PyRuntimeState.preinitializing field: set to 1 while Py_PreInitialize() is running. _PyRuntimeState: rename also pre_initialized field to preinitialized. (cherry picked from commit d3b904144e86e2442961de6a7dccecbe133d5c6d) * bpo-38070: Py_FatalError() logs runtime state (GH-16246) (cherry picked from commit 1ce16fb0977283ae42a9f8917bbca5f44aa69324) --- diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 3ab4009770..f90e7e1ab7 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -193,8 +193,11 @@ struct _gilstate_runtime_state { /* Full Python runtime state */ typedef struct pyruntimestate { - /* Is Python pre-initialized? Set to 1 by Py_PreInitialize() */ - int pre_initialized; + /* Is running Py_PreInitialize()? */ + int preinitializing; + + /* Is Python preinitialized? Set to 1 by Py_PreInitialize() */ + int preinitialized; /* Is Python core initialized? Set to 1 by _Py_InitializeCore() */ int core_initialized; @@ -202,6 +205,8 @@ typedef struct pyruntimestate { /* Is Python fully initialized? Set to 1 by Py_Initialize() */ int initialized; + /* Set by Py_FinalizeEx(). Only reset to NULL if Py_Initialize() + is called again. */ PyThreadState *finalizing; struct pyinterpreters { @@ -244,7 +249,7 @@ typedef struct pyruntimestate { } _PyRuntimeState; #define _PyRuntimeState_INIT \ - {.pre_initialized = 0, .core_initialized = 0, .initialized = 0} + {.preinitialized = 0, .core_initialized = 0, .initialized = 0} /* Note: _PyRuntimeState_INIT sets other fields to 0/NULL */ PyAPI_DATA(_PyRuntimeState) _PyRuntime; diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 4d6e2f2155..ff7acac43b 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -198,6 +198,7 @@ class CAPITest(unittest.TestCase): self.assertRegex(err.replace(b'\r', b''), br'Fatal Python error: a function returned NULL ' br'without setting an error\n' + br'Python runtime state: initialized\n' br'SystemError: returned NULL ' br'without setting an error\n' @@ -225,6 +226,7 @@ class CAPITest(unittest.TestCase): self.assertRegex(err.replace(b'\r', b''), br'Fatal Python error: a function returned a ' br'result with an error set\n' + br'Python runtime state: initialized\n' br'ValueError\n' br'\n' br'The above exception was the direct cause ' diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 1cf20db1c7..d875103934 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -90,7 +90,8 @@ class FaultHandlerTests(unittest.TestCase): def check_error(self, code, line_number, fatal_error, *, filename=None, all_threads=True, other_regex=None, - fd=None, know_current_thread=True): + fd=None, know_current_thread=True, + py_fatal_error=False): """ Check that the fault handler for fatal errors is enabled and check the traceback from the child process output. @@ -110,10 +111,12 @@ class FaultHandlerTests(unittest.TestCase): {header} \(most recent call first\): File "", line {lineno} in """ - regex = dedent(regex.format( + if py_fatal_error: + fatal_error += "\nPython runtime state: initialized" + regex = dedent(regex).format( lineno=line_number, fatal_error=fatal_error, - header=header)).strip() + header=header).strip() if other_regex: regex += '|' + other_regex output, exitcode = self.get_output(code, filename=filename, fd=fd) @@ -170,7 +173,8 @@ class FaultHandlerTests(unittest.TestCase): """, 3, 'in new thread', - know_current_thread=False) + know_current_thread=False, + py_fatal_error=True) def test_sigabrt(self): self.check_fatal_error(""" @@ -226,7 +230,8 @@ class FaultHandlerTests(unittest.TestCase): faulthandler._fatal_error(b'xyz') """, 2, - 'xyz') + 'xyz', + py_fatal_error=True) def test_fatal_error_without_gil(self): self.check_fatal_error(""" @@ -234,7 +239,8 @@ class FaultHandlerTests(unittest.TestCase): faulthandler._fatal_error(b'xyz', True) """, 2, - 'xyz') + 'xyz', + py_fatal_error=True) @unittest.skipIf(sys.platform.startswith('openbsd'), "Issue #12868: sigaltstack() doesn't work on " diff --git a/Objects/object.c b/Objects/object.c index 585a9748c8..df2531371f 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -464,7 +464,7 @@ void _PyObject_Dump(PyObject* op) { if (op == NULL) { - fprintf(stderr, "\n"); + fprintf(stderr, "\n"); fflush(stderr); return; } @@ -472,7 +472,7 @@ _PyObject_Dump(PyObject* op) if (_PyObject_IsFreed(op)) { /* It seems like the object memory has been freed: don't access it to prevent a segmentation fault. */ - fprintf(stderr, "\n"); + fprintf(stderr, "\n", op); return; } @@ -2162,18 +2162,19 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg, fflush(stderr); if (obj == NULL) { - fprintf(stderr, "\n"); + fprintf(stderr, "\n"); } else if (_PyObject_IsFreed(obj)) { /* It seems like the object memory has been freed: don't access it to prevent a segmentation fault. */ - fprintf(stderr, "\n"); + fprintf(stderr, "\n", obj); } else if (Py_TYPE(obj) == NULL) { - fprintf(stderr, "\n"); + fprintf(stderr, "\n", obj); } else if (_PyObject_IsFreed((PyObject *)Py_TYPE(obj))) { - fprintf(stderr, "\n", (void *)Py_TYPE(obj)); + fprintf(stderr, "\n", + obj, (void *)Py_TYPE(obj)); } else { /* Display the traceback where the object has been allocated. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 751c4d6d1d..97bb99256d 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -719,11 +719,15 @@ _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args) } _PyRuntimeState *runtime = &_PyRuntime; - if (runtime->pre_initialized) { + if (runtime->preinitialized) { /* If it's already configured: ignored the new configuration */ return _PyStatus_OK(); } + /* Note: preinitialized remains 1 on error, it is only set to 0 + at exit on success. */ + runtime->preinitializing = 1; + PyPreConfig config; _PyPreConfig_InitFromPreConfig(&config, src_config); @@ -737,7 +741,8 @@ _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args) return status; } - runtime->pre_initialized = 1; + runtime->preinitializing = 0; + runtime->preinitialized = 1; return _PyStatus_OK(); } @@ -777,7 +782,7 @@ _Py_PreInitializeFromConfig(const PyConfig *config, } _PyRuntimeState *runtime = &_PyRuntime; - if (runtime->pre_initialized) { + if (runtime->preinitialized) { /* Already initialized: do nothing */ return _PyStatus_OK(); } @@ -1961,13 +1966,14 @@ done: static void -_Py_FatalError_DumpTracebacks(int fd) +_Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, + PyThreadState *tstate) { fputc('\n', stderr); fflush(stderr); /* display the current Python stack */ - _Py_DumpTracebackThreads(fd, NULL, NULL); + _Py_DumpTracebackThreads(fd, interp, tstate); } /* Print the current exception (if an exception is set) with its traceback, @@ -2062,10 +2068,39 @@ fatal_output_debug(const char *msg) } #endif + +static void +fatal_error_dump_runtime(FILE *stream, _PyRuntimeState *runtime) +{ + fprintf(stream, "Python runtime state: "); + if (runtime->finalizing) { + fprintf(stream, "finalizing (tstate=%p)", runtime->finalizing); + } + else if (runtime->initialized) { + fprintf(stream, "initialized"); + } + else if (runtime->core_initialized) { + fprintf(stream, "core initialized"); + } + else if (runtime->preinitialized) { + fprintf(stream, "preinitialized"); + } + else if (runtime->preinitializing) { + fprintf(stream, "preinitializing"); + } + else { + fprintf(stream, "unknown"); + } + fprintf(stream, "\n"); + fflush(stream); +} + + static void _Py_NO_RETURN fatal_error(const char *prefix, const char *msg, int status) { - const int fd = fileno(stderr); + FILE *stream = stderr; + const int fd = fileno(stream); static int reentrant = 0; if (reentrant) { @@ -2075,45 +2110,48 @@ fatal_error(const char *prefix, const char *msg, int status) } reentrant = 1; - fprintf(stderr, "Fatal Python error: "); + fprintf(stream, "Fatal Python error: "); if (prefix) { - fputs(prefix, stderr); - fputs(": ", stderr); + fputs(prefix, stream); + fputs(": ", stream); } if (msg) { - fputs(msg, stderr); + fputs(msg, stream); } else { - fprintf(stderr, ""); + fprintf(stream, ""); } - fputs("\n", stderr); - fflush(stderr); /* it helps in Windows debug build */ + fputs("\n", stream); + fflush(stream); /* it helps in Windows debug build */ - /* Check if the current thread has a Python thread state - and holds the GIL */ - PyThreadState *tss_tstate = PyGILState_GetThisThreadState(); - if (tss_tstate != NULL) { - PyThreadState *tstate = _PyThreadState_GET(); - if (tss_tstate != tstate) { - /* The Python thread does not hold the GIL */ - tss_tstate = NULL; - } - } - else { - /* Py_FatalError() has been called from a C thread - which has no Python thread state. */ + _PyRuntimeState *runtime = &_PyRuntime; + fatal_error_dump_runtime(stream, runtime); + + PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyInterpreterState *interp = NULL; + if (tstate != NULL) { + interp = tstate->interp; } - int has_tstate_and_gil = (tss_tstate != NULL); + /* Check if the current thread has a Python thread state + and holds the GIL. + + tss_tstate is NULL if Py_FatalError() is called from a C thread which + has no Python thread state. + + tss_tstate != tstate if the current Python thread does not hold the GIL. + */ + PyThreadState *tss_tstate = PyGILState_GetThisThreadState(); + int has_tstate_and_gil = (tss_tstate != NULL && tss_tstate == tstate); if (has_tstate_and_gil) { /* If an exception is set, print the exception with its traceback */ if (!_Py_FatalError_PrintExc(fd)) { /* No exception is set, or an exception is set without traceback */ - _Py_FatalError_DumpTracebacks(fd); + _Py_FatalError_DumpTracebacks(fd, interp, tss_tstate); } } else { - _Py_FatalError_DumpTracebacks(fd); + _Py_FatalError_DumpTracebacks(fd, interp, tss_tstate); } /* The main purpose of faulthandler is to display the traceback. diff --git a/Python/traceback.c b/Python/traceback.c index 0463eb6d8c..8e2f15e85d 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -797,12 +797,15 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) PyFrameObject *frame; unsigned int depth; - if (write_header) + if (write_header) { PUTS(fd, "Stack (most recent call first):\n"); + } frame = _PyThreadState_GetFrame(tstate); - if (frame == NULL) + if (frame == NULL) { + PUTS(fd, "\n"); return; + } depth = 0; while (frame != NULL) { @@ -870,9 +873,9 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, Python thread state of the current thread. PyThreadState_Get() doesn't give the state of the thread that caused - the fault if the thread released the GIL, and so this function - cannot be used. Read the thread specific storage (TSS) instead: call - PyGILState_GetThisThreadState(). */ + the fault if the thread released the GIL, and so + _PyThreadState_GET() cannot be used. Read the thread specific + storage (TSS) instead: call PyGILState_GetThisThreadState(). */ current_tstate = PyGILState_GetThisThreadState(); }