Skip to content

[3.13] gh-123022: Fix crash with Py_Initialize in background thread (GH-123052)#123114

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
colesbury merged 1 commit into from
Aug 17, 2024
Merged

[3.13] gh-123022: Fix crash with Py_Initialize in background thread (GH-123052) #123114

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion Lib/test/test_embed.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
# Run the tests in Programs/_testembed.c (tests for the CPython embedding APIs)
from test import support
from test.support import import_helper, os_helper, MS_WINDOWS
from test.support import import_helper, os_helper, threading_helper, MS_WINDOWS
import unittest

from collections import namedtuple
Expand DownExpand Up@@ -1792,6 +1792,13 @@ def test_init_main_interpreter_settings(self):

self.assertEqual(out, expected)

@threading_helper.requires_working_threading()
def test_init_in_background_thread(self):
# gh-123022: Check that running Py_Initialize() in a background
# thread doesn't crash.
out, err = self.run_embedded_interpreter("test_init_in_background_thread")
self.assertEqual(err, "")


class SetConfigTests(unittest.TestCase):
def test_set_config(self):
Expand Down
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Core_and_Builtins/2024-08-15-19-28-43.gh-issue-123022.m3EF9E.rst
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
Fix crash in free-threaded build when calling :c:func:`Py_Initialize` from
a non-main thread.
16 changes: 12 additions & 4 deletions Objects/mimalloc/os.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -115,8 +115,12 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size)
if (hint == 0 || hint > MI_HINT_MAX){// wrap or initialize
uintptr_t init = MI_HINT_BASE;
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode
uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap());
init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB
mi_heap_t* heap = mi_prim_get_default_heap();
// gh-123022: default heap may not be initialized in CPython in background threads
if (mi_heap_is_initialized(heap)){
uintptr_t r = _mi_heap_random_next(heap);
init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB
}
#endif
uintptr_t expected = hint + size;
mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init);
Expand DownExpand Up@@ -553,8 +557,12 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size){
// Initialize the start address after the 32TiB area
start = ((uintptr_t)32 << 40); // 32TiB virtual start address
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode
uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap());
start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB
mi_heap_t* heap = mi_prim_get_default_heap();
// gh-123022: default heap may not be initialized in CPython in background threads
if (mi_heap_is_initialized(heap)){
uintptr_t r = _mi_heap_random_next(heap);
start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB
}
#endif
}
end = start + size;
Expand Down
18 changes: 18 additions & 0 deletions Programs/_testembed.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -8,6 +8,7 @@
#include <Python.h>
#include "pycore_initconfig.h" // _PyConfig_InitCompatConfig()
#include "pycore_runtime.h" // _PyRuntime
#include "pycore_pythread.h" // PyThread_start_joinable_thread()
#include "pycore_import.h" // _PyImport_FrozenBootstrap
#include <inttypes.h>
#include <stdio.h>
Expand DownExpand Up@@ -2022,6 +2023,22 @@ static int test_init_main_interpreter_settings(void)
return 0;
}

static void do_init(void *unused)
{
_testembed_Py_Initialize();
Py_Finalize();
}

static int test_init_in_background_thread(void)
{
PyThread_handle_t handle;
PyThread_ident_t ident;
if (PyThread_start_joinable_thread(&do_init, NULL, &ident, &handle) < 0){
return -1;
}
return PyThread_join_thread(handle);
}


#ifndef MS_WINDOWS
#include "test_frozenmain.h" // M_test_frozenmain
Expand DownExpand Up@@ -2211,6 +2228,7 @@ static struct TestCase TestCases[] ={
{"test_get_argc_argv", test_get_argc_argv},
{"test_init_use_frozen_modules", test_init_use_frozen_modules},
{"test_init_main_interpreter_settings", test_init_main_interpreter_settings},
{"test_init_in_background_thread", test_init_in_background_thread},

// Audit
{"test_open_code_hook", test_open_code_hook},
Expand Down