From 7acb376e8df572f5e7e76ee33372cc46f17a9d93 Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Thu, 26 Mar 2026 23:14:47 +0100 Subject: [PATCH 01/14] Refactor reference queue polling state --- .../capi/transitions/CApiTransitions.java | 75 +++++++++++++++---- 1 file changed, 59 insertions(+), 16 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/CApiTransitions.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/CApiTransitions.java index 6fac3e14ce..57d4f3673c 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/CApiTransitions.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/CApiTransitions.java @@ -42,6 +42,11 @@ import static com.oracle.graal.python.builtins.objects.cext.capi.PythonNativeWrapper.PythonAbstractObjectNativeWrapper.IMMORTAL_REFCNT; import static com.oracle.graal.python.builtins.objects.cext.capi.PythonNativeWrapper.PythonAbstractObjectNativeWrapper.MANAGED_REFCNT; +import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.CApiTransitions.PollingState.RQ_DISABLED_PERMANENT; +import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.CApiTransitions.PollingState.RQ_DISABLED_TEMP; +import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.CApiTransitions.PollingState.RQ_POLLING; +import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.CApiTransitions.PollingState.RQ_READY; +import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.CApiTransitions.PollingState.RQ_UNINITIALIZED; import java.lang.ref.ReferenceQueue; import java.lang.ref.WeakReference; @@ -149,6 +154,23 @@ public abstract class CApiTransitions { private static final TruffleLogger LOGGER = CApiContext.getLogger(CApiTransitions.class); + enum PollingState { + /** startup barrier not finished yet, polling must not run */ + RQ_UNINITIALIZED, + + /** normal steady state, polling allowed */ + RQ_READY, + + /** one thread is currently polling */ + RQ_POLLING, + + /** temporarily disabled by GraalPyPrivate_DisableReferneceQueuePolling */ + RQ_DISABLED_TEMP, + + /** shutdown/finalization, end state */ + RQ_DISABLED_PERMANENT + } + private CApiTransitions() { } @@ -183,7 +205,7 @@ public HandleContext(boolean useShadowTable) { public final ReferenceQueue referenceQueue = new ReferenceQueue<>(); - volatile boolean referenceQueuePollActive = false; + volatile PollingState referenceQueuePollingState = RQ_UNINITIALIZED; @TruffleBoundary public static T putShadowTable(HashMap table, long pointer, T ref) { @@ -429,29 +451,36 @@ public static int pollReferenceQueue() { PythonContext context = PythonContext.get(null); HandleContext handleContext = context.nativeContext; int manuallyCollected = 0; - if (!handleContext.referenceQueuePollActive) { - try (GilNode.UncachedAcquire ignored = GilNode.uncachedAcquire()) { - ReferenceQueue queue = handleContext.referenceQueue; - int count = 0; - long start = 0; - ArrayList referencesToBeFreed = handleContext.referencesToBeFreed; + if (handleContext.referenceQueuePollingState != RQ_READY) { + return manuallyCollected; + } + try (GilNode.UncachedAcquire ignored = GilNode.uncachedAcquire()) { + if (handleContext.referenceQueuePollingState != RQ_READY) { + return manuallyCollected; + } + ReferenceQueue queue = handleContext.referenceQueue; + int count = 0; + long start = 0; + boolean polling = false; + ArrayList referencesToBeFreed = handleContext.referencesToBeFreed; + try { while (true) { Object entry = queue.poll(); if (entry == null) { if (count > 0) { - assert handleContext.referenceQueuePollActive; + assert handleContext.referenceQueuePollingState == RQ_POLLING || handleContext.referenceQueuePollingState == RQ_DISABLED_PERMANENT; releaseNativeObjects(context, referencesToBeFreed); - handleContext.referenceQueuePollActive = false; LOGGER.fine("collected " + count + " references from native reference queue in " + ((System.nanoTime() - start) / 1000000) + "ms"); } return manuallyCollected; } if (count == 0) { - assert !handleContext.referenceQueuePollActive; - handleContext.referenceQueuePollActive = true; + assert handleContext.referenceQueuePollingState == RQ_READY; + handleContext.referenceQueuePollingState = RQ_POLLING; + polling = true; start = System.nanoTime(); } else { - assert handleContext.referenceQueuePollActive; + assert handleContext.referenceQueuePollingState == RQ_POLLING; } count++; LOGGER.fine(() -> PythonUtils.formatJString("releasing %s, no remaining managed references", entry)); @@ -529,9 +558,12 @@ public static int pollReferenceQueue() { processPyCapsuleReference(reference); } } + } finally { + if (polling && handleContext.referenceQueuePollingState == RQ_POLLING) { + handleContext.referenceQueuePollingState = RQ_READY; + } } } - return manuallyCollected; } /** @@ -698,15 +730,26 @@ public static void freeNativeReplacementStructs(PythonContext context, HandleCon } public static boolean disableReferenceQueuePolling(HandleContext handleContext) { - if (!handleContext.referenceQueuePollActive) { - handleContext.referenceQueuePollActive = true; + if (handleContext.referenceQueuePollingState == RQ_READY) { + handleContext.referenceQueuePollingState = RQ_DISABLED_TEMP; return false; } return true; } public static void enableReferenceQueuePolling(HandleContext handleContext) { - handleContext.referenceQueuePollActive = false; + if (handleContext.referenceQueuePollingState == RQ_DISABLED_TEMP) { + handleContext.referenceQueuePollingState = RQ_READY; + } + } + + public static void initializeReferenceQueuePolling(HandleContext handleContext) { + assert handleContext.referenceQueuePollingState == RQ_UNINITIALIZED : handleContext.referenceQueuePollingState; + handleContext.referenceQueuePollingState = RQ_READY; + } + + public static void disableReferenceQueuePollingPermanently(HandleContext handleContext) { + handleContext.referenceQueuePollingState = RQ_DISABLED_PERMANENT; } private static void freeNativeStub(PythonObjectReference ref) { From 98cb2c2d8fc15bb2435c1afb0f5ee97d9501da5c Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Thu, 26 Mar 2026 23:14:57 +0100 Subject: [PATCH 02/14] Eagerly initialize native thread state --- .../com.oracle.graal.python.cext/src/capi.c | 10 ++- .../src/pystate.c | 5 +- .../objects/cext/capi/CApiContext.java | 87 ++++++++++++++++++- .../objects/cext/capi/NativeCAPISymbol.java | 1 + .../graal/python/runtime/PythonContext.java | 64 +++++++++++--- 5 files changed, 145 insertions(+), 22 deletions(-) diff --git a/graalpython/com.oracle.graal.python.cext/src/capi.c b/graalpython/com.oracle.graal.python.cext/src/capi.c index a7517cd427..ce88ac5f95 100644 --- a/graalpython/com.oracle.graal.python.cext/src/capi.c +++ b/graalpython/com.oracle.graal.python.cext/src/capi.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -272,9 +272,13 @@ PyObject* _Py_NotImplementedStructReference; */ THREAD_LOCAL PyThreadState *tstate_current = NULL; +PyAPI_FUNC(void) GraalPyPrivate_InitThreadStateCurrent(PyThreadState *tstate) { + tstate_current = tstate; +} + static void initialize_globals() { - // store the thread state into a thread local variable - tstate_current = GraalPyPrivate_ThreadState_Get(&tstate_current); + // initialize the current thread's TLS slot for PyThreadState_Get() + GraalPyPrivate_InitThreadStateCurrent(); _Py_NoneStructReference = GraalPyPrivate_None(); _Py_NotImplementedStructReference = GraalPyPrivate_NotImplemented(); _Py_EllipsisObjectReference = GraalPyPrivate_Ellipsis(); diff --git a/graalpython/com.oracle.graal.python.cext/src/pystate.c b/graalpython/com.oracle.graal.python.cext/src/pystate.c index b54a724306..90e67497f1 100644 --- a/graalpython/com.oracle.graal.python.cext/src/pystate.c +++ b/graalpython/com.oracle.graal.python.cext/src/pystate.c @@ -83,10 +83,7 @@ extern "C" { static inline PyThreadState * _get_thread_state() { PyThreadState *ts = tstate_current; - if (UNLIKELY(ts == NULL)) { - ts = GraalPyPrivate_ThreadState_Get(&tstate_current); - tstate_current = ts; - } + assert(ts != NULL); return ts; } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java index dbc5401ef9..c03cfbb927 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java @@ -62,7 +62,12 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.concurrent.CancellationException; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; @@ -127,6 +132,7 @@ import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.CompilerDirectives.ValueType; import com.oracle.truffle.api.RootCallTarget; +import com.oracle.truffle.api.ThreadLocalAction; import com.oracle.truffle.api.TruffleFile; import com.oracle.truffle.api.TruffleLanguage.Env; import com.oracle.truffle.api.TruffleLogger; @@ -814,8 +820,21 @@ public static CApiContext ensureCapiWasLoaded(Node node, PythonContext context, TruffleSafepoint safepoint = TruffleSafepoint.getCurrent(); boolean prevAllowSideEffects = safepoint.setAllowSideEffects(false); try { - loadCApi(node, context, name, path, reason); + CApiContext cApiContext = loadCApi(node, context, name, path, reason); + initializeThreadStateCurrentForAttachedThreads(node, context); + CApiTransitions.initializeReferenceQueuePolling(context.nativeContext); + context.runCApiHooks(); context.setCApiInitialized(); // volatile write + context.finishCApiThreadStateInit(); + try { + cApiContext.runBackgroundGCTask(context); + } catch (RuntimeException e) { + // This can happen when other languages restrict multithreading + LOGGER.warning(() -> "didn't start the background GC task due to: " + e.getMessage()); + } + } catch (Throwable t) { + context.finishCApiThreadStateInit(); + throw t; } finally { safepoint.setAllowSideEffects(prevAllowSideEffects); } @@ -827,6 +846,67 @@ public static CApiContext ensureCapiWasLoaded(Node node, PythonContext context, return context.getCApiContext(); } + @SuppressWarnings("try") + private static void initializeThreadStateCurrentForAttachedThreads(Node node, PythonContext context) throws ApiInitException { + Thread[] threads = getOtherAliveAttachedThreads(context); + if (threads.length == 0) { + return; + } + ThreadLocalAction action = new ThreadLocalAction(true, false) { + @Override + protected void perform(ThreadLocalAction.Access access) { + PCallCapiFunction.callUncached(NativeCAPISymbol.FUN_INIT_THREAD_STATE_CURRENT); + } + }; + waitForThreadLocalActions(node, context, submitThreadLocalActions(context, threads, action)); + } + + private static Thread[] getOtherAliveAttachedThreads(PythonContext context) { + Thread currentThread = Thread.currentThread(); + ArrayList threads = new ArrayList<>(); + for (Thread thread : context.getThreads()) { + if (thread != currentThread && thread.isAlive()) { + threads.add(thread); + } + } + return threads.toArray(Thread[]::new); + } + + private static ArrayList> submitThreadLocalActions(PythonContext context, Thread[] threads, ThreadLocalAction action) { + ArrayList> futures = new ArrayList<>(threads.length); + for (Thread thread : threads) { + futures.add(context.getEnv().submitThreadLocal(new Thread[]{thread}, action)); + } + return futures; + } + + @SuppressWarnings("try") + private static void waitForThreadLocalActions(Node node, PythonContext context, ArrayList> futures) throws ApiInitException { + Node waitLocation = node != null ? node : context.getLanguage().unavailableSafepointLocation; + try (GilNode.UncachedRelease ignored = GilNode.uncachedRelease()) { + for (Future future : futures) { + TruffleSafepoint.setBlockedThreadInterruptible(waitLocation, voidFuture -> { + try { + voidFuture.get(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } catch (TimeoutException | ExecutionException e) { + throw new RuntimeException(e); + } catch (CancellationException e) { + // Ignore threads that went away while initialization was in progress. + } + }, future); + } + } catch (RuntimeException e) { + Throwable cause = e.getCause(); + if (cause instanceof TimeoutException) { + throw new ApiInitException(toTruffleStringUncached("Timed out while initializing native thread state on an attached thread.")); + } + throw e; + } + } + private static CApiContext loadCApi(Node node, PythonContext context, TruffleString name, TruffleString path, String reason) throws IOException, ImportException, ApiInitException { Env env = context.getEnv(); InteropLibrary U = InteropLibrary.getUncached(); @@ -885,9 +965,9 @@ private static CApiContext loadCApi(Node node, PythonContext context, TruffleStr U.execute(initFunction, builtinArrayWrapper, gcState); } + context.startCApiThreadStateInit(); assert PythonCApiAssertions.assertBuiltins(capiLibrary); cApiContext.pyDateTimeCAPICapsule = PyDateTimeCAPIWrapper.initWrapper(context, cApiContext); - context.runCApiHooks(); /* * C++ libraries sometimes declare global objects that have destructors that call @@ -902,7 +982,6 @@ private static CApiContext loadCApi(Node node, PythonContext context, TruffleStr Object finalizingPointer = SignatureLibrary.getUncached().call(finalizeSignature, finalizeFunction); try { cApiContext.addNativeFinalizer(context, finalizingPointer); - cApiContext.runBackgroundGCTask(context); } catch (RuntimeException e) { // This can happen when other languages restrict multithreading LOGGER.warning(() -> "didn't register a native finalizer due to: " + e.getMessage()); @@ -1106,7 +1185,7 @@ public void finalizeCApi() { * allocated resources (e.g. native object stubs). Calling * 'CApiTransitions.pollReferenceQueue' could then lead to a double-free. */ - CApiTransitions.disableReferenceQueuePolling(handleContext); + CApiTransitions.disableReferenceQueuePollingPermanently(handleContext); TruffleSafepoint sp = TruffleSafepoint.getCurrent(); boolean prev = sp.setAllowActions(false); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/NativeCAPISymbol.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/NativeCAPISymbol.java index a842633b53..14451d038c 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/NativeCAPISymbol.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/NativeCAPISymbol.java @@ -134,6 +134,7 @@ public enum NativeCAPISymbol implements NativeCExtSymbol { FUN_TRUFFLE_CHECK_TYPE_READY("GraalPyPrivate_CheckTypeReady", ArgDescriptor.Void, PyTypeObject), FUN_GRAALPY_GC_COLLECT("GraalPyPrivate_GC_Collect", Py_ssize_t, Int), FUN_SUBTYPE_TRAVERSE("GraalPyPrivate_SubtypeTraverse", Int, PyObject, Pointer, Pointer), + FUN_INIT_THREAD_STATE_CURRENT("GraalPyPrivate_InitThreadStateCurrent", Void), /* PyDateTime_CAPI */ diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java index c420e5b03c..37929c6018 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java @@ -40,11 +40,11 @@ import static com.oracle.graal.python.nodes.BuiltinNames.T_SHA3; import static com.oracle.graal.python.nodes.BuiltinNames.T_STDERR; import static com.oracle.graal.python.nodes.BuiltinNames.T_STDOUT; -import static com.oracle.graal.python.nodes.BuiltinNames.T___STDOUT__; import static com.oracle.graal.python.nodes.BuiltinNames.T_SYS; import static com.oracle.graal.python.nodes.BuiltinNames.T_THREADING; import static com.oracle.graal.python.nodes.BuiltinNames.T___BUILTINS__; import static com.oracle.graal.python.nodes.BuiltinNames.T___MAIN__; +import static com.oracle.graal.python.nodes.BuiltinNames.T___STDOUT__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___ANNOTATIONS__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___FILE__; import static com.oracle.graal.python.nodes.SpecialMethodNames.T_INSERT; @@ -112,6 +112,8 @@ import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.cext.PythonNativeClass; import com.oracle.graal.python.builtins.objects.cext.capi.CApiContext; +import com.oracle.graal.python.builtins.objects.cext.capi.CExtNodes.PCallCapiFunction; +import com.oracle.graal.python.builtins.objects.cext.capi.NativeCAPISymbol; import com.oracle.graal.python.builtins.objects.cext.capi.PThreadState; import com.oracle.graal.python.builtins.objects.cext.capi.PythonNativeWrapper.PythonAbstractObjectNativeWrapper; import com.oracle.graal.python.builtins.objects.cext.capi.transitions.CApiTransitions; @@ -722,12 +724,19 @@ PythonThreadState getThreadState(Node n) { private OutputStream out; private OutputStream err; private InputStream in; + + private static final int CAPI_UNINITIALIZED = 0; + private static final int CAPI_INITIALIZING = 1; + private static final int CAPI_INITIALIZED = 2; + private static final int CAPI_FAILED = 3; + + /** Initialization state of the C API context. */ + private volatile int cApiState; private final ReentrantLock cApiInitializationLock = new ReentrantLock(false); - private volatile boolean cApiWasInitialized = false; @CompilationFinal private CApiContext cApiContext; @CompilationFinal private boolean nativeAccessAllowed; - private TruffleString soABI; // cache for soAPI + private TruffleString soABI; private static final class GlobalInterpreterLock extends ReentrantLock { private static final long serialVersionUID = 1L; @@ -1967,7 +1976,7 @@ public void clearAtexitHooks() { } public void registerCApiHook(Runnable hook) { - if (hasCApiContext()) { + if (isCApiInitialized()) { hook.run(); } else { capiHooks.add(hook); @@ -2567,6 +2576,10 @@ public void initializeMultiThreading() { public synchronized void attachThread(Thread thread, ContextThreadLocal threadState) { CompilerAsserts.neverPartOfCompilation(); threadStateMapping.put(thread, threadState.get(thread)); + if (isCapiInitializing() || isCApiInitialized()) { + // initialize this thread's native TLS slot eagerly instead of on first use + PCallCapiFunction.callUncached(NativeCAPISymbol.FUN_INIT_THREAD_STATE_CURRENT); + } } public synchronized void disposeThread(Thread thread, boolean canRunGuestCode) { @@ -2595,24 +2608,53 @@ private static void releaseSentinelLock(WeakReference sentinelLockWeakref } public boolean hasCApiContext() { - // This may be called during C API initialization, we have a context so that we can finish - // the initialization, but the C API is not fully initialized yet - assert (cApiContext != null) || !cApiWasInitialized; + /* + * This may be called during C API initialization, we have a context so that we can finish + * the initialization, but the C API is not fully initialized yet + */ + assert (cApiContext != null) || cApiState != CAPI_INITIALIZED; return cApiContext != null; } public boolean isCApiInitialized() { - assert (cApiContext != null) || !cApiWasInitialized; - return cApiWasInitialized; + assert cApiContext != null || cApiState != CAPI_INITIALIZED; + return cApiState == CAPI_INITIALIZED; + } + + public boolean isCapiInitializing() { + assert cApiContext != null || cApiState != CAPI_INITIALIZING; + return cApiState == CAPI_INITIALIZING; + } + + public void startCApiThreadStateInit() { + assert cApiContext != null; + assert cApiState == CAPI_UNINITIALIZED; + cApiState = CAPI_INITIALIZING; + } + + private void setCapiState(int state) { + /*- Allowed transitions: + * UNINITIALIZED -> INITIALIZING + * INITIALIZING -> INITIALIZED, FAILED + */ + assert state != CAPI_UNINITIALIZED; + assert cApiState != CAPI_UNINITIALIZED || state == CAPI_INITIALIZING; + assert cApiState != CAPI_INITIALIZING || state == CAPI_INITIALIZED || state == CAPI_FAILED; + cApiState = state; + } + + public void finishCApiThreadStateInit() { + cApiState = CAPI_INITIALIZED; } public void setCApiInitialized() { assert cApiContext != null; - cApiWasInitialized = true; + assert cApiState != CAPI_INITIALIZED; + cApiState = CAPI_INITIALIZED; } public CApiContext getCApiContext() { - assert (cApiContext != null) || !cApiWasInitialized; + assert (cApiContext != null) || cApiState == CAPI_UNINITIALIZED; return cApiContext; } From 6fc35839727c554b9ec4a308a7088e53ae236a7f Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Wed, 25 Mar 2026 18:09:51 +0100 Subject: [PATCH 03/14] Consolidate C API state methods --- .../builtins/modules/ImpModuleBuiltins.java | 2 +- .../objects/cext/capi/CApiContext.java | 57 +++++++++------- .../builtins/objects/cext/capi/CExtNodes.java | 3 +- .../graal/python/runtime/PythonContext.java | 68 +++++++------------ 4 files changed, 59 insertions(+), 71 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java index 1d4752f59b..ff8e6742a2 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java @@ -284,7 +284,7 @@ private static int doExec(Node node, PythonContext context, PythonModule extensi return 0; } - if (!context.hasCApiContext()) { + if (context.getCApiState() != PythonContext.CApiState.INITIALIZED) { throw PRaiseNode.raiseStatic(node, PythonBuiltinClassType.SystemError, ErrorMessages.CAPI_NOT_YET_INITIALIZED); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java index c03cfbb927..5223a8c8f2 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java @@ -800,7 +800,7 @@ public static CApiContext ensureCapiWasLoaded(Node node, PythonContext context, assert PythonContext.get(null).ownsGil(); // unsafe lazy initialization // The initialization may run Python code (e.g., module import in // GraalPyPrivate_InitBuiltinTypesAndStructs), so just holding the GIL is not enough - if (!context.isCApiInitialized()) { + if (context.getCApiState() != PythonContext.CApiState.INITIALIZED) { // We import those modules ahead of the initialization without the initialization lock // to avoid deadlocks. We would have imported them in the initialization anyway, this @@ -814,30 +814,39 @@ public static CApiContext ensureCapiWasLoaded(Node node, PythonContext context, TruffleSafepoint.setBlockedThreadInterruptible(node, ReentrantLock::lockInterruptibly, initLock); } try { - if (!context.isCApiInitialized()) { - // loadCApi must set C API context half-way through its execution so that it can - // run internal Java code that needs C API context - TruffleSafepoint safepoint = TruffleSafepoint.getCurrent(); - boolean prevAllowSideEffects = safepoint.setAllowSideEffects(false); + PythonContext.CApiState state = context.getCApiState(); + if (state == PythonContext.CApiState.INITIALIZED || state == PythonContext.CApiState.INITIALIZING) { + return context.getCApiContext(); + } + if (state == PythonContext.CApiState.FAILED) { + throw new ApiInitException(toTruffleStringUncached("The C API initialization has previously failed.")); + } + + assert state == PythonContext.CApiState.UNINITIALIZED : state; + // loadCApi must set C API context half-way through its execution so that it can + // run internal Java code that needs C API context + TruffleSafepoint safepoint = TruffleSafepoint.getCurrent(); + boolean prevAllowSideEffects = safepoint.setAllowSideEffects(false); + try { + CApiContext cApiContext = loadCApi(node, context, name, path, reason); + assert context.getCApiState() == PythonContext.CApiState.INITIALIZING; + initializeThreadStateCurrentForAttachedThreads(node, context); + CApiTransitions.initializeReferenceQueuePolling(context.nativeContext); + context.runCApiHooks(); + context.setCApiState(PythonContext.CApiState.INITIALIZED); // volatile write try { - CApiContext cApiContext = loadCApi(node, context, name, path, reason); - initializeThreadStateCurrentForAttachedThreads(node, context); - CApiTransitions.initializeReferenceQueuePolling(context.nativeContext); - context.runCApiHooks(); - context.setCApiInitialized(); // volatile write - context.finishCApiThreadStateInit(); - try { - cApiContext.runBackgroundGCTask(context); - } catch (RuntimeException e) { - // This can happen when other languages restrict multithreading - LOGGER.warning(() -> "didn't start the background GC task due to: " + e.getMessage()); - } - } catch (Throwable t) { - context.finishCApiThreadStateInit(); - throw t; - } finally { - safepoint.setAllowSideEffects(prevAllowSideEffects); + cApiContext.runBackgroundGCTask(context); + } catch (RuntimeException e) { + // This can happen when other languages restrict multithreading + LOGGER.warning(() -> "didn't start the background GC task due to: " + e.getMessage()); + } + } catch (Throwable t) { + if (context.getCApiState() == PythonContext.CApiState.INITIALIZING) { + context.setCApiState(PythonContext.CApiState.FAILED); } + throw t; + } finally { + safepoint.setAllowSideEffects(prevAllowSideEffects); } } finally { initLock.unlock(); @@ -952,6 +961,7 @@ private static CApiContext loadCApi(Node node, PythonContext context, TruffleStr Object initFunction = U.readMember(capiLibrary, "initialize_graal_capi"); CApiContext cApiContext = new CApiContext(context, capiLibrary, loc); context.setCApiContext(cApiContext); + context.setCApiState(PythonContext.CApiState.INITIALIZING); try (BuiltinArrayWrapper builtinArrayWrapper = new BuiltinArrayWrapper()) { /* @@ -965,7 +975,6 @@ private static CApiContext loadCApi(Node node, PythonContext context, TruffleStr U.execute(initFunction, builtinArrayWrapper, gcState); } - context.startCApiThreadStateInit(); assert PythonCApiAssertions.assertBuiltins(capiLibrary); cApiContext.pyDateTimeCAPICapsule = PyDateTimeCAPIWrapper.initWrapper(context, cApiContext); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java index 089ac42070..b2fcbd59ad 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java @@ -849,7 +849,8 @@ static Object doWithoutContext(NativeCAPISymbol symbol, Object[] args, @Cached EnsureTruffleStringNode ensureTruffleStringNode) { try { PythonContext pythonContext = PythonContext.get(inliningTarget); - if (!pythonContext.hasCApiContext()) { + PythonContext.CApiState capiState = pythonContext.getCApiState(); + if (capiState != PythonContext.CApiState.INITIALIZING && capiState != PythonContext.CApiState.INITIALIZED) { CompilerDirectives.transferToInterpreterAndInvalidate(); CApiContext.ensureCapiWasLoaded("call internal native GraalPy function"); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java index 37929c6018..450e3c8121 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java @@ -725,13 +725,15 @@ PythonThreadState getThreadState(Node n) { private OutputStream err; private InputStream in; - private static final int CAPI_UNINITIALIZED = 0; - private static final int CAPI_INITIALIZING = 1; - private static final int CAPI_INITIALIZED = 2; - private static final int CAPI_FAILED = 3; + public enum CApiState { + UNINITIALIZED, + INITIALIZING, + INITIALIZED, + FAILED + } /** Initialization state of the C API context. */ - private volatile int cApiState; + private volatile CApiState cApiState = CApiState.UNINITIALIZED; private final ReentrantLock cApiInitializationLock = new ReentrantLock(false); @CompilationFinal private CApiContext cApiContext; @CompilationFinal private boolean nativeAccessAllowed; @@ -1976,7 +1978,7 @@ public void clearAtexitHooks() { } public void registerCApiHook(Runnable hook) { - if (isCApiInitialized()) { + if (getCApiState() == CApiState.INITIALIZED) { hook.run(); } else { capiHooks.add(hook); @@ -2576,7 +2578,8 @@ public void initializeMultiThreading() { public synchronized void attachThread(Thread thread, ContextThreadLocal threadState) { CompilerAsserts.neverPartOfCompilation(); threadStateMapping.put(thread, threadState.get(thread)); - if (isCapiInitializing() || isCApiInitialized()) { + CApiState state = getCApiState(); + if (state == CApiState.INITIALIZING || state == CApiState.INITIALIZED) { // initialize this thread's native TLS slot eagerly instead of on first use PCallCapiFunction.callUncached(NativeCAPISymbol.FUN_INIT_THREAD_STATE_CURRENT); } @@ -2607,54 +2610,28 @@ private static void releaseSentinelLock(WeakReference sentinelLockWeakref } } - public boolean hasCApiContext() { - /* - * This may be called during C API initialization, we have a context so that we can finish - * the initialization, but the C API is not fully initialized yet - */ - assert (cApiContext != null) || cApiState != CAPI_INITIALIZED; - return cApiContext != null; - } - - public boolean isCApiInitialized() { - assert cApiContext != null || cApiState != CAPI_INITIALIZED; - return cApiState == CAPI_INITIALIZED; - } - - public boolean isCapiInitializing() { - assert cApiContext != null || cApiState != CAPI_INITIALIZING; - return cApiState == CAPI_INITIALIZING; - } - - public void startCApiThreadStateInit() { - assert cApiContext != null; - assert cApiState == CAPI_UNINITIALIZED; - cApiState = CAPI_INITIALIZING; + public CApiState getCApiState() { + assert cApiContext != null || cApiState == CApiState.UNINITIALIZED : cApiState; + return cApiState; } - private void setCapiState(int state) { + public void setCApiState(CApiState state) { /*- Allowed transitions: * UNINITIALIZED -> INITIALIZING * INITIALIZING -> INITIALIZED, FAILED */ - assert state != CAPI_UNINITIALIZED; - assert cApiState != CAPI_UNINITIALIZED || state == CAPI_INITIALIZING; - assert cApiState != CAPI_INITIALIZING || state == CAPI_INITIALIZED || state == CAPI_FAILED; - cApiState = state; - } - - public void finishCApiThreadStateInit() { - cApiState = CAPI_INITIALIZED; - } - - public void setCApiInitialized() { + assert state != CApiState.UNINITIALIZED; + assert cApiInitializationLock.isHeldByCurrentThread(); assert cApiContext != null; - assert cApiState != CAPI_INITIALIZED; - cApiState = CAPI_INITIALIZED; + assert cApiState != CApiState.UNINITIALIZED || state == CApiState.INITIALIZING; + assert cApiState != CApiState.INITIALIZING || state == CApiState.INITIALIZED || state == CApiState.FAILED; + assert cApiState != CApiState.INITIALIZED; + assert cApiState != CApiState.FAILED; + cApiState = state; } public CApiContext getCApiContext() { - assert (cApiContext != null) || cApiState == CAPI_UNINITIALIZED; + assert cApiContext != null || cApiState == CApiState.UNINITIALIZED; return cApiContext; } @@ -2664,6 +2641,7 @@ public ReentrantLock getcApiInitializationLock() { public void setCApiContext(CApiContext capiContext) { assert this.cApiContext == null : "tried to create new C API context but it was already created"; + assert getCApiState() == CApiState.UNINITIALIZED; this.cApiContext = capiContext; } From 69fde5b44d14afcf056f784e3267ec4b69abc221 Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Thu, 26 Mar 2026 09:09:30 +0100 Subject: [PATCH 04/14] Avoid upcalls during C API initialization --- .../com.oracle.graal.python.cext/src/capi.c | 6 +-- .../objects/cext/capi/CApiContext.java | 6 +-- .../objects/cext/capi/NativeCAPISymbol.java | 2 +- .../graal/python/runtime/PythonContext.java | 50 +++++++++++++++---- 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/graalpython/com.oracle.graal.python.cext/src/capi.c b/graalpython/com.oracle.graal.python.cext/src/capi.c index ce88ac5f95..85944db149 100644 --- a/graalpython/com.oracle.graal.python.cext/src/capi.c +++ b/graalpython/com.oracle.graal.python.cext/src/capi.c @@ -272,13 +272,13 @@ PyObject* _Py_NotImplementedStructReference; */ THREAD_LOCAL PyThreadState *tstate_current = NULL; -PyAPI_FUNC(void) GraalPyPrivate_InitThreadStateCurrent(PyThreadState *tstate) { +PyAPI_FUNC(PyThreadState **) GraalPyPrivate_InitThreadStateCurrent(PyThreadState *tstate) { tstate_current = tstate; + return &tstate_current; } static void initialize_globals() { - // initialize the current thread's TLS slot for PyThreadState_Get() - GraalPyPrivate_InitThreadStateCurrent(); + GraalPyPrivate_InitThreadStateCurrent(GraalPyPrivate_ThreadState_Get(&tstate_current)); _Py_NoneStructReference = GraalPyPrivate_None(); _Py_NotImplementedStructReference = GraalPyPrivate_NotImplemented(); _Py_EllipsisObjectReference = GraalPyPrivate_Ellipsis(); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java index 5223a8c8f2..2db012ae1e 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java @@ -841,9 +841,7 @@ public static CApiContext ensureCapiWasLoaded(Node node, PythonContext context, LOGGER.warning(() -> "didn't start the background GC task due to: " + e.getMessage()); } } catch (Throwable t) { - if (context.getCApiState() == PythonContext.CApiState.INITIALIZING) { - context.setCApiState(PythonContext.CApiState.FAILED); - } + context.setCApiState(PythonContext.CApiState.FAILED); throw t; } finally { safepoint.setAllowSideEffects(prevAllowSideEffects); @@ -864,7 +862,7 @@ private static void initializeThreadStateCurrentForAttachedThreads(Node node, Py ThreadLocalAction action = new ThreadLocalAction(true, false) { @Override protected void perform(ThreadLocalAction.Access access) { - PCallCapiFunction.callUncached(NativeCAPISymbol.FUN_INIT_THREAD_STATE_CURRENT); + context.initializeNativeThreadState(); } }; waitForThreadLocalActions(node, context, submitThreadLocalActions(context, threads, action)); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/NativeCAPISymbol.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/NativeCAPISymbol.java index 14451d038c..a45b84c36b 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/NativeCAPISymbol.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/NativeCAPISymbol.java @@ -134,7 +134,7 @@ public enum NativeCAPISymbol implements NativeCExtSymbol { FUN_TRUFFLE_CHECK_TYPE_READY("GraalPyPrivate_CheckTypeReady", ArgDescriptor.Void, PyTypeObject), FUN_GRAALPY_GC_COLLECT("GraalPyPrivate_GC_Collect", Py_ssize_t, Int), FUN_SUBTYPE_TRAVERSE("GraalPyPrivate_SubtypeTraverse", Int, PyObject, Pointer, Pointer), - FUN_INIT_THREAD_STATE_CURRENT("GraalPyPrivate_InitThreadStateCurrent", Void), + FUN_INIT_THREAD_STATE_CURRENT("GraalPyPrivate_InitThreadStateCurrent", Pointer, PyThreadState), /* PyDateTime_CAPI */ diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java index 450e3c8121..005ebb10bc 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java @@ -2577,11 +2577,40 @@ public void initializeMultiThreading() { public synchronized void attachThread(Thread thread, ContextThreadLocal threadState) { CompilerAsserts.neverPartOfCompilation(); - threadStateMapping.put(thread, threadState.get(thread)); - CApiState state = getCApiState(); - if (state == CApiState.INITIALIZING || state == CApiState.INITIALIZED) { - // initialize this thread's native TLS slot eagerly instead of on first use - PCallCapiFunction.callUncached(NativeCAPISymbol.FUN_INIT_THREAD_STATE_CURRENT); + PythonThreadState pythonThreadState = threadState.get(thread); + threadStateMapping.put(thread, pythonThreadState); + ReentrantLock initLock = getcApiInitializationLock(); + /* + * Synchronize with C API initialization so that we do not miss eager initialization of this + * thread's 'tstate_current'. Otherwise, a thread could attach while another thread is + * sweeping all already-attached threads during C API initialization, observe + * 'INITIALIZING', skip eager initialization here, and then also miss the initialization + * sweep because it was not yet part of the thread snapshot. + */ + initLock.lock(); + try { + if (getCApiState() == CApiState.INITIALIZED) { + // initialize this thread's native TLS slot eagerly instead of on first use + initializeNativeThreadState(pythonThreadState); + } + } finally { + initLock.unlock(); + } + } + + @TruffleBoundary + public void initializeNativeThreadState() { + initializeNativeThreadState(getThreadState(getLanguage())); + } + + @SuppressWarnings("try") + public void initializeNativeThreadState(PythonThreadState pythonThreadState) { + CompilerAsserts.neverPartOfCompilation(); + try (GilNode.UncachedAcquire ignored = GilNode.uncachedAcquire()) { + assert getCApiContext() != null; + Object nativeThreadState = PThreadState.getOrCreateNativeThreadState(pythonThreadState); + Object nativeThreadLocalVarPointer = PCallCapiFunction.callUncached(NativeCAPISymbol.FUN_INIT_THREAD_STATE_CURRENT, nativeThreadState); + pythonThreadState.setNativeThreadLocalVarPointer(nativeThreadLocalVarPointer); } } @@ -2611,19 +2640,20 @@ private static void releaseSentinelLock(WeakReference sentinelLockWeakref } public CApiState getCApiState() { - assert cApiContext != null || cApiState == CApiState.UNINITIALIZED : cApiState; + assert cApiContext != null || cApiState == CApiState.UNINITIALIZED || cApiState == CApiState.FAILED : cApiState; return cApiState; } public void setCApiState(CApiState state) { /*- Allowed transitions: - * UNINITIALIZED -> INITIALIZING + * UNINITIALIZED -> INITIALIZING, FAILED * INITIALIZING -> INITIALIZED, FAILED */ assert state != CApiState.UNINITIALIZED; assert cApiInitializationLock.isHeldByCurrentThread(); - assert cApiContext != null; - assert cApiState != CApiState.UNINITIALIZED || state == CApiState.INITIALIZING; + assert state != CApiState.INITIALIZING || cApiContext != null; + assert state != CApiState.INITIALIZED || cApiContext != null; + assert cApiState != CApiState.UNINITIALIZED || state == CApiState.INITIALIZING || state == CApiState.FAILED; assert cApiState != CApiState.INITIALIZING || state == CApiState.INITIALIZED || state == CApiState.FAILED; assert cApiState != CApiState.INITIALIZED; assert cApiState != CApiState.FAILED; @@ -2631,7 +2661,7 @@ public void setCApiState(CApiState state) { } public CApiContext getCApiContext() { - assert cApiContext != null || cApiState == CApiState.UNINITIALIZED; + assert cApiContext != null || cApiState == CApiState.UNINITIALIZED || cApiState == CApiState.FAILED; return cApiContext; } From fd52bebb215ac5180e6b9fbf8d6256fc9f01da33 Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Thu, 26 Mar 2026 10:12:50 +0100 Subject: [PATCH 05/14] Pass PyThreadState to initialize_graal_capi --- .../com.oracle.graal.python.cext/src/capi.c | 9 ++-- .../cext/PythonCextPyStateBuiltins.java | 27 +---------- .../objects/cext/capi/CApiContext.java | 9 +++- .../objects/cext/capi/PThreadState.java | 47 ++++++++++++++----- 4 files changed, 49 insertions(+), 43 deletions(-) diff --git a/graalpython/com.oracle.graal.python.cext/src/capi.c b/graalpython/com.oracle.graal.python.cext/src/capi.c index 85944db149..3c8e0c4afd 100644 --- a/graalpython/com.oracle.graal.python.cext/src/capi.c +++ b/graalpython/com.oracle.graal.python.cext/src/capi.c @@ -277,8 +277,8 @@ PyAPI_FUNC(PyThreadState **) GraalPyPrivate_InitThreadStateCurrent(PyThreadState return &tstate_current; } -static void initialize_globals() { - GraalPyPrivate_InitThreadStateCurrent(GraalPyPrivate_ThreadState_Get(&tstate_current)); +static void initialize_globals(PyThreadState *tstate) { + GraalPyPrivate_InitThreadStateCurrent(tstate); _Py_NoneStructReference = GraalPyPrivate_None(); _Py_NotImplementedStructReference = GraalPyPrivate_NotImplemented(); _Py_EllipsisObjectReference = GraalPyPrivate_Ellipsis(); @@ -671,7 +671,7 @@ Py_LOCAL_SYMBOL TruffleContext* TRUFFLE_CONTEXT; */ Py_LOCAL_SYMBOL int8_t *_graalpy_finalizing = NULL; -PyAPI_FUNC(void) initialize_graal_capi(TruffleEnv* env, void **builtin_closures, GCState *gc) { +PyAPI_FUNC(PyThreadState **) initialize_graal_capi(TruffleEnv* env, void **builtin_closures, GCState *gc, PyThreadState *tstate) { clock_t t = clock(); if (env) { @@ -710,7 +710,7 @@ PyAPI_FUNC(void) initialize_graal_capi(TruffleEnv* env, void **builtin_closures, initialize_builtin_types_and_structs(); // initialize global variables like '_Py_NoneStruct', etc. - initialize_globals(); + initialize_globals(tstate); initialize_exceptions(); initialize_hashes(); initialize_bufferprocs(); @@ -721,6 +721,7 @@ PyAPI_FUNC(void) initialize_graal_capi(TruffleEnv* env, void **builtin_closures, Py_FileSystemDefaultEncoding = "utf-8"; // strdup(PyUnicode_AsUTF8(GraalPyPrivate_FileSystemDefaultEncoding())); GraalPyPrivate_Log(PY_TRUFFLE_LOG_FINE, "initialize_graal_capi: %fs", ((double) (clock() - t)) / CLOCKS_PER_SEC); + return &tstate_current; } /* diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextPyStateBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextPyStateBuiltins.java index 571a273492..89859c9550 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextPyStateBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextPyStateBuiltins.java @@ -43,7 +43,6 @@ import static com.oracle.graal.python.builtins.modules.cext.PythonCextBuiltins.CApiCallPath.Direct; import static com.oracle.graal.python.builtins.modules.cext.PythonCextBuiltins.CApiCallPath.Ignored; import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.Int; -import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.Pointer; import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PyFrameObjectTransfer; import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PyObject; import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PyObjectBorrowed; @@ -71,7 +70,6 @@ import com.oracle.graal.python.runtime.GilNode; import com.oracle.graal.python.runtime.PythonContext; import com.oracle.graal.python.runtime.PythonContext.PythonThreadState; -import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.graal.python.util.OverflowException; import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.ThreadLocalAction; @@ -79,8 +77,6 @@ import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; import com.oracle.truffle.api.dsl.Specialization; -import com.oracle.truffle.api.interop.InteropLibrary; -import com.oracle.truffle.api.library.CachedLibrary; import com.oracle.truffle.api.nodes.Node; import com.oracle.truffle.api.nodes.RootNode; @@ -116,22 +112,6 @@ static Object restore( } } - @CApiBuiltin(ret = PyThreadState, args = {Pointer}, call = Ignored) - abstract static class GraalPyPrivate_ThreadState_Get extends CApiUnaryBuiltinNode { - - @Specialization(limit = "1") - static Object get(Object tstateCurrentPtr, - @Bind Node inliningTarget, - @Bind PythonContext context, - @CachedLibrary("tstateCurrentPtr") InteropLibrary lib) { - PythonThreadState pythonThreadState = context.getThreadState(context.getLanguage(inliningTarget)); - if (!lib.isNull(tstateCurrentPtr)) { - pythonThreadState.setNativeThreadLocalVarPointer(tstateCurrentPtr); - } - return PThreadState.getOrCreateNativeThreadState(pythonThreadState); - } - } - @CApiBuiltin(ret = Void, args = {}, call = Ignored) abstract static class GraalPyPrivate_BeforeThreadDetach extends CApiNullaryBuiltinNode { @Specialization @@ -151,12 +131,7 @@ static PDict get( @Bind Node inliningTarget, @Bind PythonContext context) { PythonThreadState threadState = context.getThreadState(context.getLanguage(inliningTarget)); - PDict threadStateDict = threadState.getDict(); - if (threadStateDict == null) { - threadStateDict = PFactory.createDict(context.getLanguage()); - threadState.setDict(threadStateDict); - } - return threadStateDict; + return PThreadState.getOrCreateThreadStateDict(context, threadState); } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java index 2db012ae1e..44c0d349f5 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java @@ -968,9 +968,14 @@ private static CApiContext loadCApi(Node node, PythonContext context, TruffleStr * then already require the GC state. */ Object gcState = cApiContext.createGCState(); - Object signature = env.parseInternal(Source.newBuilder(J_NFI_LANGUAGE, "(ENV,POINTER,POINTER):VOID", "exec").build()).call(); + PythonThreadState currentThreadState = context.getThreadState(context.getLanguage()); + Object nativeThreadState = PThreadState.getOrCreateNativeThreadState(currentThreadState); + Object signature = env.parseInternal(Source.newBuilder(J_NFI_LANGUAGE, "(ENV,POINTER,POINTER,POINTER):POINTER", "exec").build()).call(); initFunction = SignatureLibrary.getUncached().bind(signature, initFunction); - U.execute(initFunction, builtinArrayWrapper, gcState); + Object nativeThreadLocalVarPointer = U.execute(initFunction, builtinArrayWrapper, gcState, nativeThreadState); + assert U.isPointer(nativeThreadLocalVarPointer); + assert !U.isNull(nativeThreadLocalVarPointer); + currentThreadState.setNativeThreadLocalVarPointer(nativeThreadLocalVarPointer); } assert PythonCApiAssertions.assertBuiltins(capiLibrary); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/PThreadState.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/PThreadState.java index 08fc5d4735..e7992678e7 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/PThreadState.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/PThreadState.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2019, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -41,16 +41,18 @@ package com.oracle.graal.python.builtins.objects.cext.capi; import com.oracle.graal.python.PythonLanguage; +import com.oracle.graal.python.builtins.objects.PNone; import com.oracle.graal.python.builtins.objects.cext.capi.PythonNativeWrapper.PythonStructNativeWrapper; import com.oracle.graal.python.builtins.objects.cext.capi.transitions.CApiTransitions; import com.oracle.graal.python.builtins.objects.cext.capi.transitions.CApiTransitions.PythonToNativeNode; -import com.oracle.graal.python.builtins.objects.cext.capi.transitions.CApiTransitionsFactory.PythonToNativeNodeGen; import com.oracle.graal.python.builtins.objects.cext.common.NativePointer; import com.oracle.graal.python.builtins.objects.cext.structs.CFields; import com.oracle.graal.python.builtins.objects.cext.structs.CStructAccess; +import com.oracle.graal.python.builtins.objects.cext.structs.CStructAccess.ReadObjectNode; import com.oracle.graal.python.builtins.objects.cext.structs.CStructs; import com.oracle.graal.python.builtins.objects.dict.PDict; import com.oracle.graal.python.runtime.PythonContext; +import com.oracle.graal.python.runtime.PythonContext.CApiState; import com.oracle.graal.python.runtime.PythonContext.PythonThreadState; import com.oracle.graal.python.runtime.object.PFactory; import com.oracle.truffle.api.CompilerDirectives; @@ -78,7 +80,7 @@ public final class PThreadState extends PythonStructNativeWrapper { @TruffleBoundary private PThreadState(PythonThreadState threadState) { super(threadState); - long ptr = allocateCLayout(threadState); + long ptr = allocateCLayout(); CApiTransitions.createReference(this, ptr, true); // TODO: wrap in NativePointer for NFI replacement = new NativePointer(ptr); @@ -110,18 +112,41 @@ public PythonThreadState getThreadState() { } @TruffleBoundary - private static long allocateCLayout(PythonThreadState threadState) { - PythonToNativeNode toNative = PythonToNativeNodeGen.getUncached(); + public static PDict getOrCreateThreadStateDict(PythonContext context, PythonThreadState threadState) { + /* + * C API initialization must be finished at that time. This implies that there is already a + * native thread state. + */ + assert context.getCApiState() == CApiState.INITIALIZED; + Object nativeThreadState = PThreadState.getNativeThreadState(threadState); + assert nativeThreadState != null; + + PDict threadStateDict = threadState.getDict(); + if (threadStateDict != null) { + assert threadStateDict == ReadObjectNode.getUncached().read(nativeThreadState, CFields.PyThreadState__dict); + return threadStateDict; + } + + threadStateDict = PFactory.createDict(context.getLanguage()); + threadState.setDict(threadStateDict); + assert ReadObjectNode.getUncached().read(nativeThreadState, CFields.PyThreadState__dict) == PNone.NO_VALUE; + CStructAccess.WritePointerNode.writeUncached(nativeThreadState, CFields.PyThreadState__dict, PythonToNativeNode.executeUncached(threadStateDict)); + + return threadStateDict; + } + + @TruffleBoundary + private static long allocateCLayout() { long ptr = CStructAccess.AllocateNode.allocUncachedPointer(CStructs.PyThreadState.size()); CStructAccess.WritePointerNode writePtrNode = CStructAccess.WritePointerNode.getUncached(); PythonContext pythonContext = PythonContext.get(null); - PDict threadStateDict = threadState.getDict(); - if (threadStateDict == null) { - threadStateDict = PFactory.createDict(pythonContext.getLanguage()); - threadState.setDict(threadStateDict); - } - writePtrNode.write(ptr, CFields.PyThreadState__dict, toNative.execute(threadStateDict)); + /* + * As in CPython, the thread state dict is initialized lazily. This is necessary to avoid + * cycles in the bootstrapping process because creating the dict will need the GC state + * which needs the thread state. + */ + writePtrNode.write(ptr, CFields.PyThreadState__dict, pythonContext.getNativeNull()); CApiContext cApiContext = pythonContext.getCApiContext(); Object smallInts = CStructAccess.AllocateNode.allocUncached((PY_NSMALLNEGINTS + PY_NSMALLPOSINTS) * CStructAccess.POINTER_SIZE); writePtrNode.write(ptr, CFields.PyThreadState__small_ints, smallInts); From 7816da4e9c97f6de5beaf42791cb4d6a1bef1b8a Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Thu, 26 Mar 2026 23:27:41 +0100 Subject: [PATCH 06/14] Fix copyrights --- graalpython/com.oracle.graal.python.cext/src/pystate.c | 2 +- .../python/builtins/modules/cext/PythonCextPyStateBuiltins.java | 2 +- .../builtins/objects/cext/capi/transitions/CApiTransitions.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/graalpython/com.oracle.graal.python.cext/src/pystate.c b/graalpython/com.oracle.graal.python.cext/src/pystate.c index 90e67497f1..c3beb17613 100644 --- a/graalpython/com.oracle.graal.python.cext/src/pystate.c +++ b/graalpython/com.oracle.graal.python.cext/src/pystate.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2024, 2025, Oracle and/or its affiliates. +/* Copyright (c) 2024, 2026, Oracle and/or its affiliates. * Copyright (C) 1996-2024 Python Software Foundation * * Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextPyStateBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextPyStateBuiltins.java index 89859c9550..b5b897bcc5 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextPyStateBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextPyStateBuiltins.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/CApiTransitions.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/CApiTransitions.java index 57d4f3673c..2baf394e78 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/CApiTransitions.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/CApiTransitions.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 From 0a0ec5e247dfb4ac395cfa3c8f9d8f615200d31e Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Fri, 27 Mar 2026 09:53:43 +0100 Subject: [PATCH 07/14] Guard C API polling by thread state --- .../cext/capi/transitions/CApiTransitions.java | 12 ++++++++++++ .../oracle/graal/python/runtime/PythonContext.java | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/CApiTransitions.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/CApiTransitions.java index 2baf394e78..dac761033b 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/CApiTransitions.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/transitions/CApiTransitions.java @@ -454,10 +454,22 @@ public static int pollReferenceQueue() { if (handleContext.referenceQueuePollingState != RQ_READY) { return manuallyCollected; } + /* + * Polling the reference queue may deallocate native GC objects and therefore re-enter + * native code paths that use '_PyThreadState_GET()' to obtain the current thread's GC + * state. So, we may only poll once the current thread has installed its native + * 'tstate_current' pointer. + */ + if (!context.getThreadState(context.getLanguage()).isNativeThreadStateInitialized()) { + return manuallyCollected; + } try (GilNode.UncachedAcquire ignored = GilNode.uncachedAcquire()) { if (handleContext.referenceQueuePollingState != RQ_READY) { return manuallyCollected; } + if (!context.getThreadState(context.getLanguage()).isNativeThreadStateInitialized()) { + return manuallyCollected; + } ReferenceQueue queue = handleContext.referenceQueue; int count = 0; long start = 0; diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java index 005ebb10bc..0d02d86c67 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java @@ -592,6 +592,10 @@ public void setNativeThreadLocalVarPointer(Object ptr) { String.format("ptr = %s; nativeThreadLocalVarPointer = %s", ptr, nativeThreadLocalVarPointer); this.nativeThreadLocalVarPointer = ptr; } + + public boolean isNativeThreadStateInitialized() { + return nativeThreadLocalVarPointer != null; + } } private static final class AtExitHook { From 34d72e4aaec2f83e970702dfbe88624d3217d018 Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Tue, 31 Mar 2026 14:52:08 +0200 Subject: [PATCH 08/14] Initialize native thread state before downcall --- .../objects/cext/capi/CApiContext.java | 45 +++---------------- .../builtins/objects/cext/capi/CExtNodes.java | 9 ++++ .../cext/capi/ExternalFunctionNodes.java | 1 + .../graal/python/runtime/PythonContext.java | 6 +++ 4 files changed, 21 insertions(+), 40 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java index 44c0d349f5..2274b85b21 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java @@ -62,12 +62,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; -import java.util.concurrent.CancellationException; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; @@ -830,7 +825,7 @@ public static CApiContext ensureCapiWasLoaded(Node node, PythonContext context, try { CApiContext cApiContext = loadCApi(node, context, name, path, reason); assert context.getCApiState() == PythonContext.CApiState.INITIALIZING; - initializeThreadStateCurrentForAttachedThreads(node, context); + initializeThreadStateCurrentForAttachedThreads(context); CApiTransitions.initializeReferenceQueuePolling(context.nativeContext); context.runCApiHooks(); context.setCApiState(PythonContext.CApiState.INITIALIZED); // volatile write @@ -853,8 +848,7 @@ public static CApiContext ensureCapiWasLoaded(Node node, PythonContext context, return context.getCApiContext(); } - @SuppressWarnings("try") - private static void initializeThreadStateCurrentForAttachedThreads(Node node, PythonContext context) throws ApiInitException { + private static void initializeThreadStateCurrentForAttachedThreads(PythonContext context) { Thread[] threads = getOtherAliveAttachedThreads(context); if (threads.length == 0) { return; @@ -865,7 +859,7 @@ protected void perform(ThreadLocalAction.Access access) { context.initializeNativeThreadState(); } }; - waitForThreadLocalActions(node, context, submitThreadLocalActions(context, threads, action)); + submitThreadLocalActions(context, threads, action); } private static Thread[] getOtherAliveAttachedThreads(PythonContext context) { @@ -879,38 +873,9 @@ private static Thread[] getOtherAliveAttachedThreads(PythonContext context) { return threads.toArray(Thread[]::new); } - private static ArrayList> submitThreadLocalActions(PythonContext context, Thread[] threads, ThreadLocalAction action) { - ArrayList> futures = new ArrayList<>(threads.length); + private static void submitThreadLocalActions(PythonContext context, Thread[] threads, ThreadLocalAction action) { for (Thread thread : threads) { - futures.add(context.getEnv().submitThreadLocal(new Thread[]{thread}, action)); - } - return futures; - } - - @SuppressWarnings("try") - private static void waitForThreadLocalActions(Node node, PythonContext context, ArrayList> futures) throws ApiInitException { - Node waitLocation = node != null ? node : context.getLanguage().unavailableSafepointLocation; - try (GilNode.UncachedRelease ignored = GilNode.uncachedRelease()) { - for (Future future : futures) { - TruffleSafepoint.setBlockedThreadInterruptible(waitLocation, voidFuture -> { - try { - voidFuture.get(10, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } catch (TimeoutException | ExecutionException e) { - throw new RuntimeException(e); - } catch (CancellationException e) { - // Ignore threads that went away while initialization was in progress. - } - }, future); - } - } catch (RuntimeException e) { - Throwable cause = e.getCause(); - if (cause instanceof TimeoutException) { - throw new ApiInitException(toTruffleStringUncached("Timed out while initializing native thread state on an attached thread.")); - } - throw e; + context.getEnv().submitThreadLocal(new Thread[]{thread}, action); } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java index b2fcbd59ad..59951ca288 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java @@ -853,6 +853,15 @@ static Object doWithoutContext(NativeCAPISymbol symbol, Object[] args, if (capiState != PythonContext.CApiState.INITIALIZING && capiState != PythonContext.CApiState.INITIALIZED) { CompilerDirectives.transferToInterpreterAndInvalidate(); CApiContext.ensureCapiWasLoaded("call internal native GraalPy function"); + capiState = pythonContext.getCApiState(); + } + if (capiState == PythonContext.CApiState.INITIALIZED) { + PythonContext.PythonThreadState threadState = pythonContext.getThreadState(pythonContext.getLanguage(inliningTarget)); + // Native thread-state bootstrap may itself call internal C API helpers before + // 'tstate_current' has been published for this thread. + if (!threadState.isNativeThreadStateInitializationInProgress()) { + pythonContext.ensureNativeThreadStateInitialized(threadState); + } } // TODO review EnsureTruffleStringNode with GR-37896 Object callable = CApiContext.getNativeSymbol(inliningTarget, symbol); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java index 6ab36aadf8..0a00d3f176 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java @@ -804,6 +804,7 @@ private static Object invoke(VirtualFrame frame, PythonContext ctx, CApiTiming t CheckFunctionResultNode checkResultNode, CExtToJavaNode convertReturnValue, PForeignToPTypeNode fromForeign, GetThreadStateNode getThreadStateNode, ExternalFunctionInvokeNode invokeNode) { PythonThreadState threadState = getThreadStateNode.execute(inliningTarget, ctx); + ctx.ensureNativeThreadStateInitialized(threadState); Object result = invokeNode.execute(frame, inliningTarget, threadState, timing, name, callable, cArguments); result = checkResultNode.execute(threadState, name, result); if (convertReturnValue != null) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java index 0d02d86c67..9a9441e997 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java @@ -2607,6 +2607,12 @@ public void initializeNativeThreadState() { initializeNativeThreadState(getThreadState(getLanguage())); } + public void ensureNativeThreadStateInitialized(PythonThreadState pythonThreadState) { + if (getCApiState() == CApiState.INITIALIZED && !pythonThreadState.isNativeThreadStateInitialized()) { + initializeNativeThreadState(pythonThreadState); + } + } + @SuppressWarnings("try") public void initializeNativeThreadState(PythonThreadState pythonThreadState) { CompilerAsserts.neverPartOfCompilation(); From a680be5e2fa413125c1d9dbed9672cc72f5b10d6 Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Tue, 31 Mar 2026 19:46:07 +0200 Subject: [PATCH 09/14] Tighten native thread state fallback --- .../src/pystate.c | 10 +++++ .../cext/PythonCextPyStateBuiltins.java | 37 +++++++++++++++++++ .../objects/cext/capi/CApiContext.java | 8 +--- .../builtins/objects/cext/capi/CExtNodes.java | 9 ----- .../cext/capi/ExternalFunctionNodes.java | 1 - .../graal/python/runtime/PythonContext.java | 11 +++--- 6 files changed, 53 insertions(+), 23 deletions(-) diff --git a/graalpython/com.oracle.graal.python.cext/src/pystate.c b/graalpython/com.oracle.graal.python.cext/src/pystate.c index c3beb17613..9a5c1872d1 100644 --- a/graalpython/com.oracle.graal.python.cext/src/pystate.c +++ b/graalpython/com.oracle.graal.python.cext/src/pystate.c @@ -83,6 +83,16 @@ extern "C" { static inline PyThreadState * _get_thread_state() { PyThreadState *ts = tstate_current; + if (UNLIKELY(ts == NULL)) { + /* + * Very unlikely fallback: this can happen if another thread initializes the C API while + * the current thread is attached to Python but blocked and therefore misses eager + * initialization of its native 'tstate_current' TLS slot. + */ + ts = GraalPyPrivate_ThreadState_Get(&tstate_current); + assert(ts != NULL); + tstate_current = ts; + } assert(ts != NULL); return ts; } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextPyStateBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextPyStateBuiltins.java index b5b897bcc5..872840159d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextPyStateBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextPyStateBuiltins.java @@ -43,6 +43,7 @@ import static com.oracle.graal.python.builtins.modules.cext.PythonCextBuiltins.CApiCallPath.Direct; import static com.oracle.graal.python.builtins.modules.cext.PythonCextBuiltins.CApiCallPath.Ignored; import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.Int; +import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.Pointer; import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PyFrameObjectTransfer; import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PyObject; import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PyObjectBorrowed; @@ -112,6 +113,42 @@ static Object restore( } } + /** + * Very unlikely fallback for threads that were already attached when another thread initialized + * the C API, but were blocked at that time and therefore could not process the thread-local + * action that eagerly initializes their native 'tstate_current' TLS slot. + */ + @CApiBuiltin(ret = PyThreadState, args = {Pointer}, acquireGil = false, call = Ignored) + abstract static class GraalPyPrivate_ThreadState_Get extends CApiUnaryBuiltinNode { + private static final TruffleLogger LOGGER = CApiContext.getLogger(GraalPyPrivate_ThreadState_Get.class); + + @Specialization + @TruffleBoundary + static Object get(Object tstateCurrentPtr) { + PythonContext context = PythonContext.get(null); + PythonThreadState threadState = context.getThreadState(context.getLanguage()); + + /* + * The C caller may have observed 'tstate_current == NULL' before entering this upcall. + * While entering this builtin, the same thread may process a queued thread-local action + * from C API initialization and initialize its native thread state eagerly. So the + * fallback decision made in C can be stale by the time we get here. + */ + if (threadState.isNativeThreadStateInitialized()) { + LOGGER.fine(() -> String.format("Lazy initialization attempt of native thread state for thread %s aborted. Was initialized in the meantime.", Thread.currentThread())); + Object nativeThreadState = PThreadState.getNativeThreadState(threadState); + assert nativeThreadState != null; + return nativeThreadState; + } + + LOGGER.fine(() -> "Lazy (fallback) initialization of native thread state for thread " + Thread.currentThread()); + assert PThreadState.getNativeThreadState(threadState) == null; + Object nativeThreadState = PThreadState.getOrCreateNativeThreadState(threadState); + threadState.setNativeThreadLocalVarPointer(tstateCurrentPtr); + return nativeThreadState; + } + } + @CApiBuiltin(ret = Void, args = {}, call = Ignored) abstract static class GraalPyPrivate_BeforeThreadDetach extends CApiNullaryBuiltinNode { @Specialization diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java index 2274b85b21..22ae518194 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java @@ -859,7 +859,7 @@ protected void perform(ThreadLocalAction.Access access) { context.initializeNativeThreadState(); } }; - submitThreadLocalActions(context, threads, action); + context.getEnv().submitThreadLocal(threads, action); } private static Thread[] getOtherAliveAttachedThreads(PythonContext context) { @@ -873,12 +873,6 @@ private static Thread[] getOtherAliveAttachedThreads(PythonContext context) { return threads.toArray(Thread[]::new); } - private static void submitThreadLocalActions(PythonContext context, Thread[] threads, ThreadLocalAction action) { - for (Thread thread : threads) { - context.getEnv().submitThreadLocal(new Thread[]{thread}, action); - } - } - private static CApiContext loadCApi(Node node, PythonContext context, TruffleString name, TruffleString path, String reason) throws IOException, ImportException, ApiInitException { Env env = context.getEnv(); InteropLibrary U = InteropLibrary.getUncached(); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java index 59951ca288..b2fcbd59ad 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CExtNodes.java @@ -853,15 +853,6 @@ static Object doWithoutContext(NativeCAPISymbol symbol, Object[] args, if (capiState != PythonContext.CApiState.INITIALIZING && capiState != PythonContext.CApiState.INITIALIZED) { CompilerDirectives.transferToInterpreterAndInvalidate(); CApiContext.ensureCapiWasLoaded("call internal native GraalPy function"); - capiState = pythonContext.getCApiState(); - } - if (capiState == PythonContext.CApiState.INITIALIZED) { - PythonContext.PythonThreadState threadState = pythonContext.getThreadState(pythonContext.getLanguage(inliningTarget)); - // Native thread-state bootstrap may itself call internal C API helpers before - // 'tstate_current' has been published for this thread. - if (!threadState.isNativeThreadStateInitializationInProgress()) { - pythonContext.ensureNativeThreadStateInitialized(threadState); - } } // TODO review EnsureTruffleStringNode with GR-37896 Object callable = CApiContext.getNativeSymbol(inliningTarget, symbol); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java index 0a00d3f176..6ab36aadf8 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java @@ -804,7 +804,6 @@ private static Object invoke(VirtualFrame frame, PythonContext ctx, CApiTiming t CheckFunctionResultNode checkResultNode, CExtToJavaNode convertReturnValue, PForeignToPTypeNode fromForeign, GetThreadStateNode getThreadStateNode, ExternalFunctionInvokeNode invokeNode) { PythonThreadState threadState = getThreadStateNode.execute(inliningTarget, ctx); - ctx.ensureNativeThreadStateInitialized(threadState); Object result = invokeNode.execute(frame, inliningTarget, threadState, timing, name, callable, cArguments); result = checkResultNode.execute(threadState, name, result); if (convertReturnValue != null) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java index 9a9441e997..e07389ca40 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java @@ -593,6 +593,10 @@ public void setNativeThreadLocalVarPointer(Object ptr) { this.nativeThreadLocalVarPointer = ptr; } + public Object getNativeThreadLocalVarPointer() { + return nativeThreadLocalVarPointer; + } + public boolean isNativeThreadStateInitialized() { return nativeThreadLocalVarPointer != null; } @@ -2604,15 +2608,10 @@ public synchronized void attachThread(Thread thread, ContextThreadLocal "Initializing native thread state for thread " + Thread.currentThread()); initializeNativeThreadState(getThreadState(getLanguage())); } - public void ensureNativeThreadStateInitialized(PythonThreadState pythonThreadState) { - if (getCApiState() == CApiState.INITIALIZED && !pythonThreadState.isNativeThreadStateInitialized()) { - initializeNativeThreadState(pythonThreadState); - } - } - @SuppressWarnings("try") public void initializeNativeThreadState(PythonThreadState pythonThreadState) { CompilerAsserts.neverPartOfCompilation(); From 33d13f7c3b4faf680f72d23b06120534a6a486ce Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Wed, 1 Apr 2026 14:25:53 +0200 Subject: [PATCH 10/14] Add C API init regression test --- .../src/tests/test_capi_init.py | 141 ++++++++++++++++++ .../modules/GraalPythonModuleBuiltins.java | 11 ++ 2 files changed, 152 insertions(+) create mode 100644 graalpython/com.oracle.graal.python.test/src/tests/test_capi_init.py diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_capi_init.py b/graalpython/com.oracle.graal.python.test/src/tests/test_capi_init.py new file mode 100644 index 0000000000..6953e47fb4 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_capi_init.py @@ -0,0 +1,141 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +import subprocess +import sys +import textwrap +import unittest + + +@unittest.skipUnless(sys.implementation.name == "graalpy", "GraalPy-specific C API initialization test") +@unittest.skipIf(os.name == "nt", "uses os.pipe() blocking semantics") +class TestCApiInit(unittest.TestCase): + def run_in_subprocess(self, code): + python_args = [sys.executable, "--experimental-options", "--python.EnableDebuggingBuiltins"] + if not __graalpython__.is_native: + python_args += [f"--vm.Dpython.EnableBytecodeDSLInterpreter={str(__graalpython__.is_bytecode_dsl_interpreter).lower()}"] + proc = subprocess.run( + [*python_args, "-c", code], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if proc.returncode != 0: + self.fail( + "Subprocess failed with exit code {}\nstdout:\n{}\nstderr:\n{}".format( + proc.returncode, proc.stdout, proc.stderr + ) + ) + + def test_import_ctypes_while_other_thread_is_blocked_on_io(self): + code = textwrap.dedent( + """ + import os + import queue + import threading + + import __graalpython__ + + assert __graalpython__.get_capi_state() == "UNINITIALIZED" + + read_fd, write_fd = os.pipe() + about_to_block = threading.Event() + import_started = threading.Event() + import_done = threading.Event() + errors = queue.Queue() + + def blocked_thread(): + try: + assert __graalpython__.get_capi_state() == "UNINITIALIZED" + about_to_block.set() + data = os.read(read_fd, 1) + assert data == b"x" + assert __graalpython__.get_capi_state() == "INITIALIZED" + import ctypes + assert ctypes.sizeof(ctypes.py_object) > 0 + except BaseException as e: + errors.put(e) + + def importing_thread(): + try: + assert __graalpython__.get_capi_state() == "UNINITIALIZED" + import_started.set() + import _ctypes + import ctypes + assert __graalpython__.get_capi_state() == "INITIALIZED" + assert _ctypes.sizeof(ctypes.py_object) > 0 + except BaseException as e: + errors.put(e) + finally: + import_done.set() + + blocked = threading.Thread(target=blocked_thread, daemon=True) + importer = threading.Thread(target=importing_thread, daemon=True) + try: + blocked.start() + assert about_to_block.wait(10), "blocked thread did not start" + assert __graalpython__.get_capi_state() == "UNINITIALIZED" + + importer.start() + assert import_started.wait(10), "importing thread did not start" + assert import_done.wait(20), "C API initialization did not finish" + assert __graalpython__.get_capi_state() == "INITIALIZED" + finally: + try: + os.write(write_fd, b"x") + except OSError: + pass + blocked.join(20) + importer.join(20) + os.close(read_fd) + os.close(write_fd) + + assert not blocked.is_alive(), "blocked thread did not finish" + assert not importer.is_alive(), "importing thread did not finish" + + if not errors.empty(): + raise errors.get() + """ + ) + self.run_in_subprocess(code) + + +if __name__ == "__main__": + unittest.main() diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java index 1b667a4ec4..b38463e470 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java @@ -292,6 +292,7 @@ public void postInitialize(Python3Core core) { if (!context.getOption(PythonOptions.EnableDebuggingBuiltins)) { mod.setAttribute(tsLiteral("dump_truffle_ast"), PNone.NO_VALUE); mod.setAttribute(tsLiteral("tdebug"), PNone.NO_VALUE); + mod.setAttribute(tsLiteral("get_capi_state"), PNone.NO_VALUE); mod.setAttribute(tsLiteral("set_storage_strategy"), PNone.NO_VALUE); mod.setAttribute(tsLiteral("get_storage_strategy"), PNone.NO_VALUE); mod.setAttribute(tsLiteral("storage_to_native"), PNone.NO_VALUE); @@ -1194,6 +1195,16 @@ Object doit() { } } + @Builtin(name = "get_capi_state", minNumOfPositionalArgs = 0) + @GenerateNodeFactory + abstract static class GetCApiStateNode extends PythonBuiltinNode { + @Specialization + @TruffleBoundary + Object doit() { + return toTruffleStringUncached(getContext().getCApiState().name()); + } + } + @Builtin(name = "is_native_object", minNumOfPositionalArgs = 1) @GenerateNodeFactory abstract static class IsNativeObject extends PythonUnaryBuiltinNode { From 8ae8936793a8f544b3a2a79d3082917f28ed5c8d Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Wed, 1 Apr 2026 14:45:21 +0200 Subject: [PATCH 11/14] Add comment about restrictions --- .../builtins/objects/cext/capi/PThreadState.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/PThreadState.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/PThreadState.java index e7992678e7..a3a0cb8613 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/PThreadState.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/PThreadState.java @@ -136,6 +136,16 @@ public static PDict getOrCreateThreadStateDict(PythonContext context, PythonThre return threadStateDict; } + /** + * This method runs on a critical bootstrap path when creating the native thread state. It may + * execute while the C API state is still INITIALIZING and before the current thread has + * installed its native 'tstate_current' TLS slot. So, this code must stay very restricted: only + * use bootstrap-safe allocation and raw struct writes here. + * + * In particular, do not introduce conversions such as PythonToNative(NewRef)Node or any other + * code paths that may poll the native reference queue, materialize additional native wrappers, + * or otherwise assume that the native thread state is already fully initialized. + */ @TruffleBoundary private static long allocateCLayout() { long ptr = CStructAccess.AllocateNode.allocUncachedPointer(CStructs.PyThreadState.size()); From 2d1881b035e70e48f241d8b98d10b9c0e82ff242 Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Tue, 7 Apr 2026 09:12:58 +0200 Subject: [PATCH 12/14] Revert "Add C API init regression test" This reverts commit 33d13f7c3b4faf680f72d23b06120534a6a486ce. --- .../src/tests/test_capi_init.py | 141 ------------------ .../modules/GraalPythonModuleBuiltins.java | 11 -- 2 files changed, 152 deletions(-) delete mode 100644 graalpython/com.oracle.graal.python.test/src/tests/test_capi_init.py diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_capi_init.py b/graalpython/com.oracle.graal.python.test/src/tests/test_capi_init.py deleted file mode 100644 index 6953e47fb4..0000000000 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_capi_init.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. -# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. -# -# The Universal Permissive License (UPL), Version 1.0 -# -# Subject to the condition set forth below, permission is hereby granted to any -# person obtaining a copy of this software, associated documentation and/or -# data (collectively the "Software"), free of charge and under any and all -# copyright rights in the Software, and any and all patent rights owned or -# freely licensable by each licensor hereunder covering either (i) the -# unmodified Software as contributed to or provided by such licensor, or (ii) -# the Larger Works (as defined below), to deal in both -# -# (a) the Software, and -# -# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if -# one is included with the Software each a "Larger Work" to which the Software -# is contributed by such licensors), -# -# without restriction, including without limitation the rights to copy, create -# derivative works of, display, perform, and distribute the Software and make, -# use, sell, offer for sale, import, export, have made, and have sold the -# Software and the Larger Work(s), and to sublicense the foregoing rights on -# either these or other terms. -# -# This license is subject to the following condition: -# -# The above copyright notice and either this complete permission notice or at a -# minimum a reference to the UPL must be included in all copies or substantial -# portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import os -import subprocess -import sys -import textwrap -import unittest - - -@unittest.skipUnless(sys.implementation.name == "graalpy", "GraalPy-specific C API initialization test") -@unittest.skipIf(os.name == "nt", "uses os.pipe() blocking semantics") -class TestCApiInit(unittest.TestCase): - def run_in_subprocess(self, code): - python_args = [sys.executable, "--experimental-options", "--python.EnableDebuggingBuiltins"] - if not __graalpython__.is_native: - python_args += [f"--vm.Dpython.EnableBytecodeDSLInterpreter={str(__graalpython__.is_bytecode_dsl_interpreter).lower()}"] - proc = subprocess.run( - [*python_args, "-c", code], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - if proc.returncode != 0: - self.fail( - "Subprocess failed with exit code {}\nstdout:\n{}\nstderr:\n{}".format( - proc.returncode, proc.stdout, proc.stderr - ) - ) - - def test_import_ctypes_while_other_thread_is_blocked_on_io(self): - code = textwrap.dedent( - """ - import os - import queue - import threading - - import __graalpython__ - - assert __graalpython__.get_capi_state() == "UNINITIALIZED" - - read_fd, write_fd = os.pipe() - about_to_block = threading.Event() - import_started = threading.Event() - import_done = threading.Event() - errors = queue.Queue() - - def blocked_thread(): - try: - assert __graalpython__.get_capi_state() == "UNINITIALIZED" - about_to_block.set() - data = os.read(read_fd, 1) - assert data == b"x" - assert __graalpython__.get_capi_state() == "INITIALIZED" - import ctypes - assert ctypes.sizeof(ctypes.py_object) > 0 - except BaseException as e: - errors.put(e) - - def importing_thread(): - try: - assert __graalpython__.get_capi_state() == "UNINITIALIZED" - import_started.set() - import _ctypes - import ctypes - assert __graalpython__.get_capi_state() == "INITIALIZED" - assert _ctypes.sizeof(ctypes.py_object) > 0 - except BaseException as e: - errors.put(e) - finally: - import_done.set() - - blocked = threading.Thread(target=blocked_thread, daemon=True) - importer = threading.Thread(target=importing_thread, daemon=True) - try: - blocked.start() - assert about_to_block.wait(10), "blocked thread did not start" - assert __graalpython__.get_capi_state() == "UNINITIALIZED" - - importer.start() - assert import_started.wait(10), "importing thread did not start" - assert import_done.wait(20), "C API initialization did not finish" - assert __graalpython__.get_capi_state() == "INITIALIZED" - finally: - try: - os.write(write_fd, b"x") - except OSError: - pass - blocked.join(20) - importer.join(20) - os.close(read_fd) - os.close(write_fd) - - assert not blocked.is_alive(), "blocked thread did not finish" - assert not importer.is_alive(), "importing thread did not finish" - - if not errors.empty(): - raise errors.get() - """ - ) - self.run_in_subprocess(code) - - -if __name__ == "__main__": - unittest.main() diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java index b38463e470..1b667a4ec4 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java @@ -292,7 +292,6 @@ public void postInitialize(Python3Core core) { if (!context.getOption(PythonOptions.EnableDebuggingBuiltins)) { mod.setAttribute(tsLiteral("dump_truffle_ast"), PNone.NO_VALUE); mod.setAttribute(tsLiteral("tdebug"), PNone.NO_VALUE); - mod.setAttribute(tsLiteral("get_capi_state"), PNone.NO_VALUE); mod.setAttribute(tsLiteral("set_storage_strategy"), PNone.NO_VALUE); mod.setAttribute(tsLiteral("get_storage_strategy"), PNone.NO_VALUE); mod.setAttribute(tsLiteral("storage_to_native"), PNone.NO_VALUE); @@ -1195,16 +1194,6 @@ Object doit() { } } - @Builtin(name = "get_capi_state", minNumOfPositionalArgs = 0) - @GenerateNodeFactory - abstract static class GetCApiStateNode extends PythonBuiltinNode { - @Specialization - @TruffleBoundary - Object doit() { - return toTruffleStringUncached(getContext().getCApiState().name()); - } - } - @Builtin(name = "is_native_object", minNumOfPositionalArgs = 1) @GenerateNodeFactory abstract static class IsNativeObject extends PythonUnaryBuiltinNode { From d177a459f3144e96c67cbc3c1af93d7348ed39dd Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Tue, 7 Apr 2026 11:04:55 +0200 Subject: [PATCH 13/14] Fix datetime %Z parsing --- .../src/tests/test_datetime.py | 24 +++++++++++++++- .../modules/datetime/DateTimeBuiltins.java | 28 ++++++++++--------- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_datetime.py b/graalpython/com.oracle.graal.python.test/src/tests/test_datetime.py index 68835c0ea2..03d18c7258 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_datetime.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_datetime.py @@ -38,6 +38,11 @@ # SOFTWARE. import datetime +import os +import subprocess +import sys +import textwrap +import time import unittest class DateTest(unittest.TestCase): @@ -542,12 +547,29 @@ def test_strptime(self): actual = datetime.datetime.strptime("+00:00 GMT", "%z %Z") self.assertEqual(actual.tzinfo.tzname(None), "GMT") - import time timezone_name = time.localtime().tm_zone self.assertIsNotNone(timezone_name) actual = datetime.datetime.strptime(f"+00:00 {timezone_name}", "%z %Z") self.assertEqual(actual.tzinfo.tzname(None), timezone_name) + if hasattr(time, "tzset") and sys.executable: + proc = subprocess.run( + [sys.executable, "-c", textwrap.dedent("""\ + import datetime + import time + + time.tzset() + timezone_name = time.localtime().tm_zone + actual = datetime.datetime.strptime(f"+00:00 {timezone_name}", "%z %Z") + assert actual.tzinfo.tzname(None) == timezone_name + """)], + env={**os.environ, "TZ": "Etc/GMT-1"}, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + # time zone name without utc offset is ignored actual = datetime.datetime.strptime("UTC", "%Z") self.assertIsNone(actual.tzinfo) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/datetime/DateTimeBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/datetime/DateTimeBuiltins.java index 2e9ba79e03..a801017003 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/datetime/DateTimeBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/datetime/DateTimeBuiltins.java @@ -2310,22 +2310,14 @@ private static Object parse(String string, String format, PythonContext context, TimeZone timeZone = TimeModuleBuiltins.getGlobalTimeZone(context); String zoneName = timeZone.getDisplayName(false, TimeZone.SHORT); String zoneNameDaylightSaving = timeZone.getDisplayName(true, TimeZone.SHORT); + String matchedZoneName = matchTimeZoneName(string, i, zoneName, zoneNameDaylightSaving, "UTC", "GMT"); - if (string.startsWith("UTC", i)) { - builder.setTimeZoneName("UTC"); - i += 3; - } else if (string.startsWith("GMT", i)) { - builder.setTimeZoneName("GMT"); - i += 3; - } else if (string.startsWith(zoneName, i)) { - builder.setTimeZoneName(zoneName); - i += zoneName.length(); - } else if (string.startsWith(zoneNameDaylightSaving, i)) { - builder.setTimeZoneName(zoneNameDaylightSaving); - i += zoneNameDaylightSaving.length(); - } else { + if (matchedZoneName == null) { throw PRaiseNode.raiseStatic(inliningTarget, ValueError, ErrorMessages.TIME_DATA_S_DOES_NOT_MATCH_FORMAT_S, string, format); } + + builder.setTimeZoneName(matchedZoneName); + i += matchedZoneName.length(); } case 'j' -> { var pos = new ParsePosition(i); @@ -2487,6 +2479,16 @@ private static Integer parseDigits(String source, int from, int digitsCount) { return result; } + private static String matchTimeZoneName(String string, int from, String... candidates) { + String matched = null; + for (String candidate : candidates) { + if (candidate != null && string.startsWith(candidate, from) && (matched == null || candidate.length() > matched.length())) { + matched = candidate; + } + } + return matched; + } + @TruffleBoundary private static Integer parseDigitsUpTo(String source, ParsePosition from, int maxDigitsCount) { int result = 0; From 15f82a23cde29a733942e135367ada055fd6d1f9 Mon Sep 17 00:00:00 2001 From: Florian Angerer Date: Tue, 7 Apr 2026 16:46:33 +0200 Subject: [PATCH 14/14] Fix gethostbyname_ex error type --- .../src/tests/test_socket.py | 9 ++++++++- .../python/builtins/modules/SocketModuleBuiltins.java | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_socket.py b/graalpython/com.oracle.graal.python.test/src/tests/test_socket.py index b9a10bbb8b..c49686d2c3 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_socket.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_socket.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -59,6 +59,13 @@ def test_inet_aton_errs(self): self.assertRaises(OSError, lambda : socket.inet_aton('255.255.256.1')) self.assertRaises(TypeError, lambda : socket.inet_aton(255)) + +class TestHostLookupErrors(unittest.TestCase): + def test_gethostbyname_ex_invalid_host_raises_gaierror(self): + with self.assertRaises(socket.gaierror): + socket.gethostbyname_ex("nonexistent.invalid") + + def test_get_name_info(): import socket try : diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SocketModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SocketModuleBuiltins.java index 16980ff22e..1001fccd42 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SocketModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SocketModuleBuiltins.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -385,7 +385,7 @@ static Object get(VirtualFrame frame, Object nameObj, addrInfoCursorLib.release(cursor); } } catch (GetAddrInfoException e) { - throw constructAndRaiseNode.get(inliningTarget).executeWithArgsOnly(frame, SocketHError, new Object[]{e.getMessageAsTruffleString()}); + throw constructAndRaiseNode.get(inliningTarget).executeWithArgsOnly(frame, SocketGAIError, new Object[]{e.getErrorCode(), e.getMessageAsTruffleString()}); } catch (PosixException e) { throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e); }