diff --git a/ChangeLog.md b/ChangeLog.md index b645921..246f156 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,12 +1,18 @@ 0.3.0.0 [XXXX.XX.XX] -------------------- -* Support for builds with `python3-config` -* Support for async -* `inline_python` module is now available. -* Now haskell exception from haskell callback in converted to - `inline_python.HaskellError` and is rethrown if it's not catched by python. -* Memory leak is fixed. Python exception object were never freed when exception - propagated to haskell side. +* Support for asynchronous execution added in module `Python.Inline.Async`. + It adds API copied from `async` and interruptible python computations. +* `runPyInMain` could be reliably interrupted by asynchronous exceptions. +* Package now uses `Custom` build type. It now supports configuring python using + `python3-config` instead of `pkg-config` when `-fpython3-config` manual cabal + flag is set. Default behavior is unchanged. +* Python module `inline_python` is now available. It contains exception types + used by library: `AsyncCancelled` and `HaskellError` which wraps haskell + exception from callback. +* Haskell exception raised in haskell callback will be rethrown if not caught by + python instead of being converted to `PyError`. +* Memory leak in exception handling is fixed. Python exception object were never + freed when exception propagated to haskell side. 0.2.1.0 [2026.01.13] ---------------- diff --git a/src/Python/Inline.hs b/src/Python/Inline.hs index c837ceb..ff2989c 100644 --- a/src/Python/Inline.hs +++ b/src/Python/Inline.hs @@ -177,11 +177,3 @@ import Python.Internal.Eval -- redone and python does not give much guarantee about what is happening here. -- Use it with caution. We recommend using @importlib.reload@ only during -- development and not in production. --- --- 5. __Asynchronous exceptions__ --- --- The code run by 'runPy' is not interruptible by Haskell asynchronous --- exceptions and may block indefinitely. If your code call any Haskell --- function as callback, they won't receive asynchronous exception either. See --- https://github.com/Shimuuar/inline-python/issues/48 for details and --- workarounds. diff --git a/src/Python/Inline/Async.hs b/src/Python/Inline/Async.hs index 759c8b1..d55cbb5 100644 --- a/src/Python/Inline/Async.hs +++ b/src/Python/Inline/Async.hs @@ -1,13 +1,13 @@ -- | --- Asynchronous computation using python. Normally library tries to --- execute python code in the same thread. Moreover it use global lock --- in addition to GIL in order to avoid blocking capability on GIL. --- This module provide API for working with concurrent python. --- Its API is heavily modelled after @async@ package. +-- Asynchronous computation using python. Its API is modelled after +-- @async@ package. It evaluates python on separate OS thread so it's +-- more heavyweight than 'Python.Inline.runPy'. But it's possible to +-- properly interrupt running computation with 'cancelPy' or to use +-- 'withPyAsync' to ensure that async computation properly terminated. -- --- Note it's very experimental and not well tested. Also mixing --- concurrency primitives from two languages makes difficult task of --- concurrent programming even more complicated. +-- Since arbitrary IO is available either in @Py@ via @liftIO@ or in +-- haskell callbacks from python code. It's possible to use haskell +-- concurrency primitives to communicate with python thread. module Python.Inline.Async ( PyAsync , PyAsyncCancelled(..) @@ -20,4 +20,3 @@ module Python.Inline.Async ) where import Python.Internal.Eval - diff --git a/src/Python/Inline/QQ.hs b/src/Python/Inline/QQ.hs index adce6bf..c93a0d7 100644 --- a/src/Python/Inline/QQ.hs +++ b/src/Python/Inline/QQ.hs @@ -30,8 +30,13 @@ -- > do_that() -- > |] -- --- If control over python's global and local variables is --- required. APIs from "Python.Inline.Eval" should be used instead. +-- +-- == Variable scope +-- +-- Python has two copes: global and local variables. Both are simply +-- @dict[str,Any]@. Quasiquoters use different dictionaries for +-- globals and locals. If tighter control over variables scope is +-- required APIs from "Python.Inline.Eval" should be used instead. module Python.Inline.QQ ( pymain , py_ @@ -46,9 +51,10 @@ import Python.Internal.EvalQQ import Python.Internal.Eval --- | Evaluate sequence of python statements. It works in the same way --- as python's @exec@. All module imports and all variables defined --- in this quasiquote will be visible to later quotes. +-- | Evaluate sequence of python statements. It uses python's @exec@. +-- Both global and local state for this quasiquoter are variables of +-- @\__main__@ module. Any variables including imported modules will +-- remain visible to later quasiquotes. -- -- It creates value of type @Py ()@ pymain :: QuasiQuoter @@ -59,9 +65,11 @@ pymain = QuasiQuoter , quoteDec = error "quoteDec" } --- | Evaluate sequence of python statements. All module imports and --- all variables defined in this quasiquote will be discarded and --- won't be visible in later quotes. +-- | Evaluate sequence of python statements. Global variables for this +-- quasiquoter are one defined in @\__main__@ module and locals use +-- newly allocated dictionary. It will be discarded after execution +-- so variables defined in this quasiquote are visible only inside +-- of it. -- -- It creates value of type @Py ()@ py_ :: QuasiQuoter @@ -73,7 +81,8 @@ py_ = QuasiQuoter } -- | Evaluate single python expression. It only accepts single --- expressions same as python's @eval@. +-- expressions same as python's @eval@. Its globals are variables in +-- @\__main__@ module and locals are new dictionary same as in @py_@. -- -- This quote creates object of type @Py PyObject@ pye :: QuasiQuoter @@ -85,9 +94,12 @@ pye = QuasiQuoter } -- | Another quasiquoter which works around that sequence of python --- statements doesn't have any value associated with it. Content of +-- statements doesn't have any value associated with it. Content of -- quasiquote is function body. So to get value out of it one must --- call return +-- call return. Its globals are variables in @\__main__@ module and +-- locals are new dictionary same as in @py_@. +-- +-- This quote creates object of type @Py PyObject@ pyf :: QuasiQuoter pyf = QuasiQuoter { quoteExp = \txt -> [| evaluatorPyf $(expQQ Fun txt) |] diff --git a/src/Python/Internal/Eval.hs b/src/Python/Internal/Eval.hs index 6fc3b77..35b6030 100644 --- a/src/Python/Internal/Eval.hs +++ b/src/Python/Internal/Eval.hs @@ -524,9 +524,15 @@ data EvalReq -- ^ Dummy request. Do nothing --- | Execute python action. It will take and hold global lock while --- code is executed. Python exceptions raised during execution are --- converted to haskell exception 'PyError'. +-- | Execute python action. This is simplest executor with lowest +-- overhead but it comes with several caveats. When thread executes +-- python code it could not be interrupted since it's in foreign +-- call. Use 'runPyAsync' if you need ability to interrupt. Also +-- python uses GIL so only one thread can evaluate python code at +-- time. +-- +-- Python exceptions raised during execution are converted to +-- haskell exception 'PyError'. runPy :: Py a -> IO a -- See NOTE: [Python and threading] runPy py @@ -538,10 +544,15 @@ runPy py go = ensurePyLock $ mask_ $ unsafeRunPy (ensureGIL py) --- | Same as 'runPy' but will make sure that code is run in python's --- main thread. It's thread in which python's interpreter was --- initialized. Some python's libraries may need that. It has higher --- call overhead compared to 'runPy'. +-- | This function executes python code on python's main thread. It's +-- OS thread in which interpreter was initialized and it has some +-- special status in python. Some libraries could only work when +-- called from main thread. It has higher call overhead compared to +-- 'runPy' and only one haskell thread could be executing something +-- on main thread at time. +-- +-- When executing on threaded runtime this function could be +-- interrupted by asynchronous exceptions. runPyInMain :: Py a -> IO a -- See NOTE: [Python and threading, Main thread] runPyInMain py @@ -625,7 +636,12 @@ unsafeRunPy (Py io) = io -- thread is dead already. --- | Exception thrown to a thread doing async python computation. +-- | Exception thrown to a thread doing async python computation. On +-- python side it corresponds to +-- @inline_python.AsyncCancelled@. Latter is automatically converted +-- to @PyAsyncCancelled@. +-- +-- @since 0.3 data PyAsyncCancelled = PyAsyncCancelled deriving (Show, Eq) @@ -633,7 +649,9 @@ instance Exception PyAsyncCancelled -- | Handle to asynchronous python computation spawned by -- 'runPyAsync'. It's performed on separate OS thread. Use --- 'wait'\/'waitCatch' to obtain computation result. +-- 'waitPy'\/'waitPyCatch' to obtain computation result. +-- +-- @since 0.3 data PyAsync a = PyAsync { asyncTID :: !ThreadId -- Thread ID , asyncTidStack :: !(TVar [ThreadId]) -- Stack of callback thread ID @@ -644,15 +662,21 @@ data PyAsync a = PyAsync -- | Wait for result of asynchronous computation. If it threw an -- exception it will be rethrown by @wait@. +-- +-- @since 0.3 waitPy :: PyAsync a -> STM a waitPy a = either throwSTM pure =<< a.asyncWait -- | Wait for result of asynchronous computation. Exception thrown by -- it will be returned as @Left@. +-- +-- @since 0.3 waitPyCatch :: PyAsync a -> STM (Either SomeException a) waitPyCatch = (.asyncWait) --- | Create new OS thread and execute python code on it. +-- | Execute python computation on dedicated OS thread. +-- +-- @since 0.3 runPyAsync :: Py a -> IO (PyAsync a) runPyAsync py = do ensureInit @@ -700,15 +724,16 @@ withAsyncInitTLS stack = bracket ini fini . const --- | Cancel execution of asynchronous computation. Most likely thread --- will be executing some python so first it attempts to raise async --- exception in python code. Then it throws 'PyAsyncCancelled' in case --- it executes haskell code. This means thread could be terminate --- either with 'PyError' or 'PyAsyncCancelled'. +-- | Cancel execution of asynchronous computation. It throws +-- 'PyAsyncCancelled' to haskell threads including any haskell +-- callbacks from python. Python will interrupted by asynchronously +-- raising @inline_python.AsyncCancelled@. -- -- Note that python code generally is not written under assumption -- that it could be smitten with exception at an absolutely any --- moment. +-- moment. It could cause problems. +-- +-- @since 0.3 cancelPy :: PyAsync a -> IO () cancelPy PyAsync{asyncTID=tid, asyncTidStack, asyncPyTID, asyncAlive} = do -- See NOTE: [Py Async], [Interrupting python] @@ -744,11 +769,15 @@ cancelPy PyAsync{asyncTID=tid, asyncTidStack, asyncPyTID, asyncAlive} = do killThread tid_kill_cb -- | Variant of 'cancel' which isn't interruptible. +-- +-- @since 0.3 uninterruptibleCancelPy :: PyAsync a -> IO () uninterruptibleCancelPy = uninterruptibleMask_ . cancelPy -- | Create new OS thread and execute python code on it. Will use -- 'uninterruptibleCancel' after callback finishes execution. +-- +-- @since 0.3 withPyAsync :: Py a -> (PyAsync a -> IO b) -> IO b withPyAsync py = bracket (runPyAsync py) uninterruptibleCancelPy