diff --git a/bindings/pyroot/pythonizations/python/ROOT/_jupyroot/helpers/handlers.py b/bindings/pyroot/pythonizations/python/ROOT/_jupyroot/helpers/handlers.py index 3280c6ca32865..2b20140462929 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_jupyroot/helpers/handlers.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_jupyroot/helpers/handlers.py @@ -12,15 +12,97 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. # ################################################################################ +import codecs import contextvars +import ctypes +import os import queue import sys +import threading +import time from threading import Thread from time import sleep as timeSleep -import ROOT.libROOTPythonizations as _lib from ROOT._jupyroot import helpers +# C stdio, whose buffers have to be flushed for output to reach the pipes +if sys.platform == "win32": + _crt = ctypes.cdll.ucrtbase +else: + _crt = ctypes.CDLL(None) +_crt.fflush.argtypes = [ctypes.c_void_p] + +# How long to wait at the end of a capture for the rest of the output. It only +# takes this long when a process started during the capture still holds the +# pipe, so that its end is never reached. +_END_CAPTURE_TIMEOUT = 1.0 + + +class _FileDescriptorCapture(object): + """Redirects a file descriptor into a pipe and collects what is written + to it, also by C and C++ code. + + A thread reads the pipe for as long as the capture runs, so that writers + never wait on a full pipe. A writer of C stdio does so holding the lock of + the stream, and flushing the stream to get its output would then wait for + that lock forever. + """ + + def __init__(self, fd): + self._fd = fd + self._saved_fd = None + self._write_fd = None + self._reader = None + self._lock = threading.Lock() + self._text = "" + + @property + def text(self): + with self._lock: + return self._text + + def clear(self): + with self._lock: + self._text = "" + + def start(self): + self._saved_fd = os.dup(self._fd) + read_fd, self._write_fd = os.pipe() + os.dup2(self._write_fd, self._fd) + self._reader = threading.Thread(target=self._read, args=(read_fd,), name="JupyROOT capture", daemon=True) + self._reader.start() + + def _read(self, read_fd): + # Decode incrementally: a read can end in the middle of a character + decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") + try: + while True: + chunk = os.read(read_fd, 65536) + text = decoder.decode(chunk, final=not chunk) + with self._lock: + # After a timeout in stop(), the output belongs to no + # capture anymore + if self._reader is threading.current_thread(): + self._text += text + if not chunk: + break + finally: + os.close(read_fd) + + def stop(self): + """Restore the file descriptor; call wait() for the rest of the output""" + os.dup2(self._saved_fd, self._fd) + os.close(self._saved_fd) + os.close(self._write_fd) + self._saved_fd = self._write_fd = None + + def wait(self, deadline): + """Wait until the output written before stop() is collected, or until + the deadline (from time.monotonic()) has passed""" + self._reader.join(max(0.0, deadline - time.monotonic())) + with self._lock: + self._reader = None + class IOHandler(object): r"""Class used to capture output from C/C++ libraries. @@ -36,28 +118,48 @@ class IOHandler(object): """ def __init__(self): - _lib.JupyROOTExecutorHandler_Ctor() + import ROOT + + # Fixes for ROOT-7999 + ROOT.SetErrorHandler(ROOT.DefaultErrorHandler) + + self._stdout = _FileDescriptorCapture(1) + self._stderr = _FileDescriptorCapture(2) + self._capturing = False def __del__(self): - _lib.JupyROOTExecutorHandler_Dtor() + self.EndCapture() def Clear(self): - _lib.JupyROOTExecutorHandler_Clear() + self._stdout.clear() + self._stderr.clear() def Poll(self): - _lib.JupyROOTExecutorHandler_Poll() + if self._capturing: + # The reader threads collect what reaches the pipes + _crt.fflush(None) def InitCapture(self): - _lib.JupyROOTExecutorHandler_InitCapture() + if not self._capturing: + self._stdout.start() + self._stderr.start() + self._capturing = True def EndCapture(self): - _lib.JupyROOTExecutorHandler_EndCapture() + if self._capturing: + _crt.fflush(None) + self._stdout.stop() + self._stderr.stop() + deadline = time.monotonic() + _END_CAPTURE_TIMEOUT + self._stdout.wait(deadline) + self._stderr.wait(deadline) + self._capturing = False def GetStdout(self): - return _lib.JupyROOTExecutorHandler_GetStdout() + return self._stdout.text def GetStderr(self): - return _lib.JupyROOTExecutorHandler_GetStderr() + return self._stderr.text def GetStreamsDicts(self): out = self.GetStdout() @@ -145,6 +247,46 @@ def HasFinished(self): return not self.poller.is_running +def _report_exception(location, e): + # Report exceptions that escape the interpreted code like TRint does at + # the ROOT prompt (see TRint::HandleTermInput), instead of swallowing them + # silently (ROOT-10589) + import ROOT + + if isinstance(e, ROOT.std.exception): + message = "{} caught: {}".format(type(e).__cpp_name__, e.what()) + ROOT.Error(location, message.replace("%", "%%")) + else: + ROOT.Error(location, "Exception caught!") + + +def _jupyroot_execute(code): + import ROOT + + status = False + try: + err = ctypes.c_uint(ROOT.TInterpreter.kNoError) + if ROOT.gInterpreter.ProcessLine(code, err): + status = True + if err.value == ROOT.TInterpreter.kProcessing: + ROOT.gInterpreter.ProcessLine(".@") + ROOT.gInterpreter.ProcessLine('cerr << "Unbalanced braces. This cell was not processed." << endl;') + except Exception as e: + _report_exception("JupyROOTExecutor", e) + return int(status) + + +def _jupyroot_declare(code): + import ROOT + + status = False + try: + status = bool(ROOT.gInterpreter.Declare(code)) + except Exception as e: + _report_exception("JupyROOTDeclarer", e) + return int(status) + + class JupyROOTDeclarer(Runner): """Asynchrously execute declarations >>> import ROOT @@ -158,7 +300,7 @@ class JupyROOTDeclarer(Runner): """ def __init__(self, poller): - super(JupyROOTDeclarer, self).__init__(_lib.JupyROOTDeclarer, poller) + super(JupyROOTDeclarer, self).__init__(_jupyroot_declare, poller) class JupyROOTExecutor(Runner): @@ -172,7 +314,7 @@ class JupyROOTExecutor(Runner): """ def __init__(self, poller): - super(JupyROOTExecutor, self).__init__(_lib.JupyROOTExecutor, poller) + super(JupyROOTExecutor, self).__init__(_jupyroot_execute, poller) def display_drawables(displayFunction): diff --git a/bindings/pyroot/pythonizations/src/IOHandler.cxx b/bindings/pyroot/pythonizations/src/IOHandler.cxx deleted file mode 100644 index 8949538f06043..0000000000000 --- a/bindings/pyroot/pythonizations/src/IOHandler.cxx +++ /dev/null @@ -1,287 +0,0 @@ -// Author: Danilo Piparo, Omar Zapata, Enric Tejedor 16/12/2015 - -/************************************************************************* - * Copyright (C) 1995-2020, Rene Brun and Fons Rademakers. * - * All rights reserved. * - * * - * For the licensing terms see $ROOTSYS/LICENSE. * - * For the list of contributors see $ROOTSYS/README/CREDITS. * - *************************************************************************/ - -#include - -#include -#ifdef _MSC_VER // Visual Studio -#include -#include -#pragma comment(lib, "Ws2_32.lib") -#define pipe(fds) _pipe(fds, 1048575, _O_BINARY) -#define read _read -#define dup _dup -#define dup2 _dup2 -#define STDIN_FILENO 0 -#define STDOUT_FILENO 1 -#define STDERR_FILENO 2 -#else -#include -#endif -#include -#include -#include -#include "TClassEdit.h" -#include "TError.h" -#include "TInterpreter.h" - -////////////////////////// -// MODULE FUNCTIONALITY // -////////////////////////// - -class JupyROOTExecutorHandler { -private: - bool fCapturing = false; - std::string fStdoutpipe; - std::string fStderrpipe; - int fStdout_pipe[2] = {0, 0}; - int fStderr_pipe[2] = {0, 0}; - int fSaved_stderr = 0; - int fSaved_stdout = 0; - -public: - JupyROOTExecutorHandler(); - void Poll(); - void InitCapture(); - void EndCapture(); - void Clear(); - std::string &GetStdout(); - std::string &GetStderr(); -}; - -#ifndef F_LINUX_SPECIFIC_BASE -#define F_LINUX_SPECIFIC_BASE 1024 -#endif -#ifndef F_SETPIPE_SZ -#define F_SETPIPE_SZ (F_LINUX_SPECIFIC_BASE + 7) -#endif - -constexpr long MAX_PIPE_SIZE = 1048575; - -JupyROOTExecutorHandler::JupyROOTExecutorHandler() {} - -static void PollImpl(FILE *stdStream, int *pipeHandle, std::string &pipeContent) -{ - fflush(stdStream); -#ifdef _MSC_VER - char buffer[60000] = ""; - struct _stat st; - _fstat(pipeHandle[0], &st); - if (st.st_size) { - _read(pipeHandle[0], buffer, 60000); - pipeContent += buffer; - } -#else - int buf_read; - char ch; - while (true) { - buf_read = read(pipeHandle[0], &ch, 1); - if (buf_read == 1) { - pipeContent += ch; - } else - break; - } -#endif -} - -void JupyROOTExecutorHandler::Poll() -{ - PollImpl(stdout, fStdout_pipe, fStdoutpipe); - PollImpl(stderr, fStderr_pipe, fStderrpipe); -} - -static void InitCaptureImpl(int &savedStdStream, int *pipeHandle, int FILENO) -{ - savedStdStream = dup(FILENO); - if (pipe(pipeHandle) != 0) { - return; - } -#ifndef _MSC_VER - long flags_stdout = fcntl(pipeHandle[0], F_GETFL); - if (flags_stdout == -1) - return; - flags_stdout |= O_NONBLOCK; - fcntl(pipeHandle[0], F_SETFL, flags_stdout); - fcntl(pipeHandle[0], F_SETPIPE_SZ, MAX_PIPE_SIZE); -#endif - dup2(pipeHandle[1], FILENO); -} - -void JupyROOTExecutorHandler::InitCapture() -{ - if (!fCapturing) { - InitCaptureImpl(fSaved_stdout, fStdout_pipe, STDOUT_FILENO); - InitCaptureImpl(fSaved_stderr, fStderr_pipe, STDERR_FILENO); - fCapturing = true; - } -} - -void JupyROOTExecutorHandler::EndCapture() -{ - if (fCapturing) { - Poll(); - dup2(fSaved_stdout, STDOUT_FILENO); - dup2(fSaved_stderr, STDERR_FILENO); - close(fSaved_stdout); - close(fSaved_stderr); - close(fStdout_pipe[0]); - close(fStdout_pipe[1]); - close(fStderr_pipe[0]); - close(fStderr_pipe[1]); - fCapturing = false; - } -} - -void JupyROOTExecutorHandler::Clear() -{ - fStdoutpipe = ""; - fStderrpipe = ""; -} - -std::string &JupyROOTExecutorHandler::GetStdout() -{ - return fStdoutpipe; -} - -std::string &JupyROOTExecutorHandler::GetStderr() -{ - return fStderrpipe; -} - -JupyROOTExecutorHandler *JupyROOTExecutorHandler_ptr = nullptr; - -// Report exceptions that escape the interpreted code like TRint does at the -// ROOT prompt (see TRint::HandleTermInput), instead of swallowing them -// silently (ROOT-10589). -static void ReportCaughtException(const char *location, const std::exception &e) -{ - int err = 0; - char *demangledType_c = TClassEdit::DemangleTypeIdName(typeid(e), err); - const char *demangledType = err ? "" : demangledType_c; - ::Error(location, "%s caught: %s", demangledType, e.what()); - free(demangledType_c); -} - -bool JupyROOTExecutorImpl(const char *code) -{ - auto status = false; - try { - auto err = TInterpreter::kNoError; - if (gInterpreter->ProcessLine(code, &err)) { - status = true; - } - - if (err == TInterpreter::kProcessing) { - gInterpreter->ProcessLine(".@"); - gInterpreter->ProcessLine("cerr << \"Unbalanced braces. This cell was not processed.\" << endl;"); - } - } catch (std::exception &e) { - ReportCaughtException("JupyROOTExecutor", e); - } catch (...) { - ::Error("JupyROOTExecutor", "Exception caught!"); - } - - return status; -} - -bool JupyROOTDeclarerImpl(const char *code) -{ - auto status = false; - try { - if (gInterpreter->Declare(code)) { - status = true; - } - } catch (std::exception &e) { - ReportCaughtException("JupyROOTDeclarer", e); - } catch (...) { - ::Error("JupyROOTDeclarer", "Exception caught!"); - } - return status; -} - -PyObject *JupyROOTExecutor(PyObject * /*self*/, PyObject *args) -{ - const char *code; - if (!PyArg_ParseTuple(args, "s", &code)) - return NULL; - - auto res = JupyROOTExecutorImpl(code); - - return PyLong_FromLong(res); -} - -PyObject *JupyROOTDeclarer(PyObject * /*self*/, PyObject *args) -{ - const char *code; - if (!PyArg_ParseTuple(args, "s", &code)) - return NULL; - - auto res = JupyROOTDeclarerImpl(code); - - return PyLong_FromLong(res); -} - -PyObject *JupyROOTExecutorHandler_Clear(PyObject * /*self*/, PyObject * /*args*/) -{ - JupyROOTExecutorHandler_ptr->Clear(); - Py_RETURN_NONE; -} - -PyObject *JupyROOTExecutorHandler_Ctor(PyObject * /*self*/, PyObject * /*args*/) -{ - if (!JupyROOTExecutorHandler_ptr) { - JupyROOTExecutorHandler_ptr = new JupyROOTExecutorHandler(); - // Fixes for ROOT-7999 - gInterpreter->ProcessLine("SetErrorHandler((ErrorHandlerFunc_t)&DefaultErrorHandler);"); - } - - Py_RETURN_NONE; -} - -PyObject *JupyROOTExecutorHandler_Poll(PyObject * /*self*/, PyObject * /*args*/) -{ - JupyROOTExecutorHandler_ptr->Poll(); - Py_RETURN_NONE; -} - -PyObject *JupyROOTExecutorHandler_EndCapture(PyObject * /*self*/, PyObject * /*args*/) -{ - JupyROOTExecutorHandler_ptr->EndCapture(); - Py_RETURN_NONE; -} - -PyObject *JupyROOTExecutorHandler_InitCapture(PyObject * /*self*/, PyObject * /*args*/) -{ - JupyROOTExecutorHandler_ptr->InitCapture(); - Py_RETURN_NONE; -} - -PyObject *JupyROOTExecutorHandler_GetStdout(PyObject * /*self*/, PyObject * /*args*/) -{ - auto out = JupyROOTExecutorHandler_ptr->GetStdout().c_str(); - return PyUnicode_FromString(out); -} - -PyObject *JupyROOTExecutorHandler_GetStderr(PyObject * /*self*/, PyObject * /*args*/) -{ - auto err = JupyROOTExecutorHandler_ptr->GetStderr().c_str(); - return PyUnicode_FromString(err); -} - -PyObject *JupyROOTExecutorHandler_Dtor(PyObject * /*self*/, PyObject * /*args*/) -{ - if (!JupyROOTExecutorHandler_ptr) - Py_RETURN_NONE; - - delete JupyROOTExecutorHandler_ptr; - JupyROOTExecutorHandler_ptr = nullptr; - - Py_RETURN_NONE; -} diff --git a/bindings/pyroot/pythonizations/src/PyROOTModule.cxx b/bindings/pyroot/pythonizations/src/PyROOTModule.cxx index 6dce34a0e970e..a6c9136cc2db2 100644 --- a/bindings/pyroot/pythonizations/src/PyROOTModule.cxx +++ b/bindings/pyroot/pythonizations/src/PyROOTModule.cxx @@ -30,8 +30,6 @@ #include #include -#include "IOHandler.cxx" - namespace PyROOT { PyObject *gRootModule = nullptr; @@ -164,24 +162,6 @@ static PyMethodDef gPyROOTMethods[] = { "Install an input hook to process GUI events"}, {"_CPPInstance__expand__", (PyCFunction)PyROOT::CPPInstanceExpand, METH_VARARGS, "Deserialize a pickled object"}, - {"JupyROOTExecutor", (PyCFunction)JupyROOTExecutor, METH_VARARGS, "Create JupyROOTExecutor"}, - {"JupyROOTDeclarer", (PyCFunction)JupyROOTDeclarer, METH_VARARGS, "Create JupyROOTDeclarer"}, - {"JupyROOTExecutorHandler_Clear", (PyCFunction)JupyROOTExecutorHandler_Clear, METH_NOARGS, - "Clear JupyROOTExecutorHandler"}, - {"JupyROOTExecutorHandler_Ctor", (PyCFunction)JupyROOTExecutorHandler_Ctor, METH_NOARGS, - "Create JupyROOTExecutorHandler"}, - {"JupyROOTExecutorHandler_Poll", (PyCFunction)JupyROOTExecutorHandler_Poll, METH_NOARGS, - "Poll JupyROOTExecutorHandler"}, - {"JupyROOTExecutorHandler_EndCapture", (PyCFunction)JupyROOTExecutorHandler_EndCapture, METH_NOARGS, - "End capture JupyROOTExecutorHandler"}, - {"JupyROOTExecutorHandler_InitCapture", (PyCFunction)JupyROOTExecutorHandler_InitCapture, METH_NOARGS, - "Init capture JupyROOTExecutorHandler"}, - {"JupyROOTExecutorHandler_GetStdout", (PyCFunction)JupyROOTExecutorHandler_GetStdout, METH_NOARGS, - "Get stdout JupyROOTExecutorHandler"}, - {"JupyROOTExecutorHandler_GetStderr", (PyCFunction)JupyROOTExecutorHandler_GetStderr, METH_NOARGS, - "Get stderr JupyROOTExecutorHandler"}, - {"JupyROOTExecutorHandler_Dtor", (PyCFunction)JupyROOTExecutorHandler_Dtor, METH_NOARGS, - "Destruct JupyROOTExecutorHandler"}, {"PyObjRefCounterAsStdAny", (PyCFunction)PyROOT::PyObjRefCounterAsStdAny, METH_VARARGS, "Wrap a reference count to any Python object in a std::any for resource management in C++"}, {NULL, NULL, 0, NULL}};