From 3744b19dc13faa3db2c1133e353c3ce7d5322d35 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 22 Jul 2026 18:20:44 +0200 Subject: [PATCH 01/91] feat: run popen Message IO on Trio host threads Move coordinator and worker framed protocol IO into dedicated Trio host threads for local popen + import bootstrap, while keeping the sync Channel/Gateway API and WorkerPool remote_exec. Adds a trio dependency; disable with EXECNET_TRIO_HOST=0. Co-authored-by: Cursor AI Co-authored-by: Cursor Grok 4.5 --- CHANGELOG.rst | 4 + doc/implnotes.rst | 43 ++- pyproject.toml | 3 + src/execnet/_trio_host.py | 527 ++++++++++++++++++++++++++++++++++++ src/execnet/_trio_worker.py | 117 ++++++++ src/execnet/gateway.py | 17 +- src/execnet/gateway_base.py | 43 ++- src/execnet/multi.py | 18 +- uv.lock | 61 +++++ 9 files changed, 818 insertions(+), 15 deletions(-) create mode 100644 src/execnet/_trio_host.py create mode 100644 src/execnet/_trio_worker.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a7af6272..eac3c8d6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,10 @@ ------------------ * `#380 `__: Add support for Python 3.13 and 3.14, and drop EOL 3.8 and 3.9. +* Trio host-thread Message IO for local ``popen`` + import bootstrap (coordinator and + worker). Adds a hard ``trio`` dependency. Disable with ``EXECNET_TRIO_HOST=0``. + Other gateway types and greenlet execmodels keep the legacy thread path. + 2.1.2 (2025-11-11) ------------------ diff --git a/doc/implnotes.rst b/doc/implnotes.rst index d13d9e87..3dae82d6 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -7,7 +7,7 @@ capable of receiving and executing code, and routing data through channels. Gateways operate on InputOutput objects offering -a write and a read(n) method. + a write and a read(n) method. Once bootstrapped a higher level protocol based on Messages is used. Messages are serialized @@ -15,18 +15,43 @@ to and from InputOutput objects. The details of this protocol are locally defined in this module. There is no need for standardizing or versioning the protocol. -After bootstrapping the BaseGateway opens a receiver thread which +Trio host-thread IO (popen / import bootstrap) +---------------------------------------------- + +For local ``popen`` gateways that use import-based bootstrap +(same installed ``execnet`` + ``trio`` on both sides), Message +protocol IO runs inside a dedicated OS thread hosting a Trio +event loop (``execnet._trio_host.TrioHost``): + +* Coordinator: ``trio.lowlevel.open_process`` plus async framed + reader/writer tasks per gateway (one host thread per ``Group``). +* Worker: ``serve_popen_trio`` adopts stdio pipe fds into Trio + streams; ``remote_exec`` still runs on the existing WorkerPool + (including ``main_thread_only`` primary-thread integration). + +Sync ``Channel`` / ``Gateway`` APIs are unchanged. Sends from +non-host threads wait until the frame is written (so abrupt +``os._exit`` cannot drop queued data). Sends from the Trio host +thread (receiver callbacks) only enqueue, to avoid deadlocking +the writer task. + +Disable with ``EXECNET_TRIO_HOST=0``. Other gateway types +(``ssh``, ``socket``, ``via``, ``python=…``, greenlet execmodels) +still use the legacy thread receiver and sync ``Popen`` path. + +Legacy thread model +------------------- + +After bootstrapping, ``BaseGateway`` opens a receiver thread which accepts encoded messages and triggers actions to interpret them. Sending of channel data items happens directly through write operations to InputOutput objects so there is no -separate thread. +separate send thread. -Code execution messages are put into an execqueue from -which they will be taken for execution. gateway.serve() -will take and execute such items, one by one. This means -that by incoming default execution is single-threaded. +Code execution messages are scheduled on a WorkerPool. +On the worker, ``serve()`` integrates the main thread as the +primary executor when using the ``thread`` / ``main_thread_only`` +models. The receiver thread terminates if the remote side sends a gateway termination message or if the IO-connection drops. -It puts an end symbol into the execqueue so -that serve() can cleanly finish as well. diff --git a/pyproject.toml b/pyproject.toml index fcd2b4b5..709f887b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,9 @@ description = "execnet: rapid multi-Python deployment" readme = {"file" = "README.rst", "content-type" = "text/x-rst"} license = "MIT" requires-python = ">=3.10" +dependencies = [ + "trio>=0.32", +] authors = [ { name = "holger krekel and others" }, ] diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py new file mode 100644 index 00000000..6ece94c4 --- /dev/null +++ b/src/execnet/_trio_host.py @@ -0,0 +1,527 @@ +"""Trio host thread for execnet Message-protocol IO. + +Coordinator and worker both run framed read/write loops here. +Sync Channel/Gateway APIs talk to this host via thread-safe queues and +``trio.from_thread``. +""" + +from __future__ import annotations + +import os +import queue +import subprocess +import threading +from collections.abc import Callable +from typing import TYPE_CHECKING +from typing import Any +from typing import Protocol +from typing import TypeVar + +import trio + +from .gateway_base import ExecModel +from .gateway_base import GatewayReceivedTerminate +from .gateway_base import Message +from .gateway_base import trace + +if TYPE_CHECKING: + from .gateway_base import BaseGateway + +T = TypeVar("T") + +_CLOSE_WRITE = object() +_ENABLED_ENV = "EXECNET_TRIO_HOST" + + +def trio_host_enabled() -> bool: + """Return whether the Trio IO path should be used when applicable.""" + value = os.environ.get(_ENABLED_ENV, "1").strip().lower() + return value not in ("0", "false", "no", "off") + + +def should_use_trio_popen(spec: Any) -> bool: + """PoC: Trio path only for local popen with import bootstrap.""" + if not trio_host_enabled(): + return False + if not getattr(spec, "popen", False): + return False + if getattr(spec, "via", None): + return False + if getattr(spec, "python", None): + return False + # Worker exec still uses WorkerPool; greenlet models stay on the legacy path. + execmodel = getattr(spec, "execmodel", None) + if execmodel not in (None, "thread", "main_thread_only"): + return False + return True + + +class AsyncByteIO(Protocol): + async def read_exact(self, n: int) -> bytes: ... + + async def write_all(self, data: bytes) -> None: ... + + async def aclose_read(self) -> None: ... + + async def aclose_write(self) -> None: ... + + +async def read_exact_receive_stream(stream: trio.abc.ReceiveStream, n: int) -> bytes: + buf = bytearray() + while len(buf) < n: + chunk = await stream.receive_some(n - len(buf)) + if not chunk: + raise EOFError("expected %d bytes, got %d" % (n, len(buf))) + buf += chunk + return bytes(buf) + + +async def read_message(io: AsyncByteIO) -> Message: + header = await io.read_exact(9) + msgtype, channel, payload = Message.from_header(header) + data = await io.read_exact(payload) if payload else b"" + return Message.from_parts(msgtype, channel, data) + + +class ProcessStreamsIO: + """Async IO over a Trio Process stdin/stdout pair.""" + + def __init__(self, process: trio.Process) -> None: + assert process.stdin is not None + assert process.stdout is not None + self.process = process + self._stdin = process.stdin + self._stdout = process.stdout + + async def read_exact(self, n: int) -> bytes: + return await read_exact_receive_stream(self._stdout, n) + + async def write_all(self, data: bytes) -> None: + await self._stdin.send_all(data) + + async def aclose_read(self) -> None: + await self._stdout.aclose() + + async def aclose_write(self) -> None: + await self._stdin.aclose() + + +class FdStreamsIO: + """Async IO over OS file descriptors (worker stdio pipes).""" + + def __init__(self, read_fd: int, write_fd: int) -> None: + self._read = trio.lowlevel.FdStream(read_fd) + self._write = trio.lowlevel.FdStream(write_fd) + + async def read_exact(self, n: int) -> bytes: + return await read_exact_receive_stream(self._read, n) + + async def write_all(self, data: bytes) -> None: + await self._write.send_all(data) + + async def aclose_read(self) -> None: + await self._read.aclose() + + async def aclose_write(self) -> None: + await self._write.aclose() + + +class SyncIOHandle: + """Sync IO facade for Group.terminate wait/kill/close_write.""" + + remoteaddress: str + + def __init__( + self, + execmodel: ExecModel, + session: ProtocolSession, + *, + remoteaddress: str | None = None, + ) -> None: + self.execmodel = execmodel + self._session = session + if remoteaddress is not None: + self.remoteaddress = remoteaddress + + def read(self, numbytes: int) -> bytes: + raise RuntimeError("sync read not supported on Trio IO handle") + + def write(self, data: bytes) -> None: + raise RuntimeError("sync write not supported on Trio IO handle") + + def close_read(self) -> None: + self._session.request_close_read() + + def close_write(self) -> None: + self._session.request_close_write() + + def wait(self) -> int | None: + return self._session.wait_process() + + def kill(self) -> None: + self._session.kill_process() + + +class ProtocolSession: + """Reader/writer tasks for one gateway connection.""" + + def __init__( + self, + gateway: BaseGateway, + io: AsyncByteIO, + *, + process: trio.Process | None = None, + host: TrioHost, + ) -> None: + self.gateway = gateway + self.io = io + self.process = process + self.host = host + self._outbound: queue.SimpleQueue[object] = queue.SimpleQueue() + self._done = threading.Event() + self._process_exitcode: int | None = None + self._process_done = threading.Event() + self._send_closed = False + self._lock = threading.Lock() + + def enqueue_message(self, message: Message) -> None: + """Enqueue a frame; wait until written when safe to block. + + Non-host threads wait for the OS write so abrupt ``os._exit`` (xdist + crash tests) cannot drop already-"sent" data still in the queue. + + The Trio host thread (receiver callbacks) must not wait — that would + deadlock the writer task on the same event loop. + """ + wait = not self.host.is_host_thread() + done = threading.Event() if wait else None + errors: list[BaseException] = [] + with self._lock: + if self._send_closed or self._done.is_set(): + raise OSError("cannot send (already closed?)") + self._outbound.put((message.pack(), done, errors)) + if done is None: + return + if not done.wait(timeout=120.0): + raise OSError("cannot send (write timed out)") + if errors: + raise OSError("cannot send (already closed?)") from errors[0] + + def request_close_write(self) -> None: + with self._lock: + if self._send_closed: + return + self._send_closed = True + self._outbound.put(_CLOSE_WRITE) + + def request_close_read(self) -> None: + # Reader observes EOF / cancel; nothing required from callers. + return + + def wait_done(self, timeout: float | None = None) -> bool: + return self._done.wait(timeout) + + def wait_process(self) -> int | None: + if self.process is None: + return None + + async def _wait() -> int | None: + assert self.process is not None + # Always await wait() so the child is reaped (no zombies). + code = await self.process.wait() + self._process_exitcode = code + self._process_done.set() + return code + + try: + return self.host.call(_wait) + except Exception: + self._process_done.wait() + return self._process_exitcode + + def kill_process(self) -> None: + if self.process is None: + return + + async def _kill() -> None: + assert self.process is not None + with trio.move_on_after(5): + self.process.kill() + self._process_exitcode = await self.process.wait() + self._process_done.set() + + try: + self.host.call(_kill) + except Exception as exc: + trace("ERROR killing trio process:", exc) + + def is_alive(self) -> bool: + return not self._done.is_set() + + async def task(self, task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED) -> None: + try: + async with trio.open_nursery() as nursery: + nursery.start_soon(self._writer) + if self.process is not None: + nursery.start_soon(self._supervisor) + task_status.started() + await self._reader() + nursery.cancel_scope.cancel() + finally: + await self._finish() + + async def _reader(self) -> None: + gateway = self.gateway + + def log(*msg: object) -> None: + gateway._trace("[trio-receiver]", *msg) + + log("RECEIVER: starting") + try: + while True: + msg = await read_message(self.io) + log("received", msg) + with gateway._receivelock: + msg.received(gateway) + del msg + except GatewayReceivedTerminate: + log("GATEWAY_TERMINATE") + except EOFError as exc: + log("EOF without prior gateway termination message") + gateway._error = exc + except Exception as exc: + log(gateway._geterrortext(exc)) + log("finishing receiver") + + async def _writer(self) -> None: + while True: + # abandon_on_cancel: queue.get blocks forever until a sentinel; + # nursery shutdown must not wait on it. + item = await trio.to_thread.run_sync( + self._outbound.get, abandon_on_cancel=True + ) + if item is _CLOSE_WRITE: + try: + await self.io.aclose_write() + except Exception as exc: + self.gateway._trace("aclose_write failed", exc) + return + assert isinstance(item, tuple) + blob, done, errors = item + assert isinstance(blob, bytes) + try: + await self.io.write_all(blob) + except Exception as exc: + self.gateway._trace("write failed", exc) + errors.append(exc) + with self._lock: + self._send_closed = True + finally: + if done is not None: + done.set() + + async def _supervisor(self) -> None: + assert self.process is not None + try: + self._process_exitcode = await self.process.wait() + finally: + self._process_done.set() + + async def _finish(self) -> None: + gateway = self.gateway + # Unblock a writer thread left in queue.get after cancel. + with self._lock: + self._send_closed = True + self._outbound.put(_CLOSE_WRITE) + gateway._trace("[trio-receiver] finishing channels") + gateway._channelfactory._finished_receiving() + # Unblock WorkerGateway.serve()/join before heavy exec-pool shutdown + # so the primary thread is not waiting on _done while terminate waits + # on the primary thread draining work. + self._done.set() + gateway._trace("[trio-receiver] terminating execution") + # May sleep/SIGINT; keep it off the Trio scheduling thread. + await trio.to_thread.run_sync( + gateway._terminate_execution, abandon_on_cancel=True + ) + try: + await self.io.aclose_read() + except Exception: + pass + try: + await self.io.aclose_write() + except Exception: + pass + + +class TrioHost: + """Dedicated OS thread running ``trio.run`` for protocol IO.""" + + def __init__(self, name: str = "execnet-trio-host") -> None: + self._name = name + self._thread: threading.Thread | None = None + self._token: trio.lowlevel.TrioToken | None = None + self._nursery: trio.Nursery | None = None + self._ready = threading.Event() + self._shutdown: trio.Event | None = None + self._started = False + + def start(self) -> None: + if self._started: + return + self._thread = threading.Thread(target=self._run, name=self._name, daemon=True) + self._thread.start() + if not self._ready.wait(timeout=30): + raise RuntimeError("TrioHost failed to start") + self._started = True + + def is_host_thread(self) -> bool: + return self._thread is not None and threading.current_thread() is self._thread + + def _run(self) -> None: + trio.run(self._main) + + async def _main(self) -> None: + self._token = trio.lowlevel.current_trio_token() + self._shutdown = trio.Event() + try: + async with trio.open_nursery() as nursery: + self._nursery = nursery + self._ready.set() + await self._shutdown.wait() + nursery.cancel_scope.cancel() + finally: + self._nursery = None + + def call(self, async_fn: Callable[..., Any], *args: Any) -> Any: + if self._token is None: + raise RuntimeError("TrioHost is not running") + return trio.from_thread.run(async_fn, *args, trio_token=self._token) + + def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: + if self._token is None: + raise RuntimeError("TrioHost is not running") + return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + + async def start_session( + self, + gateway: BaseGateway, + io: AsyncByteIO, + *, + process: trio.Process | None = None, + ) -> ProtocolSession: + if self._nursery is None: + raise RuntimeError("TrioHost nursery is not available") + session = ProtocolSession(gateway, io, process=process, host=self) + await self._nursery.start(session.task) + return session + + def stop(self, timeout: float | None = 5.0) -> None: + if not self._started or self._token is None or self._shutdown is None: + return + + def _set() -> None: + assert self._shutdown is not None + self._shutdown.set() + + try: + trio.from_thread.run_sync(_set, trio_token=self._token) + except Exception: + pass + if self._thread is not None: + self._thread.join(timeout=timeout) + self._started = False + + +async def open_popen_process(args: list[str]) -> trio.Process: + return await trio.lowlevel.open_process( + args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + +async def bootstrap_import_async( + process: trio.Process, + *, + importdir: str, + execmodel: str, + gateway_id: str, +) -> ProcessStreamsIO: + """Send import-bootstrap source and wait for the handshake byte.""" + io = ProcessStreamsIO(process) + sources = [ + "import sys", + "if %r not in sys.path:" % importdir, + " sys.path.insert(0, %r)" % importdir, + "from execnet._trio_worker import serve_popen_trio", + "sys.stdout.write('1')", + "sys.stdout.flush()", + "serve_popen_trio(id=%r, execmodel=%r)" % (f"{gateway_id}-worker", execmodel), + ] + source = "\n".join(sources) + await io.write_all((repr(source) + "\n").encode("utf-8")) + ack = await io.read_exact(1) + if ack != b"1": + raise EOFError(f"bad bootstrap handshake: {ack!r}") + return io + + +class _TempIO: + """Placeholder IO used only while constructing a Trio-backed Gateway.""" + + def __init__(self, execmodel: ExecModel) -> None: + self.execmodel = execmodel + + def read(self, numbytes: int) -> bytes: + raise RuntimeError("sync read not supported on Trio temp IO") + + def write(self, data: bytes) -> None: + raise RuntimeError("sync write not supported on Trio temp IO") + + def close_read(self) -> None: + return + + def close_write(self) -> None: + return + + def wait(self) -> int | None: + return None + + def kill(self) -> None: + return + + +def makegateway_popen_trio(group: Any, spec: Any) -> Any: + """Create a popen Gateway using Trio process + protocol IO on both sides.""" + import execnet + + from . import gateway_io + from .gateway_bootstrap import importdir + + host: TrioHost = group._ensure_trio_host() + args = gateway_io.popen_args(spec) + remote_execmodel = spec.execmodel + + async def _create_and_attach() -> Any: + process = await open_popen_process(args) + try: + async_io = await bootstrap_import_async( + process, + importdir=importdir, + execmodel=remote_execmodel, + gateway_id=spec.id, + ) + except BaseException: + with trio.move_on_after(5): + process.kill() + await process.wait() + raise + + gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + session = await host.start_session(gw, async_io, process=process) + gw._attach_trio_session(session) + gw._io = SyncIOHandle(group.execmodel, session) + return gw + + return host.call(_create_and_attach) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py new file mode 100644 index 00000000..70338f1c --- /dev/null +++ b/src/execnet/_trio_worker.py @@ -0,0 +1,117 @@ +"""Worker-side Trio networking entry for popen/import bootstrap.""" + +from __future__ import annotations + +import os +import sys + +from . import gateway_base +from .gateway_base import WorkerGateway +from .gateway_base import get_execmodel +from .gateway_base import trace + + +def _prepare_protocol_fds() -> tuple[int, int]: + """Dup protocol pipes off stdin/stdout and redirect stdio to /dev/null. + + Returns ``(read_fd, write_fd)`` for the Message protocol (child reads + coordinator stdin writes; child writes go to coordinator stdout reads). + """ + if not hasattr(os, "dup"): # pragma: no cover - jython legacy + raise RuntimeError("Trio worker requires os.dup") + + try: + devnull = os.devnull + except AttributeError: + devnull = "NUL" if os.name == "nt" else "/dev/null" + + # Protocol read end: former stdin (fed by coordinator stdout write / our stdin) + read_fd = os.dup(0) + fd = os.open(devnull, os.O_RDONLY) + os.dup2(fd, 0) + os.close(fd) + + # Protocol write end: former stdout + write_fd = os.dup(1) + fd = os.open(devnull, os.O_WRONLY) + os.dup2(fd, 1) + + if os.name == "nt": + # Match init_popen_io: keep a stderr handle then point fd 2 at null. + sys.stderr = os.fdopen(os.dup(2), "w", 1) + os.dup2(fd, 2) + os.close(fd) + + # Replace sys.stdin/out with the null fds (closefd=False). + sys.stdin = os.fdopen(0, "r", 1, closefd=False) + sys.stdout = os.fdopen(1, "w", 1, closefd=False) + return read_fd, write_fd + + +class _WorkerIOStub: + """Minimal IO stub so WorkerGateway can be constructed without sync pipes.""" + + def __init__(self, execmodel: gateway_base.ExecModel) -> None: + self.execmodel = execmodel + + def read(self, numbytes: int) -> bytes: + raise RuntimeError("sync read not used on Trio worker") + + def write(self, data: bytes) -> None: + raise RuntimeError("sync write not used on Trio worker") + + def close_read(self) -> None: + return + + def close_write(self) -> None: + return + + def wait(self) -> int | None: + return None + + def kill(self) -> None: + return + + +def serve_popen_trio(id: str, execmodel: str = "thread") -> None: + """Serve a WorkerGateway with Message IO on a Trio host thread.""" + from . import _trio_host + + model = get_execmodel(execmodel) + read_fd, write_fd = _prepare_protocol_fds() + # Keep the historic trace token so tests looking for workergateway still pass. + trace(f"creating workergateway on trio id={id!r}") + + host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") + host.start() + try: + io_stub = _WorkerIOStub(model) + gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) + + hasprimary = model.backend in ("thread", "main_thread_only") + gateway._execpool = gateway_base.WorkerPool(model, hasprimary=hasprimary) + gateway._executetask_complete = None + if model.backend == "main_thread_only": + gateway._executetask_complete = model.Event() + gateway._executetask_complete.set() + + async def _start() -> _trio_host.ProtocolSession: + async_io = _trio_host.FdStreamsIO(read_fd, write_fd) + return await host.start_session(gateway, async_io) + + session = host.call(_start) + gateway._attach_trio_session(session) + + try: + if hasprimary: + trace("integrating as primary thread (trio worker)") + gateway._execpool.integrate_as_primary_thread() + gateway.join() + except KeyboardInterrupt: + # Match WorkerGateway.serve(): swallow in the worker. + trace("swallowing keyboardinterrupt, serve finished") + finally: + host.stop(timeout=5.0) + # Trio's to_thread cache uses non-daemon threads that would otherwise + # keep this disposable worker process alive after serve returns. + os._exit(0) diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index 1d3eb59d..360b9865 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -26,11 +26,21 @@ class Gateway(gateway_base.BaseGateway): _group: Group - def __init__(self, io: IO, spec: XSpec) -> None: + def __init__( + self, + io: IO, + spec: XSpec, + *, + trio_session: object | None = None, + defer_receive: bool = False, + ) -> None: """:private:""" super().__init__(io=io, id=spec.id, _startcount=1) self.spec = spec - self._initreceive() + if trio_session is not None: + self._attach_trio_session(trio_session) + elif not defer_receive: + self._initreceive() @property def remoteaddress(self) -> str: @@ -91,6 +101,9 @@ def _rinfo(self, update: bool = False) -> RInfo: def hasreceiver(self) -> bool: """Whether gateway is able to receive data.""" + session = self._trio_session + if session is not None: + return session.is_alive() return self._receivepool.active_count() > 0 def remote_status(self) -> RemoteStatus: diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 73dc7175..0829a33d 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -563,6 +563,23 @@ def __init__(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: self.channelid = channelid self.data = data + def pack(self) -> bytes: + """Return the full wire frame (9-byte header + payload).""" + header = struct.pack("!bii", self.msgcode, self.channelid, len(self.data)) + return header + self.data + + @staticmethod + def from_header(header: bytes) -> tuple[int, int, int]: + """Unpack a 9-byte header into (msgtype, channelid, payload_len).""" + if len(header) != 9: + raise EOFError("couldn't load message header, short read") + msgtype, channel, payload = struct.unpack("!bii", header) + return msgtype, channel, payload + + @staticmethod + def from_parts(msgtype: int, channel: int, data: bytes) -> Message: + return Message(msgtype, channel, data) + @staticmethod def from_io(io: ReadIO) -> Message: try: @@ -571,12 +588,11 @@ def from_io(io: ReadIO) -> Message: raise EOFError("empty read") except EOFError as e: raise EOFError("couldn't load message header, " + e.args[0]) from None - msgtype, channel, payload = struct.unpack("!bii", header) + msgtype, channel, payload = Message.from_header(header) return Message(msgtype, channel, io.read(payload)) def to_io(self, io: WriteIO) -> None: - header = struct.pack("!bii", self.msgcode, self.channelid, len(self.data)) - io.write(header + self.data) + io.write(self.pack()) def received(self, gateway: BaseGateway) -> None: handler = self._types[self.msgcode][1] @@ -1125,6 +1141,7 @@ def readline(self) -> str: class BaseGateway: _sysex = sysex id = "" + _trio_session: Any = None def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel @@ -1137,11 +1154,18 @@ def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.__trace = trace self._geterrortext = geterrortext self._receivepool = WorkerPool(self.execmodel) + self._trio_session = None def _trace(self, *msg: object) -> None: self.__trace(self.id, *msg) + def _attach_trio_session(self, session: Any) -> None: + """Attach a Trio ProtocolSession for Message IO (no receiver thread).""" + self._trio_session = session + def _initreceive(self) -> None: + if self._trio_session is not None: + return self._receivepool.spawn(self._thread_receiver) def _thread_receiver(self) -> None: @@ -1181,6 +1205,15 @@ def _terminate_execution(self) -> None: def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: message = Message(msgcode, channelid, data) + session = self._trio_session + if session is not None: + try: + session.enqueue_message(message) + self._trace("sent", message) + except (OSError, ValueError) as e: + self._trace("failed to send", message, e) + raise OSError("cannot send (already closed?)") from e + return try: message.to_io(self._io) self._trace("sent", message) @@ -1204,6 +1237,10 @@ def newchannel(self) -> Channel: def join(self, timeout: float | None = None) -> None: """Wait for receiverthread to terminate.""" self._trace("waiting for receiver thread to finish") + session = self._trio_session + if session is not None: + session.wait_done(timeout) + return self._receivepool.waitall(timeout) diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 4dbf8b89..8079f4c4 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -52,6 +52,7 @@ def __init__( self._autoidcounter = 0 self._autoidlock = Lock() self._gateways_to_join: list[Gateway] = [] + self._trio_host: Any = None # we use the same execmodel for all of the Gateway objects # we spawn on our side. Probably we should not allow different # execmodels between different groups but not clear. @@ -62,6 +63,14 @@ def __init__( self.makegateway(xspec) atexit.register(self._cleanup_atexit) + def _ensure_trio_host(self) -> Any: + if self._trio_host is None: + from . import _trio_host + + self._trio_host = _trio_host.TrioHost(name="execnet-trio-group") + self._trio_host.start() + return self._trio_host + @property def execmodel(self) -> ExecModel: return self._execmodel @@ -143,7 +152,11 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: self.allocate_id(spec) if spec.execmodel is None: spec.execmodel = self.remote_execmodel.backend - if spec.via: + from . import _trio_host + + if _trio_host.should_use_trio_popen(spec): + gw = _trio_host.makegateway_popen_trio(self, spec) + elif spec.via: assert not spec.socket master = self[spec.via] proxy_channel = master.remote_exec(gateway_io) @@ -207,6 +220,9 @@ def _unregister(self, gateway: Gateway) -> None: def _cleanup_atexit(self) -> None: trace(f"=== atexit cleanup {self!r} ===") self.terminate(timeout=1.0) + if self._trio_host is not None: + self._trio_host.stop(timeout=1.0) + self._trio_host = None def terminate(self, timeout: float | None = None) -> None: """Trigger exit of member gateways and wait for termination diff --git a/uv.lock b/uv.lock index 893f1543..fc2da46b 100644 --- a/uv.lock +++ b/uv.lock @@ -16,6 +16,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, ] +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + [[package]] name = "backports-tarfile" version = "1.2.0" @@ -333,6 +342,9 @@ wheels = [ [[package]] name = "execnet" source = { editable = "." } +dependencies = [ + { name = "trio" }, +] [package.optional-dependencies] testing = [ @@ -358,6 +370,7 @@ requires-dist = [ { name = "pytest", marker = "extra == 'testing'", specifier = ">8.0" }, { name = "pytest-timeout", marker = "extra == 'testing'" }, { name = "tox", marker = "extra == 'testing'" }, + { name = "trio", specifier = ">=0.32" }, { name = "uv", marker = "extra == 'testing'" }, ] provides-extras = ["testing"] @@ -723,6 +736,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314, upload-time = "2024-06-04T18:44:08.352Z" }, ] +[[package]] +name = "outcome" +version = "1.3.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/df/77698abfac98571e65ffeb0c1fba8ffd692ab8458d617a0eed7d9a8d38f2/outcome-1.3.0.post0.tar.gz", hash = "sha256:9dcf02e65f2971b80047b377468e72a268e15c0af3cf1238e6ff14f7f91143b8", size = 21060, upload-time = "2023-10-26T04:26:04.361Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/8b/5ab7257531a5d830fc8000c476e63c935488d74609b50f9384a643ec0a62/outcome-1.3.0.post0-py2.py3-none-any.whl", hash = "sha256:e771c5ce06d1415e356078d3bdd68523f284b4ce5419828922b6871e65eda82b", size = 10692, upload-time = "2023-10-26T04:26:02.532Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -977,6 +1002,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "tomli" version = "2.4.1" @@ -1072,6 +1115,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6b/3d/7ba55871e9d794d40b6c8424f2e5d1b267ea5d9a4bd2175e08b57960ba13/tox-4.58.0-py3-none-any.whl", hash = "sha256:dcae21f5f015f3a67658e35644cce0d1aa0dedcd06f3927f95d84e1717f6cea5", size = 223298, upload-time = "2026-07-21T13:10:34.731Z" }, ] +[[package]] +name = "trio" +version = "0.33.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "cffi", marker = "implementation_name != 'pypy' and os_name == 'nt'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "outcome" }, + { name = "sniffio" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/b6/c744031c6f89b18b3f5f4f7338603ab381d740a7f45938c4607b2302481f/trio-0.33.0.tar.gz", hash = "sha256:a29b92b73f09d4b48ed249acd91073281a7f1063f09caba5dc70465b5c7aa970", size = 605109, upload-time = "2026-02-14T18:40:55.386Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/93/dab25dc87ac48da0fe0f6419e07d0bfd98799bed4e05e7b9e0f85a1a4b4b/trio-0.33.0-py3-none-any.whl", hash = "sha256:3bd5d87f781d9b0192d592aef28691f8951d6c2e41b7e1da4c25cde6c180ae9b", size = 510294, upload-time = "2026-02-14T18:40:53.313Z" }, +] + [[package]] name = "trove-classifiers" version = "2025.5.9.12" From beb36b92d05bab211017e5324519e05eb37faba7 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 22 Jul 2026 18:26:19 +0200 Subject: [PATCH 02/91] feat: schedule worker remote_exec from the Trio nursery Replace WorkerPool on the Trio popen worker with TrioWorkerExec: thread-model tasks run via trio.to_thread, main_thread_only hands off to the process main thread. Also wake the protocol writer with a Trio Event instead of blocking to_thread queue.get. Co-authored-by: Cursor AI Co-authored-by: Cursor Grok 4.5 --- doc/implnotes.rst | 5 +- src/execnet/_trio_host.py | 81 +++++++++++++----- src/execnet/_trio_worker.py | 159 +++++++++++++++++++++++++++++++++--- src/execnet/gateway_base.py | 7 ++ 4 files changed, 217 insertions(+), 35 deletions(-) diff --git a/doc/implnotes.rst b/doc/implnotes.rst index 3dae82d6..c8229865 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -26,8 +26,9 @@ event loop (``execnet._trio_host.TrioHost``): * Coordinator: ``trio.lowlevel.open_process`` plus async framed reader/writer tasks per gateway (one host thread per ``Group``). * Worker: ``serve_popen_trio`` adopts stdio pipe fds into Trio - streams; ``remote_exec`` still runs on the existing WorkerPool - (including ``main_thread_only`` primary-thread integration). + streams; ``remote_exec`` is scheduled from the Trio nursery + (``trio.to_thread`` for ``thread``, main-thread handoff for + ``main_thread_only``). Sync ``Channel`` / ``Gateway`` APIs are unchanged. Sends from non-host threads wait until the frame is written (so abrupt diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 6ece94c4..b0db9ad7 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -178,6 +178,7 @@ def __init__( self.process = process self.host = host self._outbound: queue.SimpleQueue[object] = queue.SimpleQueue() + self._wake: trio.Event | None = None self._done = threading.Event() self._process_exitcode: int | None = None self._process_done = threading.Event() @@ -200,6 +201,7 @@ def enqueue_message(self, message: Message) -> None: if self._send_closed or self._done.is_set(): raise OSError("cannot send (already closed?)") self._outbound.put((message.pack(), done, errors)) + self._wake_writer() if done is None: return if not done.wait(timeout=120.0): @@ -207,12 +209,25 @@ def enqueue_message(self, message: Message) -> None: if errors: raise OSError("cannot send (already closed?)") from errors[0] + def _wake_writer(self) -> None: + wake = self._wake + if wake is None: + return + if self.host.is_host_thread(): + wake.set() + else: + try: + self.host.call_sync(wake.set) + except Exception: + pass + def request_close_write(self) -> None: with self._lock: if self._send_closed: return self._send_closed = True self._outbound.put(_CLOSE_WRITE) + self._wake_writer() def request_close_read(self) -> None: # Reader observes EOF / cancel; nothing required from callers. @@ -294,31 +309,47 @@ def log(*msg: object) -> None: log("finishing receiver") async def _writer(self) -> None: + self._wake = trio.Event() while True: - # abandon_on_cancel: queue.get blocks forever until a sentinel; - # nursery shutdown must not wait on it. - item = await trio.to_thread.run_sync( - self._outbound.get, abandon_on_cancel=True - ) - if item is _CLOSE_WRITE: + while True: try: - await self.io.aclose_write() - except Exception as exc: - self.gateway._trace("aclose_write failed", exc) + item = self._outbound.get_nowait() + except queue.Empty: + break + if not await self._writer_handle_item(item): + return + # Reset wake before re-check to avoid losing a notification. + self._wake = trio.Event() + try: + item = self._outbound.get_nowait() + except queue.Empty: + await self._wake.wait() + continue + if not await self._writer_handle_item(item): return - assert isinstance(item, tuple) - blob, done, errors = item - assert isinstance(blob, bytes) + + async def _writer_handle_item(self, item: object) -> bool: + """Handle one outbound queue item. Return False when writer should stop.""" + if item is _CLOSE_WRITE: try: - await self.io.write_all(blob) + await self.io.aclose_write() except Exception as exc: - self.gateway._trace("write failed", exc) - errors.append(exc) - with self._lock: - self._send_closed = True - finally: - if done is not None: - done.set() + self.gateway._trace("aclose_write failed", exc) + return False + assert isinstance(item, tuple) + blob, done, errors = item + assert isinstance(blob, bytes) + try: + await self.io.write_all(blob) + except Exception as exc: + self.gateway._trace("write failed", exc) + errors.append(exc) + with self._lock: + self._send_closed = True + finally: + if done is not None: + done.set() + return True async def _supervisor(self) -> None: assert self.process is not None @@ -329,10 +360,10 @@ async def _supervisor(self) -> None: async def _finish(self) -> None: gateway = self.gateway - # Unblock a writer thread left in queue.get after cancel. with self._lock: self._send_closed = True self._outbound.put(_CLOSE_WRITE) + self._wake_writer() gateway._trace("[trio-receiver] finishing channels") gateway._channelfactory._finished_receiving() # Unblock WorkerGateway.serve()/join before heavy exec-pool shutdown @@ -403,6 +434,14 @@ def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: raise RuntimeError("TrioHost is not running") return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: + """Schedule a task on the root nursery (must be called on the host thread).""" + if not self.is_host_thread(): + raise RuntimeError("start_soon requires the Trio host thread") + if self._nursery is None: + raise RuntimeError("TrioHost nursery is not available") + self._nursery.start_soon(async_fn, *args) + async def start_session( self, gateway: BaseGateway, diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 70338f1c..6d247194 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -1,15 +1,151 @@ -"""Worker-side Trio networking entry for popen/import bootstrap.""" +"""Worker-side Trio networking and exec scheduling for popen/import bootstrap.""" from __future__ import annotations import os +import queue import sys +import threading +from typing import TYPE_CHECKING +from typing import Any + +import trio from . import gateway_base +from .gateway_base import MAIN_THREAD_ONLY_DEADLOCK_TEXT from .gateway_base import WorkerGateway from .gateway_base import get_execmodel +from .gateway_base import loads_internal from .gateway_base import trace +if TYPE_CHECKING: + from . import _trio_host + from .gateway_base import Channel + from .gateway_base import ExecModel + +ExecItem = tuple[Any, ...] + + +class TrioWorkerExec: + """Schedule ``remote_exec`` work from the Trio host nursery. + + * ``thread``: run ``executetask`` via ``trio.to_thread`` (concurrent). + * ``main_thread_only``: hand off to the process main thread (GUI-safe). + """ + + def __init__( + self, + host: _trio_host.TrioHost, + gateway: WorkerGateway, + *, + main_thread_only: bool, + ) -> None: + self.host = host + self.gateway = gateway + self.main_thread_only = main_thread_only + self._lock = threading.Lock() + self._running = 0 + self._shutting_down = False + self._idle = threading.Event() + self._idle.set() + self._primary_q: queue.SimpleQueue[ + tuple[Channel, ExecItem, threading.Event] | None + ] = queue.SimpleQueue() + self._primary_wake = threading.Event() + # Serialize main_thread_only admission (wait+clear) so two tasks cannot + # both observe the idle Event before either clears it. + self._admit_lock = trio.Lock() + + def active_count(self) -> int: + with self._lock: + return self._running + + def _track_start(self) -> None: + with self._lock: + self._running += 1 + self._idle.clear() + + def _track_finish(self) -> None: + with self._lock: + self._running -= 1 + if self._running == 0: + self._idle.set() + + def schedule(self, channel: Channel, sourcetask: bytes) -> None: + """Called from the Trio receiver while holding ``_receivelock``. + + Must not block: deadlock checks and exec run in a nursery task. + """ + item = loads_internal(sourcetask) + assert isinstance(item, tuple) + with self._lock: + if self._shutting_down: + channel.close("execution disallowed") + return + # Already on the Trio host thread (Message handler). + self.host.start_soon(self._run_exec, channel, item) + + async def _run_exec(self, channel: Channel, item: ExecItem) -> None: + if self.main_thread_only: + complete = self.gateway._executetask_complete + assert complete is not None + + def _wait_slot() -> bool: + return complete.wait(timeout=1) + + async with self._admit_lock: + if not await trio.to_thread.run_sync( + _wait_slot, abandon_on_cancel=True + ): + channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) + return + complete.clear() + + self._track_start() + try: + if self.main_thread_only: + done = threading.Event() + self._primary_q.put((channel, item, done)) + self._primary_wake.set() + await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) + else: + await trio.to_thread.run_sync( + self.gateway.executetask, + (channel, item), + abandon_on_cancel=True, + ) + finally: + self._track_finish() + + def integrate_as_primary_thread(self) -> None: + """Block the main thread running main_thread_only exec tasks.""" + while True: + self._primary_wake.wait() + try: + task = self._primary_q.get_nowait() + except queue.Empty: + self._primary_wake.clear() + try: + task = self._primary_q.get_nowait() + except queue.Empty: + continue + if task is None: + break + channel, item, done = task + try: + self.gateway.executetask((channel, item)) + finally: + done.set() + + def trigger_shutdown(self) -> None: + with self._lock: + self._shutting_down = True + self._primary_q.put(None) + self._primary_wake.set() + + def waitall(self, timeout: float | None = None) -> bool: + return self._idle.wait(timeout) + def _prepare_protocol_fds() -> tuple[int, int]: """Dup protocol pipes off stdin/stdout and redirect stdio to /dev/null. @@ -25,24 +161,20 @@ def _prepare_protocol_fds() -> tuple[int, int]: except AttributeError: devnull = "NUL" if os.name == "nt" else "/dev/null" - # Protocol read end: former stdin (fed by coordinator stdout write / our stdin) read_fd = os.dup(0) fd = os.open(devnull, os.O_RDONLY) os.dup2(fd, 0) os.close(fd) - # Protocol write end: former stdout write_fd = os.dup(1) fd = os.open(devnull, os.O_WRONLY) os.dup2(fd, 1) if os.name == "nt": - # Match init_popen_io: keep a stderr handle then point fd 2 at null. sys.stderr = os.fdopen(os.dup(2), "w", 1) os.dup2(fd, 2) os.close(fd) - # Replace sys.stdin/out with the null fds (closefd=False). sys.stdin = os.fdopen(0, "r", 1, closefd=False) sys.stdout = os.fdopen(1, "w", 1, closefd=False) return read_fd, write_fd @@ -51,7 +183,7 @@ def _prepare_protocol_fds() -> tuple[int, int]: class _WorkerIOStub: """Minimal IO stub so WorkerGateway can be constructed without sync pipes.""" - def __init__(self, execmodel: gateway_base.ExecModel) -> None: + def __init__(self, execmodel: ExecModel) -> None: self.execmodel = execmodel def read(self, numbytes: int) -> bytes: @@ -74,7 +206,7 @@ def kill(self) -> None: def serve_popen_trio(id: str, execmodel: str = "thread") -> None: - """Serve a WorkerGateway with Message IO on a Trio host thread.""" + """Serve a WorkerGateway with Message IO + exec scheduling on Trio.""" from . import _trio_host model = get_execmodel(execmodel) @@ -88,10 +220,13 @@ def serve_popen_trio(id: str, execmodel: str = "thread") -> None: io_stub = _WorkerIOStub(model) gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) - hasprimary = model.backend in ("thread", "main_thread_only") - gateway._execpool = gateway_base.WorkerPool(model, hasprimary=hasprimary) + main_thread_only = model.backend == "main_thread_only" + trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) + # Duck-type as WorkerPool for STATUS / _terminate_execution. + gateway._execpool = trio_exec # type: ignore[assignment] + gateway._trio_exec = trio_exec gateway._executetask_complete = None - if model.backend == "main_thread_only": + if main_thread_only: gateway._executetask_complete = model.Event() gateway._executetask_complete.set() @@ -103,9 +238,9 @@ async def _start() -> _trio_host.ProtocolSession: gateway._attach_trio_session(session) try: - if hasprimary: + if main_thread_only: trace("integrating as primary thread (trio worker)") - gateway._execpool.integrate_as_primary_thread() + trio_exec.integrate_as_primary_thread() gateway.join() except KeyboardInterrupt: # Match WorkerGateway.serve(): swallow in the worker. diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 0829a33d..8cf3cb09 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1245,7 +1245,14 @@ def join(self, timeout: float | None = None) -> None: class WorkerGateway(BaseGateway): + _trio_exec: Any = None + def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: + trio_exec = getattr(self, "_trio_exec", None) + if trio_exec is not None: + trio_exec.schedule(channel, sourcetask) + return + if self._execpool.execmodel.backend == "main_thread_only": assert self._executetask_complete is not None # It's necessary to wait for a short time in order to ensure From 37a184ff425f1f560e5da21d8c4ac4ceace069f0 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 22 Jul 2026 21:53:03 +0200 Subject: [PATCH 03/91] refactor: drop eventlet and gevent execmodels Remove the EventletExecModel and GeventExecModel backends and their get_execmodel branches, leaving only the stdlib thread and main_thread_only models. Narrow the test execmodel fixture, drop the gevent test dependency and the eventlet/gevent mypy overrides, and scrub the docs of the removed backends. This is phase 1 of moving execnet onto Trio: the ExecModel surface is kept intact for now and collapsed further once Trio drives IO on both sides. Co-Authored-By: Claude Opus 4.8 --- doc/basics.rst | 23 ++--- pyproject.toml | 8 -- src/execnet/gateway_base.py | 121 ------------------------- src/execnet/multi.py | 2 +- testing/conftest.py | 12 +-- testing/test_channel.py | 2 - testing/test_gateway.py | 30 ------- testing/test_multi.py | 2 +- testing/test_xspec.py | 4 +- uv.lock | 172 +----------------------------------- 10 files changed, 15 insertions(+), 361 deletions(-) diff --git a/doc/basics.rst b/doc/basics.rst index 78672647..723de83f 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -57,10 +57,6 @@ Examples for valid gateway specifications same interpreter as the one it is initiated from and additionally remotely sets an environment variable ``NAME`` to ``value``. -* ``popen//execmodel=eventlet`` specifies a subprocess that uses the - same interpreter as the one it is initiated from but will run the - other side using eventlet for handling IO and dispatching threads. - * ``socket=192.168.1.4:8888`` specifies a Python Socket server process that listens on ``192.168.1.4:8888`` @@ -138,30 +134,29 @@ processes then you often want to call ``group.terminate()`` yourself and specify a larger or not timeout. -threading models: gevent, eventlet, thread, main_thread_only +threading models: thread, main_thread_only ==================================================================== .. versionadded:: 1.2 (status: experimental!) -execnet supports "main_thread_only", "thread", "eventlet" and "gevent" -as thread models on each of the two sides. You need to decide which -model to use before you create any gateways:: +execnet supports "thread" and "main_thread_only" as thread models on +each of the two sides. You need to decide which model to use before +you create any gateways:: # content of threadmodel.py import execnet - # locally use "eventlet", remotely use "thread" model - execnet.set_execmodel("eventlet", "thread") + # locally use "thread", remotely use "main_thread_only" model + execnet.set_execmodel("thread", "main_thread_only") gw = execnet.makegateway() print (gw) print (gw.remote_status()) print (gw.remote_exec("channel.send(1)").receive()) -You need to have eventlet installed in your environment and then -you can execute this little test file:: +You can execute this little test file:: $ python threadmodel.py - - + + 1 How to execute in the main thread diff --git a/pyproject.toml b/pyproject.toml index 709f887b..2c73a32b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,6 @@ testing = [ "tox", "hatch", "uv", - "gevent", ] [dependency-groups] @@ -122,10 +121,3 @@ warn_unused_ignores = false disallow_untyped_calls = false disallow_untyped_defs = false disallow_incomplete_defs = false - -[[tool.mypy.overrides]] -module = [ - "eventlet.*", - "gevent.thread.*", -] -ignore_missing_imports = true diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 8cf3cb09..c9933fe0 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -179,123 +179,6 @@ class MainThreadOnlyExecModel(ThreadExecModel): backend = "main_thread_only" -class EventletExecModel(ExecModel): - backend = "eventlet" - - @property - def queue(self): - import eventlet - - return eventlet.queue - - @property - def subprocess(self): - import eventlet.green.subprocess - - return eventlet.green.subprocess - - @property - def socket(self): - import eventlet.green.socket - - return eventlet.green.socket - - def get_ident(self) -> int: - import eventlet.green.thread - - return eventlet.green.thread.get_ident() # type: ignore[no-any-return] - - def sleep(self, delay: float) -> None: - import eventlet - - eventlet.sleep(delay) - - def start(self, func, args=()) -> None: - import eventlet - - eventlet.spawn_n(func, *args) - - def fdopen(self, fd, mode, bufsize=1, closefd=True): - import eventlet.green.os - - return eventlet.green.os.fdopen(fd, mode, bufsize, closefd=closefd) - - def Lock(self): - import eventlet.green.threading - - return eventlet.green.threading.RLock() - - def RLock(self): - import eventlet.green.threading - - return eventlet.green.threading.RLock() - - def Event(self): - import eventlet.green.threading - - return eventlet.green.threading.Event() - - -class GeventExecModel(ExecModel): - backend = "gevent" - - @property - def queue(self): - import gevent.queue - - return gevent.queue - - @property - def subprocess(self): - import gevent.subprocess - - return gevent.subprocess - - @property - def socket(self): - import gevent - - return gevent.socket - - def get_ident(self) -> int: - import gevent.thread - - return gevent.thread.get_ident() # type: ignore[no-any-return] - - def sleep(self, delay: float) -> None: - import gevent - - gevent.sleep(delay) - - def start(self, func, args=()) -> None: - import gevent - - gevent.spawn(func, *args) - - def fdopen(self, fd, mode, bufsize=1, closefd=True): - import gevent.fileobject - - # Prefer FileObject (FileObjectPosix on Unix). FileObjectThread keeps a - # native threadpool alive and can prevent interpreter shutdown, which - # hangs tests/scripts that open stdio via init_popen_io and then exit. - return gevent.fileobject.FileObject(fd, mode, bufsize, closefd=closefd) - - def Lock(self): - import gevent.lock - - return gevent.lock.RLock() - - def RLock(self): - import gevent.lock - - return gevent.lock.RLock() - - def Event(self): - import gevent.event - - return gevent.event.Event() - - def get_execmodel(backend: str | ExecModel) -> ExecModel: if isinstance(backend, ExecModel): return backend @@ -303,10 +186,6 @@ def get_execmodel(backend: str | ExecModel) -> ExecModel: return ThreadExecModel() elif backend == "main_thread_only": return MainThreadOnlyExecModel() - elif backend == "eventlet": - return EventletExecModel() - elif backend == "gevent": - return GeventExecModel() else: raise ValueError(f"unknown execmodel {backend!r}") diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 8079f4c4..687f1840 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -138,7 +138,7 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: id= specifies the gateway id python= specifies which python interpreter to execute - execmodel=model 'thread', 'main_thread_only', 'eventlet', 'gevent' execution model + execmodel=model 'thread' or 'main_thread_only' execution model chdir= specifies to which directory to change nice= specifies process priority of new process env:NAME=value specifies a remote environment variable setting. diff --git a/testing/conftest.py b/testing/conftest.py index c75f96c7..4d97bf35 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -128,10 +128,6 @@ def anypython(request: pytest.FixtureRequest) -> str: executable = getexecutable(name) if executable is None: pytest.skip(f"no {name} found") - if "execmodel" in request.fixturenames and name != "sys.executable": - backend = request.getfixturevalue("execmodel").backend - if backend not in ("thread", "main_thread_only"): - pytest.xfail(f"cannot run {backend!r} execmodel with bare {name}") return executable @@ -182,14 +178,8 @@ def gw( return gw -@pytest.fixture( - params=["thread", "main_thread_only", "eventlet", "gevent"], scope="session" -) +@pytest.fixture(params=["thread", "main_thread_only"], scope="session") def execmodel(request: pytest.FixtureRequest) -> ExecModel: - if request.param not in ("thread", "main_thread_only"): - pytest.importorskip(request.param) - if request.param in ("eventlet", "gevent") and sys.platform == "win32": - pytest.xfail(request.param + " does not work on win32") return get_execmodel(request.param) diff --git a/testing/test_channel.py b/testing/test_channel.py index d4712277..ba02b57b 100644 --- a/testing/test_channel.py +++ b/testing/test_channel.py @@ -214,8 +214,6 @@ def test_channel_callback_stays_active(self, gw: Gateway) -> None: def check_channel_callback_stays_active( self, gw: Gateway, earlyfree: bool = True ) -> Channel | None: - if gw.spec.execmodel == "gevent": - pytest.xfail("investigate gevent failure") # with 'earlyfree==True', this tests the "sendonly" channel state. l: list[int] = [] channel = gw.remote_exec( diff --git a/testing/test_gateway.py b/testing/test_gateway.py index 634f237a..83a50f44 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -535,36 +535,6 @@ def test_popen_args(spec: str, expected_args: list[str]) -> None: assert args == expected_args -@pytest.mark.parametrize( - "interleave_getstatus", - [ - pytest.param(True, id="interleave-remote-status"), - pytest.param( - False, - id="no-interleave-remote-status", - marks=pytest.mark.xfail( - reason="https://github.com/pytest-dev/execnet/issues/123", - ), - ), - ], -) -def test_regression_gevent_hangs( - group: execnet.Group, interleave_getstatus: bool -) -> None: - pytest.importorskip("gevent") - gw = group.makegateway("popen//execmodel=gevent") - - print(gw.remote_status()) - - def sendback(channel) -> None: - channel.send(1234) - - ch = gw.remote_exec(sendback) - if interleave_getstatus: - print(gw.remote_status()) - assert ch.receive(timeout=0.5) == 1234 - - def test_assert_main_thread_only( execmodel: gateway_base.ExecModel, makegateway: Callable[[str], Gateway] ) -> None: diff --git a/testing/test_multi.py b/testing/test_multi.py index 12e0ed3d..f3166503 100644 --- a/testing/test_multi.py +++ b/testing/test_multi.py @@ -69,7 +69,7 @@ def test_Group_execmodel_setting(self) -> None: gm._gateways.append(1) # type: ignore[arg-type] try: with pytest.raises(ValueError): - gm.set_execmodel("eventlet") + gm.set_execmodel("main_thread_only") assert gm.execmodel.backend == "thread" finally: gm._gateways.pop() diff --git a/testing/test_xspec.py b/testing/test_xspec.py index f837b07a..5240d72d 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -64,8 +64,8 @@ def test_ssh_options(self) -> None: def test_execmodel(self) -> None: spec = XSpec("execmodel=thread") assert spec.execmodel == "thread" - spec = XSpec("execmodel=eventlet") - assert spec.execmodel == "eventlet" + spec = XSpec("execmodel=main_thread_only") + assert spec.execmodel == "main_thread_only" def test_ssh_options_and_config(self) -> None: spec = XSpec("ssh=-p 22100 user@host//python=python3") diff --git a/uv.lock b/uv.lock index fc2da46b..5dbf4f10 100644 --- a/uv.lock +++ b/uv.lock @@ -332,7 +332,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -348,7 +348,6 @@ dependencies = [ [package.optional-dependencies] testing = [ - { name = "gevent" }, { name = "hatch" }, { name = "pre-commit" }, { name = "pytest" }, @@ -364,7 +363,6 @@ testing = [ [package.metadata] requires-dist = [ - { name = "gevent", marker = "extra == 'testing'" }, { name = "hatch", marker = "extra == 'testing'" }, { name = "pre-commit", marker = "extra == 'testing'" }, { name = "pytest", marker = "extra == 'testing'", specifier = ">8.0" }, @@ -387,118 +385,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/06/79/b4c714bef36bc4ec2beeae1e0c124f0223888cd8c6feb1cdc56038116920/filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3", size = 97732, upload-time = "2026-07-21T13:17:41.55Z" }, ] -[[package]] -name = "gevent" -version = "26.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, - { name = "greenlet", marker = "platform_python_implementation == 'CPython'" }, - { name = "zope-event" }, - { name = "zope-interface" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c4/cb/98aa3a299e2fc4a2372b5d124863e02965b64579ffc29fe54d0641e65b2f/gevent-26.5.0.tar.gz", hash = "sha256:1655eb04c1e20d71b2aa4a3c7528162dd58ff6cc46a037af1f01f534c80fefba", size = 6712354, upload-time = "2026-05-20T21:22:45.132Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/72/b7/01a5880e01702f39fb09e3616c624054a0dc9a82561a865f3b1eff4bfc80/gevent-26.5.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:2ba673dcbf7747513b58fa64ca7e9d6a828bc5c604d1552d23db89006d7911df", size = 2181491, upload-time = "2026-05-20T20:35:19.326Z" }, - { url = "https://files.pythonhosted.org/packages/8d/fe/035ec5fa58a886740a744380118f03a90ac2da3f6c9cba248f28074ce40a/gevent-26.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:271b1474d81bb33036631adb16a35e5a1ee9dc414b05c999d6b01dc839a89975", size = 2212161, upload-time = "2026-05-20T20:43:25.678Z" }, - { url = "https://files.pythonhosted.org/packages/2a/ea/ea87c08931c9e4c6c40bb05a2cb19c2d6f93fe6e0052f9152ea5ade6d037/gevent-26.5.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:cd3dc60581687e2618286108f8e2f820d8446be4b34131065011c066e911d39c", size = 1768295, upload-time = "2026-05-20T21:17:29.438Z" }, - { url = "https://files.pythonhosted.org/packages/da/92/1d0e7287ae55700a8d25153ac736896bd9bcc3f85a12d374ef398db4b33c/gevent-26.5.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:dc7fa28b2d627f8e87595f39043b6dec71e8e7fb97e685e5506c47cf3ff8cb2e", size = 1862627, upload-time = "2026-05-20T21:15:59.365Z" }, - { url = "https://files.pythonhosted.org/packages/3a/4c/7f5ed67e52dfdef4ff91ae1a6fb28186d52e2496962edc8f17bdea9ab2c0/gevent-26.5.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:68c5fc21cef80268cdff88a4ae6c025fabb019b071f6f8ee4d20a7bccbddb873", size = 1804690, upload-time = "2026-05-20T21:30:51.713Z" }, - { url = "https://files.pythonhosted.org/packages/4c/75/0f5da6ca045f8a052203e1810058029f4b682507a789b413cac7d28bae28/gevent-26.5.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d325502eb0695708ef8c899f605573ed6847f3961f8159627dba267fbf3ce457", size = 2119054, upload-time = "2026-05-20T20:35:22.678Z" }, - { url = "https://files.pythonhosted.org/packages/6e/f1/fcff7f7fad2bb33f3742db6b2145825a2191c0cd31d75789b0741fd28faf/gevent-26.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a11daf3a588b932c8bf965fb18444c69aff48badec88435e988cf8d67137075a", size = 1778784, upload-time = "2026-05-20T21:16:38.182Z" }, - { url = "https://files.pythonhosted.org/packages/98/57/151314f00bdc6ba77333febb3e9dc97fdf94d79426559b4fa8332f0c2b6e/gevent-26.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1101b5ef82a3fb178550cfd80f32293dc8dd2f3d0828292223ebba29d6f76e33", size = 2145373, upload-time = "2026-05-20T20:43:27.255Z" }, - { url = "https://files.pythonhosted.org/packages/1a/b5/7a02f711db62cbed1c1a00e1f9ff50eef95ccc78d4c04a0f93636655d1b7/gevent-26.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:5233109ad4f3af16393ba9888f238919a05ce15ce68d6831ac8a0da8dfb750ae", size = 1696576, upload-time = "2026-05-20T20:15:49.62Z" }, - { url = "https://files.pythonhosted.org/packages/f8/9b/5022adc310697ef25c6fb22eb9bf0ebcad3427b51776e882709de9a8b6d7/gevent-26.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:3be804565168ffacebeb21af9f1cd689831a89f0f12fc0c3f423c730c3c9eb31", size = 1552095, upload-time = "2026-05-20T20:16:54.81Z" }, - { url = "https://files.pythonhosted.org/packages/37/0b/1a530b2db55c97cc0cf44116201f538f3033c04c1d2aca143979b412f4be/gevent-26.5.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:e80ad2a8a1e8bdaa5605e3bf4929e0cebf9ea7b8237c83362f7257698bb14280", size = 2929714, upload-time = "2026-05-20T20:13:24.656Z" }, - { url = "https://files.pythonhosted.org/packages/b9/df/32fe851ed5f68493f354e09b19bdebae0de1185be4db0b2988e71e737fd3/gevent-26.5.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fe42c037253580a3386fce275f8a2a845e540f5a729916934a732f13d42e72cc", size = 1784838, upload-time = "2026-05-20T21:17:31.063Z" }, - { url = "https://files.pythonhosted.org/packages/e5/9a/21332674f9a10e8cdf13b41b52e9d663647a1c6e1dc3c62b07c0aeefd360/gevent-26.5.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:9f463c7d6f69d13b6fe8e3b832a6175a6e95328a940f38495d25496d1ae8ad88", size = 1880440, upload-time = "2026-05-20T21:16:00.881Z" }, - { url = "https://files.pythonhosted.org/packages/9f/b1/5f8a4196113cf7f3fdd987b483f7e6b10c28ea3930c4727e31ba8cce51b6/gevent-26.5.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:96d5e96b1b14a4c1023dcfcc114533217f13febc3b6169254f23fc18d19fee29", size = 1831592, upload-time = "2026-05-20T21:30:53.832Z" }, - { url = "https://files.pythonhosted.org/packages/4e/69/1559b1f6b5107a9118fccd300240879bd581b6d87b03d568d0d155ea702c/gevent-26.5.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:bccff69c462e3650a0fd1d4e9cfc8b6effe15f3e9b1cad20a7bb5ce14b057efd", size = 2114915, upload-time = "2026-05-20T20:35:25.041Z" }, - { url = "https://files.pythonhosted.org/packages/e4/32/602c499d54472f64e5cdf6013aeab5ce6aa6fed005387e8b4f2d22f5dc8d/gevent-26.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f519139354d5ca7625df9ddb1b2ffada885c14abc5b4dbae3682e967ddf79669", size = 1796906, upload-time = "2026-05-20T21:16:39.65Z" }, - { url = "https://files.pythonhosted.org/packages/f9/3c/2fe77ee6e3d381b3c50c0b7d6c4c08c08b8ff5e8c0d9dd51a3b426d61eec/gevent-26.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0bf57df54f1c66273bf3601c2a1e41b12138fe848933718369663bc54f177ca2", size = 2140806, upload-time = "2026-05-20T20:43:28.895Z" }, - { url = "https://files.pythonhosted.org/packages/22/d5/4620797bbd9c88f4541188efc138b0d615f9834db540da36a2249ee929c5/gevent-26.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:e49ce0de007dfd7412edbc2b5d41cce33b049bb1b7086f50be5a09e601bde603", size = 1699995, upload-time = "2026-05-20T20:15:39.311Z" }, - { url = "https://files.pythonhosted.org/packages/cb/83/ac3477dfc0f9fd80c88110102c73cefc35dcded2b248544f45a8fa5412df/gevent-26.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:5c5ff29495a2eed2a244de8150f21893d6c1b15d8b4b5719ab4bbfa06db1e28f", size = 1547433, upload-time = "2026-05-20T20:15:51.656Z" }, - { url = "https://files.pythonhosted.org/packages/7d/47/5b992ab9c8037633cfd0fe698a97a878f59d8eb53c381e91e9a1a76fd215/gevent-26.5.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:9b4d3f34c913d1a6bec6d030365a517f3b527a9773b12e58cf56c3339bbe96e6", size = 2952523, upload-time = "2026-05-20T20:13:04.698Z" }, - { url = "https://files.pythonhosted.org/packages/74/11/c7dfc773eb43331a682efed610b49df6e976331f1b0e1c592a0c35d29872/gevent-26.5.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1d8da4e799431feeb4c9e441ac7431f0baabb9106976790d884289d08ac08359", size = 1787044, upload-time = "2026-05-20T21:17:32.845Z" }, - { url = "https://files.pythonhosted.org/packages/ae/28/9812933dac93560f46910a9e834805fe76f822c408bd1c20cdf299d7c311/gevent-26.5.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:51becdb4c30a8f45c1c028ad7a97bf5a1ed141f74b159a31aa9cc6aa1e6263a6", size = 1882342, upload-time = "2026-05-20T21:16:02.645Z" }, - { url = "https://files.pythonhosted.org/packages/96/4b/514f248f69b2230b69b0bb17f4158b0b05dd4b2cb469a60ab206e9fe7496/gevent-26.5.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:c42bbcd3d453b08ad8915fd3feaf3d44a3562cdf1c7b208f9837149711e16d9d", size = 1834136, upload-time = "2026-05-20T21:30:55.739Z" }, - { url = "https://files.pythonhosted.org/packages/53/67/f5f30716efca99b6200ae89a9303a7e94dae085b7de6f6d0033c52a37f4b/gevent-26.5.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:bd3445e4fbeeb46690ed8efe94b8d1d46b14aa04af8866ae7a8da5997828d1c6", size = 2115349, upload-time = "2026-05-20T20:35:28.132Z" }, - { url = "https://files.pythonhosted.org/packages/09/d8/60e8809bde7986e6c4e6d106080b3603fa09b3bb0255fed1a4d8282e3ca2/gevent-26.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b573d5b2826edc705f31f07da6889ad483a6a0d64944ebd8d32205f7c5bf46fb", size = 1799443, upload-time = "2026-05-20T21:16:41.928Z" }, - { url = "https://files.pythonhosted.org/packages/f8/41/b388b2b1f0a026ea30687e51ddf81dbb783dfb55fac0a16708d2821d99e5/gevent-26.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4d53b1b28f2082a151bded2850b53f6baed02f742d2a1584029e8bd42d457fb4", size = 2141117, upload-time = "2026-05-20T20:43:30.694Z" }, - { url = "https://files.pythonhosted.org/packages/b1/f3/ac9a4b0de487e390c5d53a908a9347c0df0102de2bbf3e8603087769191d/gevent-26.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:23569ce0c254eb821fc3dcfe250843dde8b3180b09bae9e222e41aa3fa4885b7", size = 1699862, upload-time = "2026-05-20T20:15:33.642Z" }, - { url = "https://files.pythonhosted.org/packages/2a/cf/1ef1fc9b390563c0f97702f94a557d1649b7bbb5724f9b86c2122747e92f/gevent-26.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:40cdcdb2e404b6c82b82a4576bdb33958f23fc2deb0d933e9e022b362001e647", size = 1545341, upload-time = "2026-05-20T20:16:26.229Z" }, - { url = "https://files.pythonhosted.org/packages/17/55/7d98d3888e7bb9ad4656420dec69232ecbbea48792aff9295d0ad7cf8435/gevent-26.5.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:75a0050e4b87f08ddee7e56f59e6014cd7fcdc3153046c09a847940515d12c85", size = 2968223, upload-time = "2026-05-20T20:13:17.223Z" }, - { url = "https://files.pythonhosted.org/packages/f8/b4/e8e116fcbcb9dc0bf3acc50037f86e1204c217c8ed5defde68be11b3aab6/gevent-26.5.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:fd1a0b83a04e19378d9466ae0ee2b5937cf1d7fbfdcb916b2aea82179a208574", size = 1793926, upload-time = "2026-05-20T21:17:34.321Z" }, - { url = "https://files.pythonhosted.org/packages/28/07/7b267e9754b661defb93542e97731a4df21f8a40dc0f6c853faa717cf124/gevent-26.5.0-cp314-cp314-manylinux_2_28_ppc64le.whl", hash = "sha256:4c964c15076e76391d523ec24202f579a2535f7e301a40efb1656ae046d3eb69", size = 1887632, upload-time = "2026-05-20T21:16:04.158Z" }, - { url = "https://files.pythonhosted.org/packages/5c/50/b47d29e99449bd13b557ffa451401dc13d397a9923f562ef90a4e8514502/gevent-26.5.0-cp314-cp314-manylinux_2_28_s390x.whl", hash = "sha256:45d5438d1c84da5df7e832434627624709543630977332bb4e2d05ecca362cc9", size = 1838688, upload-time = "2026-05-20T21:30:57.979Z" }, - { url = "https://files.pythonhosted.org/packages/8b/eb/5b54ccff11bc7d7bebd40a24571ccc115d5cdae4f6c32ab457b43b436e42/gevent-26.5.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:354f35924113abc954819216c2a6ee16751958c615681e0490946e31b437bd2f", size = 2120351, upload-time = "2026-05-20T20:35:32.699Z" }, - { url = "https://files.pythonhosted.org/packages/9c/70/30fd325c30e04b1e5174c61945e17421d53ddb2450366cc52cef234f8c4b/gevent-26.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a47cd2d32f6404212d374ad8014a3491d7477dcf0cc09c5a2308ad6d325fd663", size = 1806684, upload-time = "2026-05-20T21:16:43.87Z" }, - { url = "https://files.pythonhosted.org/packages/cd/e8/fbf911ac3f9524ecfaed174d100fde671904ab8db92ceaf07faaebd13386/gevent-26.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:032157cebdedb84f2f52cdd980f2f5f2623eed6a8f083aadf44b44c47f628642", size = 2146606, upload-time = "2026-05-20T20:43:32.216Z" }, - { url = "https://files.pythonhosted.org/packages/9e/4d/284fcbbfde66fd978c2980c1fbe0eabd586af6e4b728649e9cf459e8b38f/gevent-26.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:9c414935ba5fc88359110968851d3616f119082c937390d00a1c0f4f59be814f", size = 1722497, upload-time = "2026-05-20T20:16:44.274Z" }, - { url = "https://files.pythonhosted.org/packages/15/d2/9f66eb53434704402be0ba733bf3320bf589671a4b76fac52a7d6077e972/gevent-26.5.0-cp314-cp314-win_arm64.whl", hash = "sha256:2a0f5993a04b95a35b3a118b1a58ba272833f9b547b774001dea29f90620882f", size = 1574249, upload-time = "2026-05-20T20:15:50.873Z" }, - { url = "https://files.pythonhosted.org/packages/dc/d5/b4c50adb761878e3c96642b9f79bf44cee3120f3df55cd40876f51d89866/gevent-26.5.0-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:2e117df896a2660c9ebd4e2b5afc02dfd6e2ddf9b495e787e67c72d105432b09", size = 2971993, upload-time = "2026-05-20T20:12:50.845Z" }, - { url = "https://files.pythonhosted.org/packages/03/83/71c2a945e80198422d1d93dbe67355f249fb456b451bf9201199d3ef6a1a/gevent-26.5.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:af5ffe9c11ffb8a39b6bef2e8b722aa2043ae4980977915c6aa8c68b4bc26e46", size = 1796658, upload-time = "2026-05-20T21:17:35.968Z" }, - { url = "https://files.pythonhosted.org/packages/42/96/548ca77aed5cb9a44e855a6c23ebceeb3554a0ea9ca0c01c311878899a3e/gevent-26.5.0-cp315-cp315-manylinux_2_28_ppc64le.whl", hash = "sha256:7da34aef7e87c43dd3662e5785e79ed505c01399a7cb42876d2d8925969fd75f", size = 1891473, upload-time = "2026-05-20T21:16:05.657Z" }, - { url = "https://files.pythonhosted.org/packages/f6/4f/f48bd47d5287afb0fbcc56165f3ed47583f1803bad401653fe27e71ade2d/gevent-26.5.0-cp315-cp315-manylinux_2_28_s390x.whl", hash = "sha256:1c6293a7046bcc6f3d8972a74b19cd7a4cfd02d3881edf0fcf827aa514bd247b", size = 1841429, upload-time = "2026-05-20T21:30:59.907Z" }, - { url = "https://files.pythonhosted.org/packages/a0/72/1925215fc720d2561fa3ec8d4af5f098f8d0cbfa76a45fafed6e5ade7718/gevent-26.5.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:d3bde0f140a275b2fa88e4b6516bda85551930e10bc2fd95e18c1b7d11cb780c", size = 2123895, upload-time = "2026-05-20T20:35:34.964Z" }, - { url = "https://files.pythonhosted.org/packages/83/59/0f584f6b1170c9a6abd9b70ccf5e9cc5ead34eabafabc0e21876ef0fe6f7/gevent-26.5.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:e29fb4b17d9958ec8cb7f6339a111b29bc23f2c2efbef86189d1248bb4862d17", size = 1809047, upload-time = "2026-05-20T21:16:45.977Z" }, - { url = "https://files.pythonhosted.org/packages/82/88/61e854bfd98ac22eac78a97fc6db10de0f9ace46514072b435c217168729/gevent-26.5.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:b2239df2f7570efa03736678f3f053bb1bdd22a8a16cd28a2feb7d32ea5f533f", size = 2150764, upload-time = "2026-05-20T20:43:33.781Z" }, - { url = "https://files.pythonhosted.org/packages/b9/f5/af048b97433d7f9a7df7f5510b2c46918b7d073dcfb3bf6d0ef0e5a83dcc/gevent-26.5.0-cp315-cp315-win_amd64.whl", hash = "sha256:aae214952fd38d27a42dc416bb70193962ec932384b63445d29bbb5817a1c042", size = 1722600, upload-time = "2026-05-20T20:19:56.81Z" }, - { url = "https://files.pythonhosted.org/packages/11/95/fb74a2299c6a2d78d9de12deaaac640ab5d2ef96a8e0f97a3ff84b9ca84b/gevent-26.5.0-cp315-cp315-win_arm64.whl", hash = "sha256:f7067564f139e33bf26a31ee3b13d168d76eb99a44b85ced626652b158baa80c", size = 1574406, upload-time = "2026-05-20T20:17:12.125Z" }, -] - -[[package]] -name = "greenlet" -version = "3.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/34/c1/a82edae11d46c0d83481aacaa1e578fea21d94a1ef400afd734d47ad95ad/greenlet-3.2.2.tar.gz", hash = "sha256:ad053d34421a2debba45aa3cc39acf454acbcd025b3fc1a9f8a0dee237abd485", size = 185797, upload-time = "2025-05-09T19:47:35.066Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/05/66/910217271189cc3f32f670040235f4bf026ded8ca07270667d69c06e7324/greenlet-3.2.2-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:c49e9f7c6f625507ed83a7485366b46cbe325717c60837f7244fc99ba16ba9d6", size = 267395, upload-time = "2025-05-09T14:50:45.357Z" }, - { url = "https://files.pythonhosted.org/packages/a8/36/8d812402ca21017c82880f399309afadb78a0aa300a9b45d741e4df5d954/greenlet-3.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3cc1a3ed00ecfea8932477f729a9f616ad7347a5e55d50929efa50a86cb7be7", size = 625742, upload-time = "2025-05-09T15:23:58.293Z" }, - { url = "https://files.pythonhosted.org/packages/7b/77/66d7b59dfb7cc1102b2f880bc61cb165ee8998c9ec13c96606ba37e54c77/greenlet-3.2.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c9896249fbef2c615853b890ee854f22c671560226c9221cfd27c995db97e5c", size = 637014, upload-time = "2025-05-09T15:24:47.025Z" }, - { url = "https://files.pythonhosted.org/packages/36/a7/ff0d408f8086a0d9a5aac47fa1b33a040a9fca89bd5a3f7b54d1cd6e2793/greenlet-3.2.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7409796591d879425997a518138889d8d17e63ada7c99edc0d7a1c22007d4907", size = 632874, upload-time = "2025-05-09T15:29:20.014Z" }, - { url = "https://files.pythonhosted.org/packages/a1/75/1dc2603bf8184da9ebe69200849c53c3c1dca5b3a3d44d9f5ca06a930550/greenlet-3.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7791dcb496ec53d60c7f1c78eaa156c21f402dda38542a00afc3e20cae0f480f", size = 631652, upload-time = "2025-05-09T14:53:30.961Z" }, - { url = "https://files.pythonhosted.org/packages/7b/74/ddc8c3bd4c2c20548e5bf2b1d2e312a717d44e2eca3eadcfc207b5f5ad80/greenlet-3.2.2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d8009ae46259e31bc73dc183e402f548e980c96f33a6ef58cc2e7865db012e13", size = 580619, upload-time = "2025-05-09T14:53:42.049Z" }, - { url = "https://files.pythonhosted.org/packages/7e/f2/40f26d7b3077b1c7ae7318a4de1f8ffc1d8ccbad8f1d8979bf5080250fd6/greenlet-3.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fd9fb7c941280e2c837b603850efc93c999ae58aae2b40765ed682a6907ebbc5", size = 1109809, upload-time = "2025-05-09T15:26:59.063Z" }, - { url = "https://files.pythonhosted.org/packages/c5/21/9329e8c276746b0d2318b696606753f5e7b72d478adcf4ad9a975521ea5f/greenlet-3.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:00cd814b8959b95a546e47e8d589610534cfb71f19802ea8a2ad99d95d702057", size = 1133455, upload-time = "2025-05-09T14:53:55.823Z" }, - { url = "https://files.pythonhosted.org/packages/bb/1e/0dca9619dbd736d6981f12f946a497ec21a0ea27262f563bca5729662d4d/greenlet-3.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:d0cb7d47199001de7658c213419358aa8937df767936506db0db7ce1a71f4a2f", size = 294991, upload-time = "2025-05-09T15:05:56.847Z" }, - { url = "https://files.pythonhosted.org/packages/a3/9f/a47e19261747b562ce88219e5ed8c859d42c6e01e73da6fbfa3f08a7be13/greenlet-3.2.2-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:dcb9cebbf3f62cb1e5afacae90761ccce0effb3adaa32339a0670fe7805d8068", size = 268635, upload-time = "2025-05-09T14:50:39.007Z" }, - { url = "https://files.pythonhosted.org/packages/11/80/a0042b91b66975f82a914d515e81c1944a3023f2ce1ed7a9b22e10b46919/greenlet-3.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf3fc9145141250907730886b031681dfcc0de1c158f3cc51c092223c0f381ce", size = 628786, upload-time = "2025-05-09T15:24:00.692Z" }, - { url = "https://files.pythonhosted.org/packages/38/a2/8336bf1e691013f72a6ebab55da04db81a11f68e82bb691f434909fa1327/greenlet-3.2.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:efcdfb9df109e8a3b475c016f60438fcd4be68cd13a365d42b35914cdab4bb2b", size = 640866, upload-time = "2025-05-09T15:24:48.153Z" }, - { url = "https://files.pythonhosted.org/packages/f8/7e/f2a3a13e424670a5d08826dab7468fa5e403e0fbe0b5f951ff1bc4425b45/greenlet-3.2.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4bd139e4943547ce3a56ef4b8b1b9479f9e40bb47e72cc906f0f66b9d0d5cab3", size = 636752, upload-time = "2025-05-09T15:29:23.182Z" }, - { url = "https://files.pythonhosted.org/packages/fd/5d/ce4a03a36d956dcc29b761283f084eb4a3863401c7cb505f113f73af8774/greenlet-3.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:71566302219b17ca354eb274dfd29b8da3c268e41b646f330e324e3967546a74", size = 636028, upload-time = "2025-05-09T14:53:32.854Z" }, - { url = "https://files.pythonhosted.org/packages/4b/29/b130946b57e3ceb039238413790dd3793c5e7b8e14a54968de1fe449a7cf/greenlet-3.2.2-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3091bc45e6b0c73f225374fefa1536cd91b1e987377b12ef5b19129b07d93ebe", size = 583869, upload-time = "2025-05-09T14:53:43.614Z" }, - { url = "https://files.pythonhosted.org/packages/ac/30/9f538dfe7f87b90ecc75e589d20cbd71635531a617a336c386d775725a8b/greenlet-3.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:44671c29da26539a5f142257eaba5110f71887c24d40df3ac87f1117df589e0e", size = 1112886, upload-time = "2025-05-09T15:27:01.304Z" }, - { url = "https://files.pythonhosted.org/packages/be/92/4b7deeb1a1e9c32c1b59fdca1cac3175731c23311ddca2ea28a8b6ada91c/greenlet-3.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c23ea227847c9dbe0b3910f5c0dd95658b607137614eb821e6cbaecd60d81cc6", size = 1138355, upload-time = "2025-05-09T14:53:58.011Z" }, - { url = "https://files.pythonhosted.org/packages/c5/eb/7551c751a2ea6498907b2fcbe31d7a54b602ba5e8eb9550a9695ca25d25c/greenlet-3.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:0a16fb934fcabfdfacf21d79e6fed81809d8cd97bc1be9d9c89f0e4567143d7b", size = 295437, upload-time = "2025-05-09T15:00:57.733Z" }, - { url = "https://files.pythonhosted.org/packages/2c/a1/88fdc6ce0df6ad361a30ed78d24c86ea32acb2b563f33e39e927b1da9ea0/greenlet-3.2.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:df4d1509efd4977e6a844ac96d8be0b9e5aa5d5c77aa27ca9f4d3f92d3fcf330", size = 270413, upload-time = "2025-05-09T14:51:32.455Z" }, - { url = "https://files.pythonhosted.org/packages/a6/2e/6c1caffd65490c68cd9bcec8cb7feb8ac7b27d38ba1fea121fdc1f2331dc/greenlet-3.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da956d534a6d1b9841f95ad0f18ace637668f680b1339ca4dcfb2c1837880a0b", size = 637242, upload-time = "2025-05-09T15:24:02.63Z" }, - { url = "https://files.pythonhosted.org/packages/98/28/088af2cedf8823b6b7ab029a5626302af4ca1037cf8b998bed3a8d3cb9e2/greenlet-3.2.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9c7b15fb9b88d9ee07e076f5a683027bc3befd5bb5d25954bb633c385d8b737e", size = 651444, upload-time = "2025-05-09T15:24:49.856Z" }, - { url = "https://files.pythonhosted.org/packages/4a/9f/0116ab876bb0bc7a81eadc21c3f02cd6100dcd25a1cf2a085a130a63a26a/greenlet-3.2.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:752f0e79785e11180ebd2e726c8a88109ded3e2301d40abced2543aa5d164275", size = 646067, upload-time = "2025-05-09T15:29:24.989Z" }, - { url = "https://files.pythonhosted.org/packages/35/17/bb8f9c9580e28a94a9575da847c257953d5eb6e39ca888239183320c1c28/greenlet-3.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ae572c996ae4b5e122331e12bbb971ea49c08cc7c232d1bd43150800a2d6c65", size = 648153, upload-time = "2025-05-09T14:53:34.716Z" }, - { url = "https://files.pythonhosted.org/packages/2c/ee/7f31b6f7021b8df6f7203b53b9cc741b939a2591dcc6d899d8042fcf66f2/greenlet-3.2.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02f5972ff02c9cf615357c17ab713737cccfd0eaf69b951084a9fd43f39833d3", size = 603865, upload-time = "2025-05-09T14:53:45.738Z" }, - { url = "https://files.pythonhosted.org/packages/b5/2d/759fa59323b521c6f223276a4fc3d3719475dc9ae4c44c2fe7fc750f8de0/greenlet-3.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4fefc7aa68b34b9224490dfda2e70ccf2131368493add64b4ef2d372955c207e", size = 1119575, upload-time = "2025-05-09T15:27:04.248Z" }, - { url = "https://files.pythonhosted.org/packages/30/05/356813470060bce0e81c3df63ab8cd1967c1ff6f5189760c1a4734d405ba/greenlet-3.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a31ead8411a027c2c4759113cf2bd473690517494f3d6e4bf67064589afcd3c5", size = 1147460, upload-time = "2025-05-09T14:54:00.315Z" }, - { url = "https://files.pythonhosted.org/packages/07/f4/b2a26a309a04fb844c7406a4501331b9400e1dd7dd64d3450472fd47d2e1/greenlet-3.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:b24c7844c0a0afc3ccbeb0b807adeefb7eff2b5599229ecedddcfeb0ef333bec", size = 296239, upload-time = "2025-05-09T14:57:17.633Z" }, - { url = "https://files.pythonhosted.org/packages/89/30/97b49779fff8601af20972a62cc4af0c497c1504dfbb3e93be218e093f21/greenlet-3.2.2-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:3ab7194ee290302ca15449f601036007873028712e92ca15fc76597a0aeb4c59", size = 269150, upload-time = "2025-05-09T14:50:30.784Z" }, - { url = "https://files.pythonhosted.org/packages/21/30/877245def4220f684bc2e01df1c2e782c164e84b32e07373992f14a2d107/greenlet-3.2.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dc5c43bb65ec3669452af0ab10729e8fdc17f87a1f2ad7ec65d4aaaefabf6bf", size = 637381, upload-time = "2025-05-09T15:24:12.893Z" }, - { url = "https://files.pythonhosted.org/packages/8e/16/adf937908e1f913856b5371c1d8bdaef5f58f251d714085abeea73ecc471/greenlet-3.2.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:decb0658ec19e5c1f519faa9a160c0fc85a41a7e6654b3ce1b44b939f8bf1325", size = 651427, upload-time = "2025-05-09T15:24:51.074Z" }, - { url = "https://files.pythonhosted.org/packages/ad/49/6d79f58fa695b618654adac64e56aff2eeb13344dc28259af8f505662bb1/greenlet-3.2.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6fadd183186db360b61cb34e81117a096bff91c072929cd1b529eb20dd46e6c5", size = 645795, upload-time = "2025-05-09T15:29:26.673Z" }, - { url = "https://files.pythonhosted.org/packages/5a/e6/28ed5cb929c6b2f001e96b1d0698c622976cd8f1e41fe7ebc047fa7c6dd4/greenlet-3.2.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1919cbdc1c53ef739c94cf2985056bcc0838c1f217b57647cbf4578576c63825", size = 648398, upload-time = "2025-05-09T14:53:36.61Z" }, - { url = "https://files.pythonhosted.org/packages/9d/70/b200194e25ae86bc57077f695b6cc47ee3118becf54130c5514456cf8dac/greenlet-3.2.2-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3885f85b61798f4192d544aac7b25a04ece5fe2704670b4ab73c2d2c14ab740d", size = 606795, upload-time = "2025-05-09T14:53:47.039Z" }, - { url = "https://files.pythonhosted.org/packages/f8/c8/ba1def67513a941154ed8f9477ae6e5a03f645be6b507d3930f72ed508d3/greenlet-3.2.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:85f3e248507125bf4af607a26fd6cb8578776197bd4b66e35229cdf5acf1dfbf", size = 1117976, upload-time = "2025-05-09T15:27:06.542Z" }, - { url = "https://files.pythonhosted.org/packages/c3/30/d0e88c1cfcc1b3331d63c2b54a0a3a4a950ef202fb8b92e772ca714a9221/greenlet-3.2.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:1e76106b6fc55fa3d6fe1c527f95ee65e324a13b62e243f77b48317346559708", size = 1145509, upload-time = "2025-05-09T14:54:02.223Z" }, - { url = "https://files.pythonhosted.org/packages/90/2e/59d6491834b6e289051b252cf4776d16da51c7c6ca6a87ff97e3a50aa0cd/greenlet-3.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:fe46d4f8e94e637634d54477b0cfabcf93c53f29eedcbdeecaf2af32029b4421", size = 296023, upload-time = "2025-05-09T14:53:24.157Z" }, - { url = "https://files.pythonhosted.org/packages/65/66/8a73aace5a5335a1cba56d0da71b7bd93e450f17d372c5b7c5fa547557e9/greenlet-3.2.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba30e88607fb6990544d84caf3c706c4b48f629e18853fc6a646f82db9629418", size = 629911, upload-time = "2025-05-09T15:24:22.376Z" }, - { url = "https://files.pythonhosted.org/packages/48/08/c8b8ebac4e0c95dcc68ec99198842e7db53eda4ab3fb0a4e785690883991/greenlet-3.2.2-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:055916fafad3e3388d27dd68517478933a97edc2fc54ae79d3bec827de2c64c4", size = 635251, upload-time = "2025-05-09T15:24:52.205Z" }, - { url = "https://files.pythonhosted.org/packages/37/26/7db30868f73e86b9125264d2959acabea132b444b88185ba5c462cb8e571/greenlet-3.2.2-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2593283bf81ca37d27d110956b79e8723f9aa50c4bcdc29d3c0543d4743d2763", size = 632620, upload-time = "2025-05-09T15:29:28.051Z" }, - { url = "https://files.pythonhosted.org/packages/10/ec/718a3bd56249e729016b0b69bee4adea0dfccf6ca43d147ef3b21edbca16/greenlet-3.2.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89c69e9a10670eb7a66b8cef6354c24671ba241f46152dd3eed447f79c29fb5b", size = 628851, upload-time = "2025-05-09T14:53:38.472Z" }, - { url = "https://files.pythonhosted.org/packages/9b/9d/d1c79286a76bc62ccdc1387291464af16a4204ea717f24e77b0acd623b99/greenlet-3.2.2-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02a98600899ca1ca5d3a2590974c9e3ec259503b2d6ba6527605fcd74e08e207", size = 593718, upload-time = "2025-05-09T14:53:48.313Z" }, - { url = "https://files.pythonhosted.org/packages/cd/41/96ba2bf948f67b245784cd294b84e3d17933597dffd3acdb367a210d1949/greenlet-3.2.2-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:b50a8c5c162469c3209e5ec92ee4f95c8231b11db6a04db09bbe338176723bb8", size = 1105752, upload-time = "2025-05-09T15:27:08.217Z" }, - { url = "https://files.pythonhosted.org/packages/68/3b/3b97f9d33c1f2eb081759da62bd6162159db260f602f048bc2f36b4c453e/greenlet-3.2.2-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:45f9f4853fb4cc46783085261c9ec4706628f3b57de3e68bae03e8f8b3c0de51", size = 1125170, upload-time = "2025-05-09T14:54:04.082Z" }, - { url = "https://files.pythonhosted.org/packages/31/df/b7d17d66c8d0f578d2885a3d8f565e9e4725eacc9d3fdc946d0031c055c4/greenlet-3.2.2-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:9ea5231428af34226c05f927e16fc7f6fa5e39e3ad3cd24ffa48ba53a47f4240", size = 269899, upload-time = "2025-05-09T14:54:01.581Z" }, -] - [[package]] name = "h11" version = "0.16.0" @@ -984,15 +870,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/24/b4293291fa1dd830f353d2cb163295742fa87f179fcc8a20a306a81978b7/SecretStorage-3.3.3-py3-none-any.whl", hash = "sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99", size = 15221, upload-time = "2022-08-13T16:22:44.457Z" }, ] -[[package]] -name = "setuptools" -version = "80.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8d/d2/ec1acaaff45caed5c2dedb33b67055ba9d4e96b091094df90762e60135fe/setuptools-80.8.0.tar.gz", hash = "sha256:49f7af965996f26d43c8ae34539c8d99c5042fbff34302ea151eaa9c207cd257", size = 1319720, upload-time = "2025-05-20T14:02:53.503Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/58/29/93c53c098d301132196c3238c312825324740851d77a8500a2462c0fd888/setuptools-80.8.0-py3-none-any.whl", hash = "sha256:95a60484590d24103af13b686121328cc2736bee85de8936383111e421b9edc0", size = 1201470, upload-time = "2025-05-20T14:02:51.348Z" }, -] - [[package]] name = "shellingham" version = "1.5.4" @@ -1222,50 +1099,3 @@ sdist = { url = "https://files.pythonhosted.org/packages/3f/50/bad581df71744867e wheels = [ { url = "https://files.pythonhosted.org/packages/b7/1a/7e4798e9339adc931158c9d69ecc34f5e6791489d469f5e50ec15e35f458/zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931", size = 9630, upload-time = "2024-11-10T15:05:19.275Z" }, ] - -[[package]] -name = "zope-event" -version = "5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/46/c2/427f1867bb96555d1d34342f1dd97f8c420966ab564d58d18469a1db8736/zope.event-5.0.tar.gz", hash = "sha256:bac440d8d9891b4068e2b5a2c5e2c9765a9df762944bda6955f96bb9b91e67cd", size = 17350, upload-time = "2023-06-23T06:28:35.709Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/42/f8dbc2b9ad59e927940325a22d6d3931d630c3644dae7e2369ef5d9ba230/zope.event-5.0-py3-none-any.whl", hash = "sha256:2832e95014f4db26c47a13fdaef84cef2f4df37e66b59d8f1f4a8f319a632c26", size = 6824, upload-time = "2023-06-23T06:28:32.652Z" }, -] - -[[package]] -name = "zope-interface" -version = "7.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "setuptools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/30/93/9210e7606be57a2dfc6277ac97dcc864fd8d39f142ca194fdc186d596fda/zope.interface-7.2.tar.gz", hash = "sha256:8b49f1a3d1ee4cdaf5b32d2e738362c7f5e40ac8b46dd7d1a65e82a4872728fe", size = 252960, upload-time = "2024-11-28T08:45:39.224Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/76/71/e6177f390e8daa7e75378505c5ab974e0bf59c1d3b19155638c7afbf4b2d/zope.interface-7.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ce290e62229964715f1011c3dbeab7a4a1e4971fd6f31324c4519464473ef9f2", size = 208243, upload-time = "2024-11-28T08:47:29.781Z" }, - { url = "https://files.pythonhosted.org/packages/52/db/7e5f4226bef540f6d55acfd95cd105782bc6ee044d9b5587ce2c95558a5e/zope.interface-7.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:05b910a5afe03256b58ab2ba6288960a2892dfeef01336dc4be6f1b9ed02ab0a", size = 208759, upload-time = "2024-11-28T08:47:31.908Z" }, - { url = "https://files.pythonhosted.org/packages/28/ea/fdd9813c1eafd333ad92464d57a4e3a82b37ae57c19497bcffa42df673e4/zope.interface-7.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:550f1c6588ecc368c9ce13c44a49b8d6b6f3ca7588873c679bd8fd88a1b557b6", size = 254922, upload-time = "2024-11-28T09:18:11.795Z" }, - { url = "https://files.pythonhosted.org/packages/3b/d3/0000a4d497ef9fbf4f66bb6828b8d0a235e690d57c333be877bec763722f/zope.interface-7.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0ef9e2f865721553c6f22a9ff97da0f0216c074bd02b25cf0d3af60ea4d6931d", size = 249367, upload-time = "2024-11-28T08:48:24.238Z" }, - { url = "https://files.pythonhosted.org/packages/3e/e5/0b359e99084f033d413419eff23ee9c2bd33bca2ca9f4e83d11856f22d10/zope.interface-7.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27f926f0dcb058211a3bb3e0e501c69759613b17a553788b2caeb991bed3b61d", size = 254488, upload-time = "2024-11-28T08:48:28.816Z" }, - { url = "https://files.pythonhosted.org/packages/7b/90/12d50b95f40e3b2fc0ba7f7782104093b9fd62806b13b98ef4e580f2ca61/zope.interface-7.2-cp310-cp310-win_amd64.whl", hash = "sha256:144964649eba4c5e4410bb0ee290d338e78f179cdbfd15813de1a664e7649b3b", size = 211947, upload-time = "2024-11-28T08:48:18.831Z" }, - { url = "https://files.pythonhosted.org/packages/98/7d/2e8daf0abea7798d16a58f2f3a2bf7588872eee54ac119f99393fdd47b65/zope.interface-7.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1909f52a00c8c3dcab6c4fad5d13de2285a4b3c7be063b239b8dc15ddfb73bd2", size = 208776, upload-time = "2024-11-28T08:47:53.009Z" }, - { url = "https://files.pythonhosted.org/packages/a0/2a/0c03c7170fe61d0d371e4c7ea5b62b8cb79b095b3d630ca16719bf8b7b18/zope.interface-7.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:80ecf2451596f19fd607bb09953f426588fc1e79e93f5968ecf3367550396b22", size = 209296, upload-time = "2024-11-28T08:47:57.993Z" }, - { url = "https://files.pythonhosted.org/packages/49/b4/451f19448772b4a1159519033a5f72672221e623b0a1bd2b896b653943d8/zope.interface-7.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:033b3923b63474800b04cba480b70f6e6243a62208071fc148354f3f89cc01b7", size = 260997, upload-time = "2024-11-28T09:18:13.935Z" }, - { url = "https://files.pythonhosted.org/packages/65/94/5aa4461c10718062c8f8711161faf3249d6d3679c24a0b81dd6fc8ba1dd3/zope.interface-7.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a102424e28c6b47c67923a1f337ede4a4c2bba3965b01cf707978a801fc7442c", size = 255038, upload-time = "2024-11-28T08:48:26.381Z" }, - { url = "https://files.pythonhosted.org/packages/9f/aa/1a28c02815fe1ca282b54f6705b9ddba20328fabdc37b8cf73fc06b172f0/zope.interface-7.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25e6a61dcb184453bb00eafa733169ab6d903e46f5c2ace4ad275386f9ab327a", size = 259806, upload-time = "2024-11-28T08:48:30.78Z" }, - { url = "https://files.pythonhosted.org/packages/a7/2c/82028f121d27c7e68632347fe04f4a6e0466e77bb36e104c8b074f3d7d7b/zope.interface-7.2-cp311-cp311-win_amd64.whl", hash = "sha256:3f6771d1647b1fc543d37640b45c06b34832a943c80d1db214a37c31161a93f1", size = 212305, upload-time = "2024-11-28T08:49:14.525Z" }, - { url = "https://files.pythonhosted.org/packages/68/0b/c7516bc3bad144c2496f355e35bd699443b82e9437aa02d9867653203b4a/zope.interface-7.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:086ee2f51eaef1e4a52bd7d3111a0404081dadae87f84c0ad4ce2649d4f708b7", size = 208959, upload-time = "2024-11-28T08:47:47.788Z" }, - { url = "https://files.pythonhosted.org/packages/a2/e9/1463036df1f78ff8c45a02642a7bf6931ae4a38a4acd6a8e07c128e387a7/zope.interface-7.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:21328fcc9d5b80768bf051faa35ab98fb979080c18e6f84ab3f27ce703bce465", size = 209357, upload-time = "2024-11-28T08:47:50.897Z" }, - { url = "https://files.pythonhosted.org/packages/07/a8/106ca4c2add440728e382f1b16c7d886563602487bdd90004788d45eb310/zope.interface-7.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6dd02ec01f4468da0f234da9d9c8545c5412fef80bc590cc51d8dd084138a89", size = 264235, upload-time = "2024-11-28T09:18:15.56Z" }, - { url = "https://files.pythonhosted.org/packages/fc/ca/57286866285f4b8a4634c12ca1957c24bdac06eae28fd4a3a578e30cf906/zope.interface-7.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e7da17f53e25d1a3bde5da4601e026adc9e8071f9f6f936d0fe3fe84ace6d54", size = 259253, upload-time = "2024-11-28T08:48:29.025Z" }, - { url = "https://files.pythonhosted.org/packages/96/08/2103587ebc989b455cf05e858e7fbdfeedfc3373358320e9c513428290b1/zope.interface-7.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cab15ff4832580aa440dc9790b8a6128abd0b88b7ee4dd56abacbc52f212209d", size = 264702, upload-time = "2024-11-28T08:48:37.363Z" }, - { url = "https://files.pythonhosted.org/packages/5f/c7/3c67562e03b3752ba4ab6b23355f15a58ac2d023a6ef763caaca430f91f2/zope.interface-7.2-cp312-cp312-win_amd64.whl", hash = "sha256:29caad142a2355ce7cfea48725aa8bcf0067e2b5cc63fcf5cd9f97ad12d6afb5", size = 212466, upload-time = "2024-11-28T08:49:14.397Z" }, - { url = "https://files.pythonhosted.org/packages/c6/3b/e309d731712c1a1866d61b5356a069dd44e5b01e394b6cb49848fa2efbff/zope.interface-7.2-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:3e0350b51e88658d5ad126c6a57502b19d5f559f6cb0a628e3dc90442b53dd98", size = 208961, upload-time = "2024-11-28T08:48:29.865Z" }, - { url = "https://files.pythonhosted.org/packages/49/65/78e7cebca6be07c8fc4032bfbb123e500d60efdf7b86727bb8a071992108/zope.interface-7.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15398c000c094b8855d7d74f4fdc9e73aa02d4d0d5c775acdef98cdb1119768d", size = 209356, upload-time = "2024-11-28T08:48:33.297Z" }, - { url = "https://files.pythonhosted.org/packages/11/b1/627384b745310d082d29e3695db5f5a9188186676912c14b61a78bbc6afe/zope.interface-7.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:802176a9f99bd8cc276dcd3b8512808716492f6f557c11196d42e26c01a69a4c", size = 264196, upload-time = "2024-11-28T09:18:17.584Z" }, - { url = "https://files.pythonhosted.org/packages/b8/f6/54548df6dc73e30ac6c8a7ff1da73ac9007ba38f866397091d5a82237bd3/zope.interface-7.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb23f58a446a7f09db85eda09521a498e109f137b85fb278edb2e34841055398", size = 259237, upload-time = "2024-11-28T08:48:31.71Z" }, - { url = "https://files.pythonhosted.org/packages/b6/66/ac05b741c2129fdf668b85631d2268421c5cd1a9ff99be1674371139d665/zope.interface-7.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a71a5b541078d0ebe373a81a3b7e71432c61d12e660f1d67896ca62d9628045b", size = 264696, upload-time = "2024-11-28T08:48:41.161Z" }, - { url = "https://files.pythonhosted.org/packages/0a/2f/1bccc6f4cc882662162a1158cda1a7f616add2ffe322b28c99cb031b4ffc/zope.interface-7.2-cp313-cp313-win_amd64.whl", hash = "sha256:4893395d5dd2ba655c38ceb13014fd65667740f09fa5bb01caa1e6284e48c0cd", size = 212472, upload-time = "2024-11-28T08:49:56.587Z" }, -] From 60e6ce9dbb720e76aab84e6caed6c8c9cd2826fb Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 22 Jul 2026 22:50:05 +0200 Subject: [PATCH 04/91] feat: bootstrap same-python popen worker via `python -m` Launch the Trio popen worker as `python -m execnet._trio_worker ` instead of sending bootstrap source over the wire. The worker imports the installed execnet + trio, writes the `1` handshake after adopting its stdio fds, and serves; the coordinator just waits for the handshake. A rough major/minor version check warns on a real execnet mismatch between coordinator and worker while tolerating patch-level drift. Drops the import-bootstrap source send and the importdir/PYTHONPATH plumbing (no more uninstalled running). Foreign-python and remote transports stay on the legacy path pending the uv-provisioned bootstrap. Co-Authored-By: Claude Opus 4.8 --- doc/implnotes.rst | 18 +++++---- src/execnet/_trio_host.py | 79 ++++++++++++++++++------------------- src/execnet/_trio_worker.py | 57 +++++++++++++++++++++++++- 3 files changed, 105 insertions(+), 49 deletions(-) diff --git a/doc/implnotes.rst b/doc/implnotes.rst index c8229865..84492656 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -18,17 +18,21 @@ for standardizing or versioning the protocol. Trio host-thread IO (popen / import bootstrap) ---------------------------------------------- -For local ``popen`` gateways that use import-based bootstrap -(same installed ``execnet`` + ``trio`` on both sides), Message -protocol IO runs inside a dedicated OS thread hosting a Trio -event loop (``execnet._trio_host.TrioHost``): +For local same-interpreter ``popen`` gateways, Message protocol IO +runs inside a dedicated OS thread hosting a Trio event loop +(``execnet._trio_host.TrioHost``). No source is sent over the wire: +the worker is launched as ``python -m execnet._trio_worker`` and +imports the installed ``execnet`` + ``trio`` (a rough major/minor +version check guards against an incompatible install): * Coordinator: ``trio.lowlevel.open_process`` plus async framed reader/writer tasks per gateway (one host thread per ``Group``). + It waits for the worker's ``b"1"`` handshake before starting the + Message protocol. * Worker: ``serve_popen_trio`` adopts stdio pipe fds into Trio - streams; ``remote_exec`` is scheduled from the Trio nursery - (``trio.to_thread`` for ``thread``, main-thread handoff for - ``main_thread_only``). + streams and writes the handshake byte; ``remote_exec`` is + scheduled from the Trio nursery (``trio.to_thread`` for ``thread``, + main-thread handoff for ``main_thread_only``). Sync ``Channel`` / ``Gateway`` APIs are unchanged. Sends from non-host threads wait until the frame is written (so abrupt diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index b0db9ad7..ce9ef1cb 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -10,6 +10,7 @@ import os import queue import subprocess +import sys import threading from collections.abc import Callable from typing import TYPE_CHECKING @@ -40,20 +41,19 @@ def trio_host_enabled() -> bool: def should_use_trio_popen(spec: Any) -> bool: - """PoC: Trio path only for local popen with import bootstrap.""" + """Trio path only for local same-interpreter popen (module bootstrap).""" if not trio_host_enabled(): return False if not getattr(spec, "popen", False): return False if getattr(spec, "via", None): return False + # A foreign interpreter (python=) is handled by the future uv-provisioned + # remote path, not the same-interpreter module launch. if getattr(spec, "python", None): return False - # Worker exec still uses WorkerPool; greenlet models stay on the legacy path. execmodel = getattr(spec, "execmodel", None) - if execmodel not in (None, "thread", "main_thread_only"): - return False - return True + return execmodel in (None, "thread", "main_thread_only") class AsyncByteIO(Protocol): @@ -273,7 +273,9 @@ async def _kill() -> None: def is_alive(self) -> bool: return not self._done.is_set() - async def task(self, task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED) -> None: + async def task( + self, task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED + ) -> None: try: async with trio.open_nursery() as nursery: nursery.start_soon(self._writer) @@ -480,30 +482,27 @@ async def open_popen_process(args: list[str]) -> trio.Process: ) -async def bootstrap_import_async( - process: trio.Process, - *, - importdir: str, - execmodel: str, - gateway_id: str, -) -> ProcessStreamsIO: - """Send import-bootstrap source and wait for the handshake byte.""" - io = ProcessStreamsIO(process) - sources = [ - "import sys", - "if %r not in sys.path:" % importdir, - " sys.path.insert(0, %r)" % importdir, - "from execnet._trio_worker import serve_popen_trio", - "sys.stdout.write('1')", - "sys.stdout.flush()", - "serve_popen_trio(id=%r, execmodel=%r)" % (f"{gateway_id}-worker", execmodel), +def popen_module_args(spec: Any) -> list[str]: + """Launch the Trio worker as a module: ``python -m execnet._trio_worker``. + + No source is sent over the wire; the worker imports the installed execnet + + trio. Only valid for same-interpreter popen (see ``should_use_trio_popen``). + The coordinator version is passed so the worker can do a rough compatibility + check against its own installed version. + """ + import execnet + + args = [sys.executable, "-u"] + if getattr(spec, "dont_write_bytecode", False): + args.append("-B") + args += [ + "-m", + "execnet._trio_worker", + f"{spec.id}-worker", + spec.execmodel, + execnet.__version__, ] - source = "\n".join(sources) - await io.write_all((repr(source) + "\n").encode("utf-8")) - ack = await io.read_exact(1) - if ack != b"1": - raise EOFError(f"bad bootstrap handshake: {ack!r}") - return io + return args class _TempIO: @@ -532,25 +531,23 @@ def kill(self) -> None: def makegateway_popen_trio(group: Any, spec: Any) -> Any: - """Create a popen Gateway using Trio process + protocol IO on both sides.""" - import execnet + """Create a same-interpreter popen Gateway on the Trio IO path. - from . import gateway_io - from .gateway_bootstrap import importdir + The worker is launched as ``python -m execnet._trio_worker`` and imports the + installed execnet + trio; nothing is sent over the wire to bootstrap it. + """ + import execnet host: TrioHost = group._ensure_trio_host() - args = gateway_io.popen_args(spec) - remote_execmodel = spec.execmodel + args = popen_module_args(spec) async def _create_and_attach() -> Any: process = await open_popen_process(args) try: - async_io = await bootstrap_import_async( - process, - importdir=importdir, - execmodel=remote_execmodel, - gateway_id=spec.id, - ) + async_io = ProcessStreamsIO(process) + ack = await async_io.read_exact(1) + if ack != b"1": + raise EOFError(f"bad bootstrap handshake: {ack!r}") except BaseException: with trio.move_on_after(5): process.kill() diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 6d247194..2c297589 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -11,7 +11,6 @@ import trio -from . import gateway_base from .gateway_base import MAIN_THREAD_ONLY_DEADLOCK_TEXT from .gateway_base import WorkerGateway from .gateway_base import get_execmodel @@ -211,6 +210,10 @@ def serve_popen_trio(id: str, execmodel: str = "thread") -> None: model = get_execmodel(execmodel) read_fd, write_fd = _prepare_protocol_fds() + # Bootstrap handshake: the coordinator waits for this byte on our stdout + # before starting the Message protocol. We are launched as a plain module + # (``python -m execnet._trio_worker``), so nothing was sent to bootstrap us. + os.write(write_fd, b"1") # Keep the historic trace token so tests looking for workergateway still pass. trace(f"creating workergateway on trio id={id!r}") @@ -250,3 +253,55 @@ async def _start() -> _trio_host.ProtocolSession: # Trio's to_thread cache uses non-daemon threads that would otherwise # keep this disposable worker process alive after serve returns. os._exit(0) + + +def _rough_version(version: str) -> tuple[int, ...]: + """Leading numeric (major, minor) of a version string; ``()`` if unparsable.""" + parts: list[int] = [] + for chunk in version.split(".")[:2]: + number = "" + for char in chunk: + if char.isdigit(): + number += char + else: + break + if not number: + break + parts.append(int(number)) + return tuple(parts) + + +def _check_version(coordinator_version: str) -> None: + """Warn on a real (major/minor) execnet version mismatch across the wire. + + A minimal (patch-level) mismatch is tolerated. For same-interpreter popen + the versions are always identical; this guards the future remote paths. + """ + import execnet + + ours = _rough_version(execnet.__version__) + theirs = _rough_version(coordinator_version) + if ours and theirs and ours != theirs: + sys.stderr.write( + "WARNING: execnet version mismatch: coordinator %s worker %s\n" + % (coordinator_version, execnet.__version__) + ) + sys.stderr.flush() + + +def _main() -> None: + """Entry point for ``python -m execnet._trio_worker [ver]``. + + The worker imports execnet + trio from the environment; no source is sent + over the wire to bootstrap it. + """ + argv = sys.argv + worker_id = argv[1] if len(argv) > 1 else "worker" + execmodel = argv[2] if len(argv) > 2 else "thread" + if len(argv) > 3: + _check_version(argv[3]) + serve_popen_trio(id=worker_id, execmodel=execmodel) + + +if __name__ == "__main__": + _main() From dfe7391024badcd7ecc15d18e440d193cd9f5aae Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 22 Jul 2026 22:53:28 +0200 Subject: [PATCH 05/91] add claude settings --- .claude/settings.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .claude/settings.json diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 00000000..489c3d39 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,7 @@ +{ + "permissions": { + "allow": [ + "Bash(uv run pytest:*)" + ] + } +} From 603dfc6551afdc66251a77463846ff8b1ff3674e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 00:21:13 +0200 Subject: [PATCH 06/91] feat: provision foreign-python popen workers via uv Route `popen//python=` through the Trio path. If the target interpreter already imports execnet + trio, launch the worker module directly on it (preserving sys.executable); otherwise provision an ephemeral environment with uv and run the worker there. New _provision.py builds the launch: - coordinator_requirement(): `execnet==` for a released coordinator, else a wheel built from the editable source (located via PEP 610 direct_url.json) and cached keyed by version. The wheel path is read from uv's "Successfully built" output rather than reconstructed, since the build-time version can differ from the import-time one. - uv_run_argv(): `uv run --no-project [--python X] --with python -u -m execnet._trio_worker ...`; trio comes in transitively. - target_has_execnet(): cached probe deciding direct vs provisioned. Falls back to the legacy source-copy path when the target lacks execnet and uv is unavailable. Co-Authored-By: Claude Opus 4.8 --- src/execnet/_provision.py | 180 ++++++++++++++++++++++++++++++++++++++ src/execnet/_trio_host.py | 50 ++++++++--- 2 files changed, 217 insertions(+), 13 deletions(-) create mode 100644 src/execnet/_provision.py diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py new file mode 100644 index 00000000..55a7a055 --- /dev/null +++ b/src/execnet/_provision.py @@ -0,0 +1,180 @@ +"""Coordinator-side worker provisioning via ``uv``. + +Non-same-interpreter Trio workers (foreign-python popen, later ssh/socket) are +launched inside an environment that ``uv`` provisions with a matching execnet + +trio. The worker itself is always ``python -m execnet._trio_worker`` and does +the usual ``b"1"`` stdio handshake; only the launch prefix differs. + +Delivery of execnet into that environment is version-aware: + +* released coordinator (``X.Y.Z``) -> ``uv run --with execnet==X.Y.Z`` +* dev coordinator (``X.Y.Z.devN+g...``) -> build a wheel from the editable + install's source tree, cache it keyed by version, and ``uv run --with `` + +trio is pulled transitively as an execnet dependency. +""" + +from __future__ import annotations + +import json +import re +import shutil +import subprocess +import tempfile +from functools import cache +from pathlib import Path +from typing import Any + +_RELEASED_RE = re.compile(r"^\d+\.\d+\.\d+$") + + +def uv_available() -> bool: + """Whether the ``uv`` launcher is on PATH.""" + return shutil.which("uv") is not None + + +@cache +def target_has_execnet(python: str) -> bool: + """Whether interpreter ``python`` can already import execnet + trio. + + When true the worker can be launched directly on that interpreter + (preserving ``sys.executable``); otherwise it must be uv-provisioned. + """ + from .gateway_io import shell_split_path + + argv = [*shell_split_path(python), "-c", "import execnet, trio"] + try: + completed = subprocess.run( + argv, capture_output=True, timeout=30, check=False + ) + return completed.returncode == 0 + except (OSError, subprocess.SubprocessError): + return False + + +def _version_slug(version: str) -> str: + """Filesystem-safe slug for a version string (may contain ``+``/``.``).""" + return re.sub(r"[^0-9A-Za-z]+", "_", version) + + +def _wheel_cache_dir() -> Path: + d = Path(tempfile.gettempdir()) / "execnet-bootstrap-wheels" + d.mkdir(parents=True, exist_ok=True) + return d + + +def _editable_source_root() -> Path | None: + """Source tree of an editable execnet install, via PEP 610 ``direct_url.json``. + + Returns ``None`` when execnet is not installed editable (nothing to build). + """ + from importlib.metadata import PackageNotFoundError + from importlib.metadata import distribution + from urllib.parse import urlparse + from urllib.request import url2pathname + + try: + dist = distribution("execnet") + except PackageNotFoundError: + return None + raw = dist.read_text("direct_url.json") + if not raw: + return None + info = json.loads(raw) + if not info.get("dir_info", {}).get("editable"): + return None + url = info.get("url", "") + if not url.startswith("file:"): + return None + return Path(url2pathname(urlparse(url).path)) + + +_BUILT_RE = re.compile(r"^Successfully built (?P.+\.whl)\s*$", re.MULTILINE) + + +def _parse_built_wheel(uv_build_stderr: str) -> Path | None: + """Extract the exact wheel path from ``uv build``'s ``Successfully built`` line. + + Building the filename ourselves is unsafe: the build-time version (dirty tree + -> ``.dYYYYMMDD``/differing dev count) can differ from the import-time + ``__version__``, so we trust the path uv reports. + """ + matches = _BUILT_RE.findall(uv_build_stderr) + if len(matches) != 1: + return None + return Path(matches[0].strip()) + + +def _build_wheel(version: str) -> Path: + """Build (and cache) a wheel of the editable execnet source for ``version``.""" + version_dir = _wheel_cache_dir() / _version_slug(version) + if version_dir.exists(): + cached = sorted(version_dir.glob("*.whl")) + if len(cached) == 1: + return cached[0] + + root = _editable_source_root() + if root is None: + raise RuntimeError( + f"cannot provision dev execnet {version!r}: no editable source tree " + "found (install a released execnet or an editable checkout)" + ) + version_dir.mkdir(parents=True, exist_ok=True) + proc = subprocess.run( + ["uv", "build", "--wheel", "-o", str(version_dir), str(root)], + check=True, + capture_output=True, + text=True, + ) + wheel = _parse_built_wheel(proc.stderr) + if wheel is None or not wheel.exists(): + raise RuntimeError( + f"could not determine wheel path from uv build for {version!r}:\n" + f"{proc.stderr}" + ) + return wheel + + +def coordinator_requirement() -> str: + """A ``uv --with`` requirement that installs this coordinator's execnet.""" + import execnet + + version = execnet.__version__ + if _RELEASED_RE.match(version): + return f"execnet=={version}" + return str(_build_wheel(version)) + + +def uv_run_argv( + *, python: str | None, requirement: str, module_args: list[str] +) -> list[str]: + """Build ``uv run [--python X] --with python -u -m execnet._trio_worker …``. + + ``--no-project`` keeps the surrounding execnet checkout from being synced, so + the ephemeral env holds only the requirement (+ trio). + """ + argv = ["uv", "run", "--no-project"] + if python: + argv += ["--python", python] + argv += [ + "--with", + requirement, + "python", + "-u", + "-m", + "execnet._trio_worker", + *module_args, + ] + return argv + + +def uv_worker_argv(spec: Any) -> list[str]: + """Full ``uv run`` argv to launch the Trio worker for ``spec``.""" + import execnet + + module_args = [f"{spec.id}-worker", spec.execmodel, execnet.__version__] + return uv_run_argv( + python=spec.python, + requirement=coordinator_requirement(), + module_args=module_args, + ) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index ce9ef1cb..d72b681d 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -41,19 +41,27 @@ def trio_host_enabled() -> bool: def should_use_trio_popen(spec: Any) -> bool: - """Trio path only for local same-interpreter popen (module bootstrap).""" + """Trio path for local popen. + + Same-interpreter popen launches the worker module directly; a foreign + interpreter (``python=``) is provisioned via ``uv`` and only taken when + ``uv`` is available (otherwise the legacy source-copy path handles it). + """ if not trio_host_enabled(): return False if not getattr(spec, "popen", False): return False if getattr(spec, "via", None): return False - # A foreign interpreter (python=) is handled by the future uv-provisioned - # remote path, not the same-interpreter module launch. - if getattr(spec, "python", None): - return False execmodel = getattr(spec, "execmodel", None) - return execmodel in (None, "thread", "main_thread_only") + if execmodel not in (None, "thread", "main_thread_only"): + return False + if getattr(spec, "python", None): + from . import _provision + + # Direct launch if the interpreter already has execnet; else uv-provision. + return _provision.target_has_execnet(spec.python) or _provision.uv_available() + return True class AsyncByteIO(Protocol): @@ -486,13 +494,21 @@ def popen_module_args(spec: Any) -> list[str]: """Launch the Trio worker as a module: ``python -m execnet._trio_worker``. No source is sent over the wire; the worker imports the installed execnet + - trio. Only valid for same-interpreter popen (see ``should_use_trio_popen``). - The coordinator version is passed so the worker can do a rough compatibility + trio. Used for same-interpreter popen and for a ``python=`` interpreter that + already has execnet (so ``sys.executable`` stays that interpreter). The + coordinator version is passed so the worker can do a rough compatibility check against its own installed version. """ import execnet - args = [sys.executable, "-u"] + if getattr(spec, "python", None): + from .gateway_io import shell_split_path + + interpreter = shell_split_path(spec.python) + else: + interpreter = [sys.executable] + + args = [*interpreter, "-u"] if getattr(spec, "dont_write_bytecode", False): args.append("-B") args += [ @@ -531,15 +547,23 @@ def kill(self) -> None: def makegateway_popen_trio(group: Any, spec: Any) -> Any: - """Create a same-interpreter popen Gateway on the Trio IO path. + """Create a popen Gateway on the Trio IO path. - The worker is launched as ``python -m execnet._trio_worker`` and imports the - installed execnet + trio; nothing is sent over the wire to bootstrap it. + Same-interpreter popen launches ``python -m execnet._trio_worker`` directly; + a foreign interpreter (``python=``) is provisioned via ``uv``. Either way the + worker imports execnet + trio; nothing is sent over the wire to bootstrap it. """ import execnet + from . import _provision + host: TrioHost = group._ensure_trio_host() - args = popen_module_args(spec) + if spec.python and not _provision.target_has_execnet(spec.python): + # bare interpreter: provision execnet + trio via uv + args = _provision.uv_worker_argv(spec) + else: + # same interpreter, or a python= that already has execnet + args = popen_module_args(spec) async def _create_and_attach() -> Any: process = await open_popen_process(args) From 76563685af7204c19b07213b620260c833dcc138 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 07:32:11 +0200 Subject: [PATCH 07/91] test: add local ssh-connect harness + typing cleanups Add an in-process asyncssh server (asyncio-loop thread) with binary-safe command passthrough so the ssh transport can be tested against the system ssh client without an external host. Committed, intentionally-insecure ed25519 test keys back it (see testing/sshkeys/README.md); the fixture copies the client key to a 0600 temp file since git does not preserve it. asyncssh joins the testing extra. The initial test exercises the existing legacy ssh path, validating the harness before the Trio ssh transport. Also make mypy green at the source instead of casting at call sites: TrioHost.call is now generic (Callable[..., Awaitable[T]] -> T) and makegateway_popen_trio returns Gateway. Drop the stale types-gevent from the mypy hook. Co-Authored-By: Claude Opus 4.8 --- .pre-commit-config.yaml | 1 - pyproject.toml | 5 + src/execnet/_provision.py | 4 +- src/execnet/_trio_host.py | 17 ++- src/execnet/gateway.py | 2 +- testing/sshkeys/README.md | 17 +++ testing/sshkeys/insecure_client_ed25519 | 7 + testing/sshkeys/insecure_client_ed25519.pub | 1 + testing/sshkeys/insecure_host_ed25519 | 7 + testing/sshkeys/insecure_host_ed25519.pub | 1 + testing/test_ssh_local.py | 147 ++++++++++++++++++++ uv.lock | 44 +++++- 12 files changed, 241 insertions(+), 12 deletions(-) create mode 100644 testing/sshkeys/README.md create mode 100644 testing/sshkeys/insecure_client_ed25519 create mode 100644 testing/sshkeys/insecure_client_ed25519.pub create mode 100644 testing/sshkeys/insecure_host_ed25519 create mode 100644 testing/sshkeys/insecure_host_ed25519.pub create mode 100644 testing/test_ssh_local.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index be4b6e07..37ccd44d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,4 +32,3 @@ repos: additional_dependencies: - pytest - types-pywin32 - - types-gevent diff --git a/pyproject.toml b/pyproject.toml index 2c73a32b..853ba3d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ testing = [ "tox", "hatch", "uv", + "asyncssh", ] [dependency-groups] @@ -121,3 +122,7 @@ warn_unused_ignores = false disallow_untyped_calls = false disallow_untyped_defs = false disallow_incomplete_defs = false + +[[tool.mypy.overrides]] +module = ["asyncssh.*"] +ignore_missing_imports = true diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index 55a7a055..b7f6ac56 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -44,9 +44,7 @@ def target_has_execnet(python: str) -> bool: argv = [*shell_split_path(python), "-c", "import execnet, trio"] try: - completed = subprocess.run( - argv, capture_output=True, timeout=30, check=False - ) + completed = subprocess.run(argv, capture_output=True, timeout=30, check=False) return completed.returncode == 0 except (OSError, subprocess.SubprocessError): return False diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index d72b681d..42b45047 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -12,11 +12,13 @@ import subprocess import sys import threading +from collections.abc import Awaitable from collections.abc import Callable from typing import TYPE_CHECKING from typing import Any from typing import Protocol from typing import TypeVar +from typing import cast import trio @@ -26,6 +28,7 @@ from .gateway_base import trace if TYPE_CHECKING: + from .gateway import Gateway from .gateway_base import BaseGateway T = TypeVar("T") @@ -251,7 +254,7 @@ def wait_process(self) -> int | None: async def _wait() -> int | None: assert self.process is not None # Always await wait() so the child is reaped (no zombies). - code = await self.process.wait() + code: int | None = await self.process.wait() self._process_exitcode = code self._process_done.set() return code @@ -434,15 +437,17 @@ async def _main(self) -> None: finally: self._nursery = None - def call(self, async_fn: Callable[..., Any], *args: Any) -> Any: + def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: if self._token is None: raise RuntimeError("TrioHost is not running") - return trio.from_thread.run(async_fn, *args, trio_token=self._token) + return cast("T", trio.from_thread.run(async_fn, *args, trio_token=self._token)) def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: if self._token is None: raise RuntimeError("TrioHost is not running") - return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + return cast( + "T", trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + ) def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: """Schedule a task on the root nursery (must be called on the host thread).""" @@ -546,7 +551,7 @@ def kill(self) -> None: return -def makegateway_popen_trio(group: Any, spec: Any) -> Any: +def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: """Create a popen Gateway on the Trio IO path. Same-interpreter popen launches ``python -m execnet._trio_worker`` directly; @@ -565,7 +570,7 @@ def makegateway_popen_trio(group: Any, spec: Any) -> Any: # same interpreter, or a python= that already has execnet args = popen_module_args(spec) - async def _create_and_attach() -> Any: + async def _create_and_attach() -> Gateway: process = await open_popen_process(args) try: async_io = ProcessStreamsIO(process) diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index 360b9865..6e336d6f 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -103,7 +103,7 @@ def hasreceiver(self) -> bool: """Whether gateway is able to receive data.""" session = self._trio_session if session is not None: - return session.is_alive() + return bool(session.is_alive()) return self._receivepool.active_count() > 0 def remote_status(self) -> RemoteStatus: diff --git a/testing/sshkeys/README.md b/testing/sshkeys/README.md new file mode 100644 index 00000000..d0133e2c --- /dev/null +++ b/testing/sshkeys/README.md @@ -0,0 +1,17 @@ +# Intentionally insecure test SSH keys + +These ed25519 keypairs are **committed on purpose** and are **not secret**. They +exist only so the local ssh-connect tests (`testing/test_ssh_local.py`) can run +an in-process asyncssh server that the system `ssh` client authenticates against, +without generating keys at runtime. + +- `insecure_host_ed25519[.pub]` — the test SSH **server** host key. +- `insecure_client_ed25519[.pub]` — the test **client** identity; its public key + is the server's sole authorized key. + +**Never** use these anywhere real. They grant nothing beyond a throwaway server +bound to `127.0.0.1` on an ephemeral port during a test run. + +Note: git does not preserve `0600` permissions, and OpenSSH refuses a +world-readable private key, so the tests copy the client key to a temp dir and +`chmod 0600` it before use. diff --git a/testing/sshkeys/insecure_client_ed25519 b/testing/sshkeys/insecure_client_ed25519 new file mode 100644 index 00000000..6f5650c8 --- /dev/null +++ b/testing/sshkeys/insecure_client_ed25519 @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACB6MZDwubY5LfTxiuBxhoOmRZaebHcTuRC/ahHgDDDXzwAAAKBzGIF1cxiB +dQAAAAtzc2gtZWQyNTUxOQAAACB6MZDwubY5LfTxiuBxhoOmRZaebHcTuRC/ahHgDDDXzw +AAAEBCIAXAZCBC6mZFORLaloyPr6HZsRkWpVxVd/vKSZpIhXoxkPC5tjkt9PGK4HGGg6ZF +lp5sdxO5EL9qEeAMMNfPAAAAHGV4ZWNuZXQtaW5zZWN1cmUtdGVzdC1jbGllbnQB +-----END OPENSSH PRIVATE KEY----- diff --git a/testing/sshkeys/insecure_client_ed25519.pub b/testing/sshkeys/insecure_client_ed25519.pub new file mode 100644 index 00000000..b4d824b9 --- /dev/null +++ b/testing/sshkeys/insecure_client_ed25519.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHoxkPC5tjkt9PGK4HGGg6ZFlp5sdxO5EL9qEeAMMNfP execnet-insecure-test-client diff --git a/testing/sshkeys/insecure_host_ed25519 b/testing/sshkeys/insecure_host_ed25519 new file mode 100644 index 00000000..0a31038e --- /dev/null +++ b/testing/sshkeys/insecure_host_ed25519 @@ -0,0 +1,7 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACBqqoHPiDB2eUJlGVCLM2eRtfelu138wnGLDRHd1AG9RwAAAKDBS525wUud +uQAAAAtzc2gtZWQyNTUxOQAAACBqqoHPiDB2eUJlGVCLM2eRtfelu138wnGLDRHd1AG9Rw +AAAECXfzL6Ua9c9U+UkQn+Q7A1aKlBboBhABFALH+/ojmbMGqqgc+IMHZ5QmUZUIszZ5G1 +96W7XfzCcYsNEd3UAb1HAAAAGmV4ZWNuZXQtaW5zZWN1cmUtdGVzdC1ob3N0AQID +-----END OPENSSH PRIVATE KEY----- diff --git a/testing/sshkeys/insecure_host_ed25519.pub b/testing/sshkeys/insecure_host_ed25519.pub new file mode 100644 index 00000000..93d3cf93 --- /dev/null +++ b/testing/sshkeys/insecure_host_ed25519.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGqqgc+IMHZ5QmUZUIszZ5G196W7XfzCcYsNEd3UAb1H execnet-insecure-test-host diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py new file mode 100644 index 00000000..196c5d76 --- /dev/null +++ b/testing/test_ssh_local.py @@ -0,0 +1,147 @@ +"""Local ssh-connect tests backed by an in-process asyncssh server. + +The coordinator shells out to the system ``ssh`` client, which connects to an +asyncssh server running in its own asyncio-loop thread; the server runs each +requested command as a subprocess with binary-safe stdio passthrough (the +execnet Message protocol needs raw bytes). +""" + +from __future__ import annotations + +import asyncio +import os +import shutil +import sys +import threading +from collections.abc import Iterator +from pathlib import Path + +import asyncssh +import pytest + +import execnet + +pytestmark = [ + pytest.mark.skipif( + shutil.which("ssh") is None, reason="system ssh client required" + ), + # asyncssh leaves an un-awaited internal Queue.join coroutine on shutdown, + # surfaced by pytest's unraisable-exception hook during GC; harmless here. + pytest.mark.filterwarnings("ignore:coroutine 'Queue.join' was never awaited"), +] + +# Committed, intentionally-insecure test keys (see sshkeys/README.md). +SSHKEYS = Path(__file__).parent / "sshkeys" +HOST_KEY = SSHKEYS / "insecure_host_ed25519" +CLIENT_KEY = SSHKEYS / "insecure_client_ed25519" +CLIENT_PUBKEY = SSHKEYS / "insecure_client_ed25519.pub" + + +class SSHServerThread: + """asyncssh server on an ephemeral port, driven from its own asyncio thread.""" + + def __init__(self, client_key_path: str) -> None: + # OpenSSH refuses a world-readable private key; git does not preserve + # 0600, so the caller hands us a temp copy already chmod'd 0600. + self.client_key_path = client_key_path + self._loop: asyncio.AbstractEventLoop | None = None + self._server: asyncssh.SSHAcceptor | None = None + self.port: int | None = None + self._ready = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + + async def _handle(self, process: asyncssh.SSHServerProcess) -> None: + proc = await asyncio.create_subprocess_shell( + process.command or "", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + await process.redirect(stdin=proc.stdin, stdout=proc.stdout, stderr=proc.stderr) + process.exit(await proc.wait()) + + async def _serve(self) -> None: + self._server = await asyncssh.listen( + "127.0.0.1", + 0, + server_host_keys=[str(HOST_KEY)], + authorized_client_keys=str(CLIENT_PUBKEY), + process_factory=self._handle, + encoding=None, # binary stdio + ) + self.port = self._server.get_port() + self._ready.set() + await self._server.wait_closed() + + def _run(self) -> None: + self._loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._loop) + try: + self._loop.run_until_complete(self._serve()) + finally: + # Drain asyncssh's shutdown coroutines so GC does not surface an + # "un-awaited coroutine" RuntimeWarning. + pending = asyncio.all_tasks(self._loop) + for task in pending: + task.cancel() + self._loop.run_until_complete( + asyncio.gather(*pending, return_exceptions=True) + ) + self._loop.run_until_complete(self._loop.shutdown_asyncgens()) + self._loop.close() + + def start(self) -> None: + self._thread.start() + assert self._ready.wait(timeout=10), "ssh server did not start" + + def stop(self) -> None: + if self._loop is not None and self._server is not None: + self._loop.call_soon_threadsafe(self._server.close) + self._thread.join(timeout=5) + + def write_ssh_config(self, path: str) -> None: + """Write an ssh config with a ``testhost`` alias pointing at this server.""" + with open(path, "w") as f: + f.write( + "Host testhost\n" + " HostName 127.0.0.1\n" + f" Port {self.port}\n" + " User testuser\n" + f" IdentityFile {self.client_key_path}\n" + " IdentitiesOnly yes\n" + " StrictHostKeyChecking no\n" + " UserKnownHostsFile /dev/null\n" + " LogLevel ERROR\n" + ) + + +@pytest.fixture +def ssh_server(tmp_path) -> Iterator[SSHServerThread]: + # OpenSSH rejects the committed key's checkout permissions; use a 0600 copy. + client_key = tmp_path / "client_ed25519" + client_key.write_bytes(CLIENT_KEY.read_bytes()) + client_key.chmod(0o600) + server = SSHServerThread(str(client_key)) + server.start() + yield server + server.stop() + + +@pytest.fixture +def ssh_config(ssh_server: SSHServerThread, tmp_path) -> str: + path = str(tmp_path / "ssh_config") + ssh_server.write_ssh_config(path) + return path + + +def test_ssh_roundtrip(ssh_config: str) -> None: + group = execnet.Group() + try: + gw = group.makegateway( + f"ssh=testhost//ssh_config={ssh_config}//python={sys.executable}//id=ssh" + ) + channel = gw.remote_exec("channel.send(channel.receive() + 1)") + channel.send(41) + assert channel.receive() == 42 + finally: + group.terminate(timeout=5.0) diff --git a/uv.lock b/uv.lock index 5dbf4f10..e4657f7b 100644 --- a/uv.lock +++ b/uv.lock @@ -16,6 +16,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, ] +[[package]] +name = "asyncssh" +version = "2.24.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/7f/2d79247bacc562104f312d27efe541673aa177feac89de291bd61bca52be/asyncssh-2.24.0.tar.gz", hash = "sha256:4064c590e59ce2e8d82a2f66d35f3120d765828b4df5e3dbfb07b4a8c24686c9", size = 550148, upload-time = "2026-06-27T20:34:44.755Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/29/908ce0ca5e8cae76662e354a0f08df552d6d221844748b9e5ca06051cc44/asyncssh-2.24.0-py3-none-any.whl", hash = "sha256:9abd46300adcb6d4b73269b34c53cd0d17a138b9a22b5b38008ce7d5808734b7", size = 381237, upload-time = "2026-06-27T20:34:43.198Z" }, +] + [[package]] name = "attrs" version = "26.1.0" @@ -145,6 +158,8 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/57/5f/ff100cae70ebe9d8df1c01a00e510e45d9adb5c1fdda84791b199141de97/cffi-2.1.0.tar.gz", hash = "sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9", size = 531036, upload-time = "2026-07-06T21:34:30.382Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/e9/6d7724983b3d5a0908dbf74f64038ade77c18646ff6636ec7894fd392ce1/cffi-2.1.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:b65f590ef2a44640f9a05dbb548a429b4ade77913ce683ac8b1480777658a6c0", size = 183837, upload-time = "2026-07-06T21:32:09.655Z" }, + { url = "https://files.pythonhosted.org/packages/69/aa/24580a278de21fd7322635556334d9b535f1cbc00b0a3919447cdf464c65/cffi-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:164bff1657b2a74f0b6d54e11c9b375bc97b931f2ca9c43fcf875838da1570dd", size = 184226, upload-time = "2026-07-06T21:32:11.196Z" }, { url = "https://files.pythonhosted.org/packages/88/a9/02cae418ec4beb282ace11958d9d4737793439d561fadc7e6d56f2e2b354/cffi-2.1.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:c941bb58d5a6e1c3892d86e42927ed6c180302f07e6d395d08c416e594b98b46", size = 211107, upload-time = "2026-07-06T21:32:12.328Z" }, { url = "https://files.pythonhosted.org/packages/3b/30/c806937ed5e4c2c7ac30d9d6b76b5dc57ff8b75d83800d9bb11a8253cf2a/cffi-2.1.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a016194dbe13d14ee9556e734b772d8d67b947092b268d757fd4290e3ba2dfc2", size = 218733, upload-time = "2026-07-06T21:32:13.67Z" }, { url = "https://files.pythonhosted.org/packages/f9/cf/398272b8bbfd58aa314fda5a7f1cdbb26d1d78ae324a11211521315dd1f0/cffi-2.1.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:03e9810d18c646077e501f661b682fbf5dee4676048527ca3cffe66faa9960dd", size = 205543, upload-time = "2026-07-06T21:32:15.148Z" }, @@ -155,6 +170,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/75/74dfb7c3fc6ebbd408038476bd4c1d7e925c62614e7b9c534ecc34218288/cffi-2.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:11b3fb55f4f8ad92274ed26705f65d8f91457de71f5380061eb6d125a768fecd", size = 220341, upload-time = "2026-07-06T21:32:21.9Z" }, { url = "https://files.pythonhosted.org/packages/70/b6/9003c33a3e7d2c1306f5962e646457dcfe5a8cd8fce6bbe02d7af25db783/cffi-2.1.0-cp310-cp310-win32.whl", hash = "sha256:9d72af0cf10a76a600a9690078fe31c63b9588c8e86bf9fd353f713c84b5db0f", size = 174578, upload-time = "2026-07-06T21:32:23.073Z" }, { url = "https://files.pythonhosted.org/packages/8a/26/710688310447531c7a22f857c7f79d9855ec18b03e04494ced723fb37e2f/cffi-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:fb62edb5bb52cca65fab91a63afa7561607120d26090a7e8fda6fb9f064726da", size = 185071, upload-time = "2026-07-06T21:32:24.671Z" }, + { url = "https://files.pythonhosted.org/packages/d3/67/85c89a59ba36a671e79638f44d466749f08179266a57e4f2ffdf92174072/cffi-2.1.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:02cb7ff33ded4f1532476731f89ede53e2e488a8e6205515a82144246ffa7dcc", size = 183845, upload-time = "2026-07-06T21:32:26.32Z" }, + { url = "https://files.pythonhosted.org/packages/ea/dd/e3b0baa2d3d6a857ac72b7efbf18e32e487c9cdafcc13049ad765495b15e/cffi-2.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f5bce581e6b8c235e566a14768a943b172ada3ed73537bb0c0be1edee312d4e7", size = 184186, upload-time = "2026-07-06T21:32:28.025Z" }, { url = "https://files.pythonhosted.org/packages/65/68/9f3ef890cf3c6ab97bd531c5677f67613d302165d16f8142b2811782a614/cffi-2.1.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:30b65779d598c370374fefabf138d456fd6f3216bfa7bedfab1ba82025b0cd93", size = 211892, upload-time = "2026-07-06T21:32:29.565Z" }, { url = "https://files.pythonhosted.org/packages/22/d7/1a74539db16d8bfd839ff1515948948efbb162e574650fd3d846896eea95/cffi-2.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88023dfe18799507b73f1dbb0d14326a17465de1bc9c9c7655c22845e9ddc3a2", size = 218793, upload-time = "2026-07-06T21:32:30.951Z" }, { url = "https://files.pythonhosted.org/packages/ec/d1/9a5b7169499e8e8d8e636de70b97ac7c9447104d2ff1a2cd94790cea5162/cffi-2.1.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:0a96b74cda968eebbad56d973efe5098974f0a9fb323865bf99ea1fd24e3e64c", size = 205737, upload-time = "2026-07-06T21:32:32.216Z" }, @@ -166,6 +183,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/da/5c4918a2d61d86fa927d716cb3d8e4626ef8dc8f605a599d32f33897f59a/cffi-2.1.0-cp311-cp311-win32.whl", hash = "sha256:64c753a0f87a256020004f37a1c8c02c480e725f910f0b2a0f3f07debd1b2479", size = 174496, upload-time = "2026-07-06T21:32:40.467Z" }, { url = "https://files.pythonhosted.org/packages/f9/c8/6c2de1d55cf35ef8b92885d5ef280790f0fb9634d87ea1cc315176aecd61/cffi-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:4f26194e3d95e06501b942642855aed4f953d55e95d7d01b7c4483db3ecff458", size = 185113, upload-time = "2026-07-06T21:32:41.761Z" }, { url = "https://files.pythonhosted.org/packages/9e/4e/e8d7cb5783f1841a3c8fb3a7735838d7484d08ec08c9f984b14cac1ac0e9/cffi-2.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:35aaea0c7ee0e58a5cd8c2fd1a48fdf7ece0d2699b7ecdda08194e9ce5dd9b3d", size = 179927, upload-time = "2026-07-06T21:32:42.961Z" }, + { url = "https://files.pythonhosted.org/packages/1e/85/990925db5df586ec90beb97529c853497e7f85ba0234830447faf41c3057/cffi-2.1.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f", size = 184829, upload-time = "2026-07-06T21:32:44.324Z" }, + { url = "https://files.pythonhosted.org/packages/4b/92/e7bb136ad6b5352603732cf907ef862ca103f20f2031c1735a46300c20c9/cffi-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde", size = 184728, upload-time = "2026-07-06T21:32:45.683Z" }, { url = "https://files.pythonhosted.org/packages/c3/c0/d1ec30ffb370f748f2fb54425972bfef9871e0132e82fb589c46b6676049/cffi-2.1.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d", size = 214815, upload-time = "2026-07-06T21:32:48.557Z" }, { url = "https://files.pythonhosted.org/packages/1b/dc/5620cf930688be01f2d673804291de757a934c90b946dbdc3d84130c2ea4/cffi-2.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7", size = 222429, upload-time = "2026-07-06T21:32:49.848Z" }, { url = "https://files.pythonhosted.org/packages/4b/a4/77b53abbf7a1e0beb9637edbef2a94d15f9c822f591e85d439ffd91519a6/cffi-2.1.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b", size = 210315, upload-time = "2026-07-06T21:32:51.221Z" }, @@ -178,6 +197,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dd/2c/400ea43e721727dca8a65c4521390e9196757caba4a45643acb2b63271b8/cffi-2.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6", size = 180088, upload-time = "2026-07-06T21:33:02.278Z" }, { url = "https://files.pythonhosted.org/packages/96/88/a996879e2eeccb815f6e3a5967b12a308257412acec882039d386bd2aa7b/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda", size = 194331, upload-time = "2026-07-06T21:33:03.697Z" }, { url = "https://files.pythonhosted.org/packages/58/85/7ae00d5c8dd6266f4e944c3db630f3c5c9a98b61d469c714d848b1d8138a/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b", size = 196966, upload-time = "2026-07-06T21:33:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e9/45c3a76ad8d43ad9261f4c95436da61128d3ca545d72b9612c0ab5be0b1c/cffi-2.1.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:15faec4adfff450819f3aee0e2e02c812de6edb88203aa58807955db2003472a", size = 184795, upload-time = "2026-07-06T21:33:06.699Z" }, + { url = "https://files.pythonhosted.org/packages/84/4c/82f132cb4418ee6d953d982b19191e87e2a6372c8a4ce36e50b69d6ade4a/cffi-2.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:716ff8ec22f20b4d988b12884086bcef0fc99737043e503f7a3935a6be99b1ea", size = 184746, upload-time = "2026-07-06T21:33:08.071Z" }, { url = "https://files.pythonhosted.org/packages/a0/1c/4ed5a0e5bdca6cbc275556de3328dd1b76fd0c11cc13c88fe66d1d8715f2/cffi-2.1.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:63960549e4f8dc41e31accb97b975abaecfc44c03e396c093a6436763c2ea7db", size = 214747, upload-time = "2026-07-06T21:33:09.671Z" }, { url = "https://files.pythonhosted.org/packages/3a/a6/e879bb68cc23a2bc9ba8f4b7d8019f0c2694bad2ab6c4a3701d429439f58/cffi-2.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff067a8d8d880e7809e4ac88eb009bb848870115317b306666502ccad30b147f", size = 222392, upload-time = "2026-07-06T21:33:10.896Z" }, { url = "https://files.pythonhosted.org/packages/88/f6/01890cfd63c08f8eb96a8319b0443690197d240a8bd6346048cf7bde9190/cffi-2.1.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3b926723c13eba9f81d2ef3820d63aeceec3b2d4639906047bf675cb8a7a500d", size = 210285, upload-time = "2026-07-06T21:33:12.251Z" }, @@ -190,6 +211,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/af/34fee85c48f8d94efc8597bc09470c9dd274c145f1c12e0fbc6ab6d38d74/cffi-2.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:2282cd5e38aa8accd03e99d1256af8411c84cdbee6a89d841b563fdbd1f3e50f", size = 180114, upload-time = "2026-07-06T21:33:22.515Z" }, { url = "https://files.pythonhosted.org/packages/d8/f0/81478e482afa03f6d18dc8f2afb5edc45b3080853b634b5ed91961be0998/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a", size = 194142, upload-time = "2026-07-06T21:33:23.657Z" }, { url = "https://files.pythonhosted.org/packages/7d/95/8de304305cd9204974b0ca051b86d307cafca13aa575a0ef1b44d92c0d8c/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3", size = 196819, upload-time = "2026-07-06T21:33:25.007Z" }, + { url = "https://files.pythonhosted.org/packages/20/71/7c8372d30e42415602ed9f268f7cfd66f1b855fed881ecd168bcb45dbc0b/cffi-2.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1ff3456eab0d889592d1936d6125bbfbc7ae4d3354a700f8bd80450a66445d4d", size = 184965, upload-time = "2026-07-06T21:33:26.605Z" }, + { url = "https://files.pythonhosted.org/packages/d6/5c/584e626835f0375c928176c04137c96927165cb8733cdb3150ec04e5ee5e/cffi-2.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c4165821e131d6d4ca444347c2b694e2311bcfa3fe5a861cc72968f28867beac", size = 184952, upload-time = "2026-07-06T21:33:27.823Z" }, { url = "https://files.pythonhosted.org/packages/2e/d2/065fcae1c73979fac8e054462478d0ff8a29c40cdc2ed7ea5676a061df53/cffi-2.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:276f20fffd7b396e12516ba8edf9509210ac248cbbc5acbc39cd512f9f59ebe6", size = 222353, upload-time = "2026-07-06T21:33:29.178Z" }, { url = "https://files.pythonhosted.org/packages/ed/a5/e8bbb1ce5b3ac2f53ad6a10bde44318a5a8d99d4f4a000d44a6e39aeb3e4/cffi-2.1.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7d5980a3433d4b71a5e120f9dd551403d7824e31e2e67124fe2769c404c06913", size = 210051, upload-time = "2026-07-06T21:33:30.534Z" }, { url = "https://files.pythonhosted.org/packages/28/ed/c127d3ac36e899c965e3361357c3befacd6578c03f40125183e41c3b219e/cffi-2.1.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6ca4919c6e4f89aa99c42510b42cf54596892c00b3f9077f6bdd1505e24b9c8d", size = 208630, upload-time = "2026-07-06T21:33:31.753Z" }, @@ -199,6 +222,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0f/6f/ade5ce9863a57992a6ea3d0d10d7e29b8749fc127204b3d493d667b2815f/cffi-2.1.0-cp314-cp314-win32.whl", hash = "sha256:1854b724d00f6654c742097d5387569021be12d3a0f770eae1df8f8acfcc6acd", size = 177723, upload-time = "2026-07-06T21:33:51.626Z" }, { url = "https://files.pythonhosted.org/packages/41/de/92b9eeed4ae4a21d6fd9b2a2c8505cbed573299902ea73981cc13f7ff62c/cffi-2.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb", size = 187937, upload-time = "2026-07-06T21:33:53.403Z" }, { url = "https://files.pythonhosted.org/packages/2e/1a/cc6ae6c2913a03aab8898eee57963cf1035b8df5872ed8b9115fcc7e2be8/cffi-2.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:7d28dff1db6764108bc30788d85d61c876beff416d9a49cb9dd7c5a9f34f5804", size = 183001, upload-time = "2026-07-06T21:33:54.74Z" }, + { url = "https://files.pythonhosted.org/packages/14/f0/134c00ce0779ec86dea2aa1aac69339c2741a8045072676763512363a2ea/cffi-2.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7ea6b3e2c4250ff1de21c630fe72d0f63eb95c2c32ffbf64a358cf4a8836d714", size = 188538, upload-time = "2026-07-06T21:33:36.792Z" }, + { url = "https://files.pythonhosted.org/packages/50/d8/3b86aba791cb610d24e8a3e1b2cd529e71fa15096b04e4d4e360049d4a4c/cffi-2.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6af371f3767faeffc6ac1ef57cdfd25844403e9d3f476c5537caee499de96376", size = 188230, upload-time = "2026-07-06T21:33:38.011Z" }, { url = "https://files.pythonhosted.org/packages/14/d0/117dcd9209255ad8571fbc8c92ef32593a1d294dcec91ddc4e4db50606f2/cffi-2.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb4e8997a49aa2c08a3e43c9045d224448b8941d88e7ac163c7d383e560cbf98", size = 223899, upload-time = "2026-07-06T21:33:39.514Z" }, { url = "https://files.pythonhosted.org/packages/b6/3d/f20f8b886b254e3ad10e15cd4186d3aed49f3e6a35ab37aab9f8f25f7c03/cffi-2.1.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bf01d8c84cbea96b944c73b22182e6c7c432b3475632b8111dbfdc95ddad6e13", size = 211652, upload-time = "2026-07-06T21:33:40.851Z" }, { url = "https://files.pythonhosted.org/packages/28/3b/fad54de07260b93ddeef4b96d0131d57ea900675df1d410ae1deee52d7a6/cffi-2.1.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:33eb1ad83ebe8f313e0df035c406227d55a79456704a863fad9842136af5ad7d", size = 210755, upload-time = "2026-07-06T21:33:42.183Z" }, @@ -210,6 +235,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/dd/0c7dbf815a579ff005008a2d815a55d6bb047c349eef536d9dc53d3f0a8d/cffi-2.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:510aeeeac94811b138077451da1fb18b308a5feab47dd2b603af55804155e1c8", size = 186404, upload-time = "2026-07-06T21:33:50.309Z" }, { url = "https://files.pythonhosted.org/packages/55/c7/8c8c50cb11c6750051daf12164098a9a6f027ac4356967fd4d800a07f242/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:2e9dabb9abcb7ad15938c7196ad5c1718a4e6d33cc79b4c0209bdb64c4a54a5c", size = 194121, upload-time = "2026-07-06T21:33:56.109Z" }, { url = "https://files.pythonhosted.org/packages/99/e2/67680bf19a6b60d2bb7ff83baefa2a4c3d2d7dc0f3277034b802e1fc504c/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:37f525a7e7e50c017fdebe58b787be310ad59357ae43a053943a6e1a6c526001", size = 196820, upload-time = "2026-07-06T21:33:57.288Z" }, + { url = "https://files.pythonhosted.org/packages/ed/da/4bbe583a3b3a5c8c60892124fe17f3fa3656523faf0d3484eae90f091853/cffi-2.1.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:95f2954c2c9473d892eca6e0409f3568b37ab62a8eedb122461f73cc273476e3", size = 184936, upload-time = "2026-07-06T21:33:58.765Z" }, + { url = "https://files.pythonhosted.org/packages/e5/4b/1f4c36ab273980d7aa75bb126ea4f8971f24a96108acad3a0a084028c57b/cffi-2.1.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:cdf2448aab5f661c9315308ec8b93f4e8a1a67a3c733f8631067a2b67d5913dc", size = 185045, upload-time = "2026-07-06T21:34:00.085Z" }, { url = "https://files.pythonhosted.org/packages/ef/c3/ad299dc38f3583f8d916b299f028af418a9ec98bc695fcbebeae7420691c/cffi-2.1.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:90bec57cf82089383bd06a605b3eb8daebf7e5a668520beaf6e327a83a947699", size = 222342, upload-time = "2026-07-06T21:34:01.814Z" }, { url = "https://files.pythonhosted.org/packages/eb/d8/df4543cc087245044ed02ef3ad8e0a26619d0075ac7a77a12dc81177851b/cffi-2.1.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6274dcb2d15cef48daa73ed1be5a40d501d74dccd0cd6db364776d12cb6ba022", size = 210073, upload-time = "2026-07-06T21:34:03.255Z" }, { url = "https://files.pythonhosted.org/packages/2c/0e/fac738d73728c6cea2a88a2883dca54892496cbba88a1dc1f2909cb8a6f5/cffi-2.1.0-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2b71d409cccee78310ab5dec549aed052aaea483346e282c7b02362596e01bb0", size = 208551, upload-time = "2026-07-06T21:34:04.433Z" }, @@ -219,6 +246,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/d8/772b8259bf75749adffb1c546828978381fb516f60cf701f6c83daf60c85/cffi-2.1.0-cp315-cp315-win32.whl", hash = "sha256:0a42c688d19fca6e095a53c6a6e2295a5b050a8b289f109adab02a9e61a25de6", size = 177696, upload-time = "2026-07-06T21:34:26.355Z" }, { url = "https://files.pythonhosted.org/packages/2f/dd/afa2191fc6d57fedd26e5844a2fe2fcc0bbfa00961bbaa5a41e4921e7cca/cffi-2.1.0-cp315-cp315-win_amd64.whl", hash = "sha256:bccbbb5ee76a61f9d99b5bf3846a51d7fca4b6a732fe46f89295610edaf41853", size = 187914, upload-time = "2026-07-06T21:34:27.58Z" }, { url = "https://files.pythonhosted.org/packages/05/ef/6cd4f8c671517162379dc79cfae5aea9106bc38abb89628d5c16adf6a838/cffi-2.1.0-cp315-cp315-win_arm64.whl", hash = "sha256:8d35c139744adb3e727cd51b1a18324bbe44b8bd41bf8322bca4d41289f48eda", size = 183004, upload-time = "2026-07-06T21:34:28.905Z" }, + { url = "https://files.pythonhosted.org/packages/11/b6/12fc55092817a5faa26fb8c40c7f9d662e11a46ee248c137aafc42517d92/cffi-2.1.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:f9912624a0c0b834b7520d7769b3644453aabc0a7e1c839da7359f050750e9bc", size = 188378, upload-time = "2026-07-06T21:34:09.926Z" }, + { url = "https://files.pythonhosted.org/packages/8d/2e/cdac88979f295fde5daa69622c7d2111e56e7ceb94f211357fbe452339e4/cffi-2.1.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:df92f2aba50eb4d96718b68ef76f2e57a57b54f2fa62333496d16c6d585a85ca", size = 188319, upload-time = "2026-07-06T21:34:11.101Z" }, { url = "https://files.pythonhosted.org/packages/e0/27/1d0b408497e41a74795af122d7b603c418c5fed0171450f899afd04e594f/cffi-2.1.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0520e1f4c35f44e209cbbb421b67eec42e6a157f59444dfb6058874ff3610e5d", size = 223904, upload-time = "2026-07-06T21:34:12.606Z" }, { url = "https://files.pythonhosted.org/packages/8b/31/e115c985105dd7ffb32444505f18ceb874bb42d992af05d5dced7ecf1980/cffi-2.1.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3681e031db29958a7502f5c0c9d6bbc4c36cb20f7b104086fa642d1799631ff8", size = 211554, upload-time = "2026-07-06T21:34:13.987Z" }, { url = "https://files.pythonhosted.org/packages/5a/67/9e6e09409336d9e515c58367e7cfcf4f89df06ad25252675595a58eb59d5/cffi-2.1.0-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:762f99479dcb369f60ab9017ad4ab97a36a1dd7c1ee5a3b15db0f4b8659120cd", size = 210795, upload-time = "2026-07-06T21:34:15.972Z" }, @@ -270,6 +299,7 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/12/45/870e7f4bef50e5f53b9f51d4428aee5290eedf58ba443f16b1ebb7ab8e66/cryptography-48.0.1.tar.gz", hash = "sha256:266f4ee051abb2f725b74ef8072b521ce1feacf685a3364fa6a6b45548db791a", size = 832989, upload-time = "2026-06-09T22:32:31.8Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/bc/ee4137cbbe105652c0ee4252792b78fc8e7afa4b8e61d9d5dc05a7f45731/cryptography-48.0.1-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:3e4a1a3232eef2e6c732827d5722db29a0cc8b27af2a4d865b094cf954be9ca1", size = 8008324, upload-time = "2026-06-09T22:31:00.702Z" }, { url = "https://files.pythonhosted.org/packages/d5/85/6379d42181bfc713094f081360fc5784d6c816b599d45e7f082502d173ce/cryptography-48.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:32143b24adb918f078134e1e230f1eb8cc04886b92c28b5f0041aaf3e5699225", size = 4696243, upload-time = "2026-06-09T22:32:33.446Z" }, { url = "https://files.pythonhosted.org/packages/9c/87/c85d147b53323c7eb4d850920c8901377323c2a0ff8d79c262d4fee89aa2/cryptography-48.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0d27a5696721ef7a672b8c810f6aded391058e0b9486e63e6d93baf765da691", size = 4713235, upload-time = "2026-06-09T22:31:40.141Z" }, { url = "https://files.pythonhosted.org/packages/79/58/67cbf8cf1ee7c54b439ca07bbecf8362c07afc11a3724fea70f745784add/cryptography-48.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb86ce1af36fe65041b6db9a8bb064ee621a7e5fded0f80d475ec243477cd242", size = 4702323, upload-time = "2026-06-09T22:31:42.191Z" }, @@ -281,6 +311,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9f/13/8b765e2e12b07c74941caadb9d1c8fdc006c4dfbf2b8f2d610519758954d/cryptography-48.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:0ee6ea481db1ab889cba043ec1eda17bb9c1ea79db6722f779c3667f9f70322f", size = 4745488, upload-time = "2026-06-09T22:32:30.07Z" }, { url = "https://files.pythonhosted.org/packages/2e/aa/48972bce55049b32a94f4907eda4d75fa385aad8a39506cc2fc72196ecf0/cryptography-48.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f2ceef93cb096aa3c4cc4b5c94ca6131f9196d28c64d6111533402a9b2054d41", size = 4830256, upload-time = "2026-06-09T22:31:43.868Z" }, { url = "https://files.pythonhosted.org/packages/47/a2/e5079a032fb85cf6005046ca92bbd78b0c82dad2b5751ab8c311659da06f/cryptography-48.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9bd3f92d76217892b15df84ca256c2c113d386fdda7a7d8691aeeced976507c6", size = 4979117, upload-time = "2026-06-09T22:31:05.845Z" }, + { url = "https://files.pythonhosted.org/packages/b7/a0/8f50cae9c74e718ed769d63ed5c74bd0ea830c9550a74629cebd1b9c7bc7/cryptography-48.0.1-cp311-abi3-win32.whl", hash = "sha256:b9a32b876490d66c8bcc9963ef220199569748434ab01a9d6aaeabf88e7f5158", size = 3304154, upload-time = "2026-06-09T22:32:16.845Z" }, + { url = "https://files.pythonhosted.org/packages/c5/69/0572c77dbace6fef72f33755bd52ea399c71367250d366237f8691826b9e/cryptography-48.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:39489bfca54c7a1f6b297efcd8bc608ab92d16c4ca631b0cad4da46724588b24", size = 3817138, upload-time = "2026-06-09T22:32:00.388Z" }, + { url = "https://files.pythonhosted.org/packages/42/06/3e768b4c3bc78201583fa35a0e18f640dd782ff41afba88f8545481a8874/cryptography-48.0.1-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:f817adc181390bd54f2f700107a7419040fb7c1bdf2fc26f36551a06a68c3345", size = 7989830, upload-time = "2026-06-09T22:31:07.8Z" }, { url = "https://files.pythonhosted.org/packages/8a/13/6476736484b94041110c8340a3eb63962fea4975baea8cb4a512adb44d4d/cryptography-48.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d5d30989c6917b478b5817902e85fddaea2261efa8648383d965381ccb9e1ac4", size = 4689201, upload-time = "2026-06-09T22:31:09.745Z" }, { url = "https://files.pythonhosted.org/packages/79/62/65a87f34d2a431546e2509b85d55e8c90df86d668f6731da64d538512ac2/cryptography-48.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:df637c05205ea7c1d7fbcbe54bbfea648a52951155f997af13d895d0ecc96991", size = 4702822, upload-time = "2026-06-09T22:32:24.409Z" }, { url = "https://files.pythonhosted.org/packages/7f/59/810b5204b0a9b10f4b6bc06bd551a8b609803cd931806bc3b71884b225e5/cryptography-48.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:869c3b8a53bfe27147832df48b32adadf558249d50e76cb3769d40e986b13265", size = 4694875, upload-time = "2026-06-09T22:32:08.737Z" }, @@ -292,6 +325,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9f/04/618f4115cfc0add0838c82507aa18a346089428da8653ad38b3ff36f5cb3/cryptography-48.0.1-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:b4e391975f038e66432328639620a4aff2d307513b004f1ca06d6225bced815c", size = 4736660, upload-time = "2026-06-09T22:32:12.676Z" }, { url = "https://files.pythonhosted.org/packages/24/9c/06e062462a0de28a3b3911322eded4c16deb9f441b1b7575d3dc59488ab5/cryptography-48.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:42fcd8e26fe555d9b3577a135f5091fefa0aa4e99129c23fb56787a1bd4ada72", size = 4822229, upload-time = "2026-06-09T22:31:17.062Z" }, { url = "https://files.pythonhosted.org/packages/f4/be/0561971eaaee4b8a0e7d5113c536921063ab91aaf23278ac374eaf881e11/cryptography-48.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c1400da5e32a43253392277eac7490a60e497d810a63dd5608d71bbd7af507c9", size = 4966364, upload-time = "2026-06-09T22:31:32.842Z" }, + { url = "https://files.pythonhosted.org/packages/a4/27/728c77876f12b000820b69ae490f3c4083775e79e07827e9e60be07ad209/cryptography-48.0.1-cp314-cp314t-win32.whl", hash = "sha256:0df56b056bc17c1b7d6821dfa65216e62bd232d8ab05eb3db44e71d235651471", size = 3278498, upload-time = "2026-06-09T22:31:29.154Z" }, + { url = "https://files.pythonhosted.org/packages/06/e3/79a612c6d7b1e6ee0edd43633d53035bec2cfb78c82b76f7864f39e36f34/cryptography-48.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:9de21387aa95e2a895823d0745b430bed4f33503ba9ab5e0b5311f33e37d66d2", size = 3798790, upload-time = "2026-06-09T22:31:56.697Z" }, + { url = "https://files.pythonhosted.org/packages/ca/6c/00fa2a95997164c8b2072ce327c23d4ab20809ccc323ea5fab91e53a4bba/cryptography-48.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:4fdc69f8e4316bcf0c8c8ec1f26f285d12e8142d88d96c876a59a03be3f6ae67", size = 7987408, upload-time = "2026-06-09T22:32:20.777Z" }, { url = "https://files.pythonhosted.org/packages/b0/d9/45f309a7e4e5f3f8f121d6d3be9e94024a7726ec598d6e08ae04edb2f04d/cryptography-48.0.1-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48fe40804d4caa2288f24e70ca8c64c42dd826da0ad7e4f1b41b2128d679e6c8", size = 4690196, upload-time = "2026-06-09T22:31:54.74Z" }, { url = "https://files.pythonhosted.org/packages/5f/9f/a1bc8bcc798811b8527eb374bbccf30a3f3e806829d967118222bf1125eb/cryptography-48.0.1-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:86be3b1b0b6bf09482fb50a979c508d2950ed95f5621ec77f4e385962006b83a", size = 4696782, upload-time = "2026-06-09T22:31:45.615Z" }, { url = "https://files.pythonhosted.org/packages/66/c2/81a4fb4e4373c500bb526bc337ac5719dd31dd15b970b84a238168c6aa08/cryptography-48.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4ab0a343c807bbcd90c971cd1ecf072937cd01847a9e002bef88fb47ac6be577", size = 4696618, upload-time = "2026-06-09T22:31:11.564Z" }, @@ -303,10 +339,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/68/ab/8aaa12e4516ec4464033ab79b6f3b592bd5a92102467c4ace8a0d970203f/cryptography-48.0.1-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:b74ca3b8e5ecdd833bf6a002ca41b4793bb27fb8f1c06ffaf2643c9e9140e31b", size = 4731388, upload-time = "2026-06-09T22:32:04.019Z" }, { url = "https://files.pythonhosted.org/packages/1b/24/50027ea4dca85ec1f40688f3c24fb32ccacd520583c9592c3cc95628e6fb/cryptography-48.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2c37f2461406063b417837f5f3daab668652acd82423efcd7f0a9f04be972de1", size = 4824186, upload-time = "2026-06-09T22:32:18.707Z" }, { url = "https://files.pythonhosted.org/packages/52/41/04cb5eb17085ade6f50cc611fb657df6a0f5885350de8764ece89c050197/cryptography-48.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:86fe77abb1bd87afb251d4d02ada7ecf53a32cee9b67d976abb2e45a13297475", size = 4964539, upload-time = "2026-06-09T22:31:18.793Z" }, + { url = "https://files.pythonhosted.org/packages/36/bf/ed70785c496e89d7e73b7cda2d21f2447fd6d4e821714b8d04ff217fed92/cryptography-48.0.1-cp39-abi3-win32.whl", hash = "sha256:6b2c0c3e6ccf3ade7750f836ef3ee36eea250cc467d45c256895573ac08cc6f1", size = 3282307, upload-time = "2026-06-09T22:30:53.162Z" }, + { url = "https://files.pythonhosted.org/packages/b3/ff/371ea7d252656ee1eb6d83eeeef3d1d0c6baf1d6497687d081ea03814670/cryptography-48.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:9a49ca6c81417f6a5edb50375a60cccdd70fa0a91a5211829dbea74eba94d2ac", size = 3793408, upload-time = "2026-06-09T22:32:15.191Z" }, + { url = "https://files.pythonhosted.org/packages/a9/d3/eb4e394e587341fdad09a09101fa76478ead3a78b0ad63e55c22f0d75c02/cryptography-48.0.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:08a597acce1ff37f347400087776599e2348a3a8bc53b44120e463cd274efe4a", size = 3951747, upload-time = "2026-06-09T22:31:23.871Z" }, { url = "https://files.pythonhosted.org/packages/e0/4a/3f43451b4f858bfceaaaffc649e6e787e8d4fb332a1d443af39ab02cc8f1/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:735824ec41b7f74a7c45fb1591349333e4c696cb6c044e5f46356e560143e4cd", size = 4641226, upload-time = "2026-06-09T22:31:02.532Z" }, { url = "https://files.pythonhosted.org/packages/73/4e/855584c2c23b09e4ce2d3b9c30e983e679cd60b068c513c6bbdb91e11782/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:92a46e1d638daa264ba2971c0b0489c9409787943efae4d60ffda3d091ef832c", size = 4668958, upload-time = "2026-06-09T22:32:06.213Z" }, { url = "https://files.pythonhosted.org/packages/42/3b/d35750e41d803d1e516fd6d6011f065424924da7af1748cef4cc9cb3ede1/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7e234ac052af99f2700826a5c29ea99d9c1b1f80341cde62d11c8154dc8e0bd9", size = 4640793, upload-time = "2026-06-09T22:32:26.331Z" }, { url = "https://files.pythonhosted.org/packages/ca/aa/cdb7181fe865285e87e96825aaab239400f1de0c3bfba9bd9769b79f1a92/cryptography-48.0.1-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:33842cf0888951cef5bc7ac724ab844a42044c1727b967b7f8997289a0464f92", size = 4668505, upload-time = "2026-06-09T22:31:27.534Z" }, + { url = "https://files.pythonhosted.org/packages/5d/8c/ce3823c06c2804f194f9e64f0d67fa3f4094a39f2bb1a990cd03603af8fc/cryptography-48.0.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6184ca7b174f28d7c703f1290d4b297217c45355f77a98f67e9b7f14549ac54a", size = 3742204, upload-time = "2026-06-09T22:31:34.773Z" }, ] [[package]] @@ -332,7 +372,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -348,6 +388,7 @@ dependencies = [ [package.optional-dependencies] testing = [ + { name = "asyncssh" }, { name = "hatch" }, { name = "pre-commit" }, { name = "pytest" }, @@ -363,6 +404,7 @@ testing = [ [package.metadata] requires-dist = [ + { name = "asyncssh", marker = "extra == 'testing'" }, { name = "hatch", marker = "extra == 'testing'" }, { name = "pre-commit", marker = "extra == 'testing'" }, { name = "pytest", marker = "extra == 'testing'", specifier = ">8.0" }, From f153271a5f2feb4369b73ddf487c88d8c5c99439 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 08:04:26 +0200 Subject: [PATCH 08/91] feat: run ssh gateways on the Trio path Route ssh gateways through the Trio host: `ssh -C [-F cfg] ''` spawns the uv-provisioned worker module on the remote, then the coordinator does the usual b"1" handshake and attaches a Trio session. HostNotFound is raised when ssh exits 255. The popen and ssh factories now share _open_trio_gateway (spawn + handshake + attach); ssh_trio_args builds the shell-quoted remote worker command. test_ssh_roundtrip is parametrized over the trio and legacy paths against the in-process asyncssh server. test_sshconfig_config_parsing (white-box over legacy Popen2IOMaster) is pinned to EXECNET_TRIO_HOST=0, with a new ssh_trio_args test covering -F on the Trio path. Co-Authored-By: Claude Opus 4.8 --- doc/implnotes.rst | 66 +++++++++++++++++------------- src/execnet/_trio_host.py | 85 +++++++++++++++++++++++++++++++++------ src/execnet/multi.py | 2 + testing/test_gateway.py | 13 ++++++ testing/test_ssh_local.py | 8 +++- 5 files changed, 131 insertions(+), 43 deletions(-) diff --git a/doc/implnotes.rst b/doc/implnotes.rst index 84492656..84bcc065 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -15,34 +15,44 @@ to and from InputOutput objects. The details of this protocol are locally defined in this module. There is no need for standardizing or versioning the protocol. -Trio host-thread IO (popen / import bootstrap) ----------------------------------------------- - -For local same-interpreter ``popen`` gateways, Message protocol IO -runs inside a dedicated OS thread hosting a Trio event loop -(``execnet._trio_host.TrioHost``). No source is sent over the wire: -the worker is launched as ``python -m execnet._trio_worker`` and -imports the installed ``execnet`` + ``trio`` (a rough major/minor -version check guards against an incompatible install): - -* Coordinator: ``trio.lowlevel.open_process`` plus async framed - reader/writer tasks per gateway (one host thread per ``Group``). - It waits for the worker's ``b"1"`` handshake before starting the - Message protocol. -* Worker: ``serve_popen_trio`` adopts stdio pipe fds into Trio - streams and writes the handshake byte; ``remote_exec`` is - scheduled from the Trio nursery (``trio.to_thread`` for ``thread``, - main-thread handoff for ``main_thread_only``). - -Sync ``Channel`` / ``Gateway`` APIs are unchanged. Sends from -non-host threads wait until the frame is written (so abrupt -``os._exit`` cannot drop queued data). Sends from the Trio host -thread (receiver callbacks) only enqueue, to avoid deadlocking -the writer task. - -Disable with ``EXECNET_TRIO_HOST=0``. Other gateway types -(``ssh``, ``socket``, ``via``, ``python=…``, greenlet execmodels) -still use the legacy thread receiver and sync ``Popen`` path. +Trio host-thread IO +------------------- + +``popen`` and ``ssh`` gateways run their Message protocol IO inside a +dedicated OS thread hosting a Trio event loop +(``execnet._trio_host.TrioHost``). No source is sent over the wire; the +worker is launched as ``python -m execnet._trio_worker`` and imports the +installed ``execnet`` + ``trio`` (a rough major/minor version check guards +against an incompatible install). How the worker environment is obtained +depends on the target: + +* Same-interpreter ``popen`` -> ``sys.executable -m execnet._trio_worker``. +* A ``python=`` interpreter that already has execnet -> that interpreter + directly (so ``sys.executable`` is preserved). +* A bare ``python=`` interpreter or an ``ssh`` remote -> provisioned via + ``uv`` (``execnet._provision``): ``uv run --with `` where ```` + is ``execnet==`` for a released coordinator or a locally-built, + version-cached wheel for a dev coordinator. + +Coordinator and worker roles: + +* Coordinator: ``trio.lowlevel.open_process`` (directly, or wrapped in + ``ssh``) plus async framed reader/writer tasks per gateway (one host + thread per ``Group``). It waits for the worker's ``b"1"`` handshake + before starting the Message protocol. +* Worker: ``serve_popen_trio`` adopts stdio pipe fds into Trio streams and + writes the handshake byte; ``remote_exec`` is scheduled from the Trio + nursery (``trio.to_thread`` for ``thread``, main-thread handoff for + ``main_thread_only``). + +Sync ``Channel`` / ``Gateway`` APIs are unchanged. Sends from non-host +threads wait until the frame is written (so abrupt ``os._exit`` cannot drop +queued data). Sends from the Trio host thread (receiver callbacks) only +enqueue, to avoid deadlocking the writer task. + +Disable with ``EXECNET_TRIO_HOST=0``. Other gateway types (``socket``, +``via``, greenlet execmodels) still use the legacy thread receiver and sync +``Popen`` path. Legacy thread model ------------------- diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 42b45047..68755d50 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -551,24 +551,19 @@ def kill(self) -> None: return -def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: - """Create a popen Gateway on the Trio IO path. +def _open_trio_gateway( + group: Any, spec: Any, args: list[str], *, remoteaddress: str | None = None +) -> Gateway: + """Spawn ``args``, do the worker handshake, and attach a Trio session. - Same-interpreter popen launches ``python -m execnet._trio_worker`` directly; - a foreign interpreter (``python=``) is provisioned via ``uv``. Either way the - worker imports execnet + trio; nothing is sent over the wire to bootstrap it. + Shared by the popen and ssh factories; ``args`` already encodes how the + worker is launched (direct module, uv-provisioned, or wrapped in ssh). """ import execnet - from . import _provision + from .gateway_bootstrap import HostNotFound host: TrioHost = group._ensure_trio_host() - if spec.python and not _provision.target_has_execnet(spec.python): - # bare interpreter: provision execnet + trio via uv - args = _provision.uv_worker_argv(spec) - else: - # same interpreter, or a python= that already has execnet - args = popen_module_args(spec) async def _create_and_attach() -> Gateway: process = await open_popen_process(args) @@ -577,6 +572,16 @@ async def _create_and_attach() -> Gateway: ack = await async_io.read_exact(1) if ack != b"1": raise EOFError(f"bad bootstrap handshake: {ack!r}") + except EOFError: + with trio.move_on_after(5): + code = await process.wait() + # ssh exits 255 when it cannot reach/authenticate the host. + if remoteaddress is not None and code == 255: + raise HostNotFound(remoteaddress) from None + with trio.move_on_after(5): + process.kill() + await process.wait() + raise except BaseException: with trio.move_on_after(5): process.kill() @@ -586,7 +591,61 @@ async def _create_and_attach() -> Gateway: gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) session = await host.start_session(gw, async_io, process=process) gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session) + gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) return gw return host.call(_create_and_attach) + + +def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: + """Create a popen Gateway on the Trio IO path. + + Same-interpreter popen launches ``python -m execnet._trio_worker`` directly; + a foreign interpreter (``python=``) is provisioned via ``uv``. Either way the + worker imports execnet + trio; nothing is sent over the wire to bootstrap it. + """ + from . import _provision + + if spec.python and not _provision.target_has_execnet(spec.python): + # bare interpreter: provision execnet + trio via uv + args = _provision.uv_worker_argv(spec) + else: + # same interpreter, or a python= that already has execnet + args = popen_module_args(spec) + return _open_trio_gateway(group, spec, args) + + +def ssh_trio_args(spec: Any) -> list[str]: + """``ssh [-F cfg] ''`` for the Trio ssh path. + + The remote runs the uv-provisioned worker module; the worker command is + shell-quoted for the remote shell. + """ + import shlex + + from . import _provision + + remote_command = shlex.join(_provision.uv_worker_argv(spec)) + args = ["ssh", "-C"] + if getattr(spec, "ssh_config", None): + args += ["-F", spec.ssh_config] + assert spec.ssh is not None + args += spec.ssh.split() + args.append(remote_command) + return args + + +def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: + """Create an ssh Gateway on the Trio IO path (uv-provisioned worker).""" + args = ssh_trio_args(spec) + return _open_trio_gateway(group, spec, args, remoteaddress=spec.ssh) + + +def should_use_trio_ssh(spec: Any) -> bool: + """Trio path for ssh gateways (worker provisioned on the remote via uv).""" + if not trio_host_enabled(): + return False + if not getattr(spec, "ssh", None): + return False + execmodel = getattr(spec, "execmodel", None) + return execmodel in (None, "thread", "main_thread_only") diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 687f1840..29cea870 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -156,6 +156,8 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: if _trio_host.should_use_trio_popen(spec): gw = _trio_host.makegateway_popen_trio(self, spec) + elif _trio_host.should_use_trio_ssh(spec): + gw = _trio_host.makegateway_ssh_trio(self, spec) elif spec.via: assert not spec.socket master = self[spec.via] diff --git a/testing/test_gateway.py b/testing/test_gateway.py index 83a50f44..f7f882ae 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -384,6 +384,8 @@ class TestSshPopenGateway: def test_sshconfig_config_parsing( self, monkeypatch: pytest.MonkeyPatch, makegateway: Callable[[str], Gateway] ) -> None: + # white-box test of the legacy Popen2IOMaster arg construction + monkeypatch.setenv("EXECNET_TRIO_HOST", "0") l = [] monkeypatch.setattr( gateway_io, "Popen2IOMaster", lambda *args, **kwargs: l.append(args[0]) @@ -396,6 +398,17 @@ def test_sshconfig_config_parsing( i = popen_args.index("-F") assert popen_args[i + 1] == "qwe" + def test_ssh_trio_args_include_config( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from execnet import _provision + from execnet import _trio_host + + monkeypatch.setattr(_provision, "coordinator_requirement", lambda: "execnet") + args = _trio_host.ssh_trio_args(execnet.XSpec("ssh=xyz//ssh_config=qwe")) + assert args[args.index("-F") + 1] == "qwe" + assert "xyz" in args + def test_sshaddress(self, gw: Gateway, specssh: execnet.XSpec) -> None: assert gw.remoteaddress == specssh.ssh diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py index 196c5d76..72dea348 100644 --- a/testing/test_ssh_local.py +++ b/testing/test_ssh_local.py @@ -9,7 +9,6 @@ from __future__ import annotations import asyncio -import os import shutil import sys import threading @@ -134,7 +133,12 @@ def ssh_config(ssh_server: SSHServerThread, tmp_path) -> str: return path -def test_ssh_roundtrip(ssh_config: str) -> None: +@pytest.mark.parametrize("trio_host", ["1", "0"], ids=["trio", "legacy"]) +def test_ssh_roundtrip( + ssh_config: str, monkeypatch: pytest.MonkeyPatch, trio_host: str +) -> None: + # trio=1 provisions the worker over ssh with uv; legacy=0 source-copies it. + monkeypatch.setenv("EXECNET_TRIO_HOST", trio_host) group = execnet.Group() try: gw = group.makegateway( From 5197b1eee591461b959fea9be07363e50f91d630 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 08:43:07 +0200 Subject: [PATCH 09/91] feat: ship the execnet wheel to ssh remotes for dev coordinators A dev coordinator's wheel is not on the remote filesystem, so the ssh remote command becomes a POSIX-sh prelude that reads the wheel bytes from stdin (`head -c N` into a temp dir) and execs uv against it. The coordinator streams those bytes as a preamble before the Message protocol; _open_trio_gateway grew a `preamble` argument. Released coordinators still use `uv run --with execnet==` with no shipping. Also collapse the worker CLI contract: id, execmodel and coordinator version now travel as a single JSON argument (_provision.worker_cli_arg) consumed by _trio_worker._main, instead of scattered positional args built in three launchers. Co-Authored-By: Claude Opus 4.8 --- doc/implnotes.rst | 10 ++++- src/execnet/_provision.py | 87 +++++++++++++++++++++++++++---------- src/execnet/_trio_host.py | 45 +++++++++---------- src/execnet/_trio_worker.py | 18 ++++---- testing/test_gateway.py | 10 ++++- testing/test_provision.py | 40 +++++++++++++++++ 6 files changed, 152 insertions(+), 58 deletions(-) create mode 100644 testing/test_provision.py diff --git a/doc/implnotes.rst b/doc/implnotes.rst index 84bcc065..29df051c 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -32,7 +32,15 @@ depends on the target: * A bare ``python=`` interpreter or an ``ssh`` remote -> provisioned via ``uv`` (``execnet._provision``): ``uv run --with `` where ```` is ``execnet==`` for a released coordinator or a locally-built, - version-cached wheel for a dev coordinator. + version-cached wheel for a dev coordinator. For an ``ssh`` remote on a + dev coordinator the wheel is not on the remote filesystem, so the remote + command is a POSIX-sh prelude that reads the wheel bytes from stdin + (``head -c N`` into a temp dir) and ``exec``s ``uv`` against it; the + coordinator streams those bytes before the Message protocol. + +The worker configuration (id, execmodel, coordinator version) is passed as a +single JSON CLI argument (``_provision.worker_cli_arg``), so every launcher +shares one contract instead of scattered positional args. Coordinator and worker roles: diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index b7f6ac56..b24bfe8b 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -18,6 +18,7 @@ import json import re +import shlex import shutil import subprocess import tempfile @@ -143,36 +144,74 @@ def coordinator_requirement() -> str: return str(_build_wheel(version)) -def uv_run_argv( - *, python: str | None, requirement: str, module_args: list[str] -) -> list[str]: - """Build ``uv run [--python X] --with python -u -m execnet._trio_worker …``. +def worker_cli_arg(spec: Any) -> str: + """Single JSON CLI argument carrying the worker config (the whole 'spec thing'). - ``--no-project`` keeps the surrounding execnet checkout from being synced, so - the ephemeral env holds only the requirement (+ trio). + Passed to ``python -m execnet._trio_worker`` by every launcher (popen, uv, + ssh) so the worker config lives in one place rather than scattered positional + args. """ - argv = ["uv", "run", "--no-project"] - if python: - argv += ["--python", python] - argv += [ + import execnet + + return json.dumps( + { + "id": f"{spec.id}-worker", + "execmodel": spec.execmodel, + "coordinator_version": execnet.__version__, + } + ) + + +def worker_module_tokens(spec: Any) -> list[str]: + """``python -u -m execnet._trio_worker `` tokens.""" + return ["python", "-u", "-m", "execnet._trio_worker", worker_cli_arg(spec)] + + +def _uv_prefix(spec: Any) -> list[str]: + # --no-project keeps the surrounding execnet checkout from being synced. + prefix = ["uv", "run", "--no-project"] + if spec.python: + prefix += ["--python", spec.python] + return prefix + + +def uv_worker_argv(spec: Any) -> list[str]: + """``uv run`` argv to launch the Trio worker locally (wheel path is local).""" + return [ + *_uv_prefix(spec), "--with", - requirement, - "python", - "-u", - "-m", - "execnet._trio_worker", - *module_args, + coordinator_requirement(), + *worker_module_tokens(spec), ] - return argv -def uv_worker_argv(spec: Any) -> list[str]: - """Full ``uv run`` argv to launch the Trio worker for ``spec``.""" +def ssh_remote_command(spec: Any) -> tuple[str, bytes]: + """Remote shell command + stdin preamble to launch the worker over ssh. + + Released coordinator -> ``uv run --with execnet== …`` with no preamble. + Dev coordinator -> a POSIX-sh prelude that receives the wheel bytes from + stdin (``head -c N``) into a temp dir and ``exec``s uv against it; the wheel + bytes are returned as the preamble to stream before the Message protocol. + """ import execnet - module_args = [f"{spec.id}-worker", spec.execmodel, execnet.__version__] - return uv_run_argv( - python=spec.python, - requirement=coordinator_requirement(), - module_args=module_args, + version = execnet.__version__ + worker = worker_module_tokens(spec) + if _RELEASED_RE.match(version): + command = shlex.join( + [*_uv_prefix(spec), "--with", f"execnet=={version}", *worker] + ) + return command, b"" + + wheel = _build_wheel(version) + data = wheel.read_bytes() + # "$d/": expand the temp dir, concatenate the (quoted) wheel filename. + remote_wheel = '"$d/"' + shlex.quote(wheel.name) + uv_run = " ".join(shlex.quote(token) for token in [*_uv_prefix(spec), "--with"]) + worker_cmd = " ".join(shlex.quote(token) for token in worker) + prelude = ( + f"d=$(mktemp -d) && " + f"head -c {len(data)} > {remote_wheel} && " + f"exec {uv_run} {remote_wheel} {worker_cmd}" ) + return prelude, data diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 68755d50..f9cf7a71 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -500,11 +500,9 @@ def popen_module_args(spec: Any) -> list[str]: No source is sent over the wire; the worker imports the installed execnet + trio. Used for same-interpreter popen and for a ``python=`` interpreter that - already has execnet (so ``sys.executable`` stays that interpreter). The - coordinator version is passed so the worker can do a rough compatibility - check against its own installed version. + already has execnet (so ``sys.executable`` stays that interpreter). """ - import execnet + from . import _provision if getattr(spec, "python", None): from .gateway_io import shell_split_path @@ -516,13 +514,7 @@ def popen_module_args(spec: Any) -> list[str]: args = [*interpreter, "-u"] if getattr(spec, "dont_write_bytecode", False): args.append("-B") - args += [ - "-m", - "execnet._trio_worker", - f"{spec.id}-worker", - spec.execmodel, - execnet.__version__, - ] + args += ["-m", "execnet._trio_worker", _provision.worker_cli_arg(spec)] return args @@ -552,12 +544,19 @@ def kill(self) -> None: def _open_trio_gateway( - group: Any, spec: Any, args: list[str], *, remoteaddress: str | None = None + group: Any, + spec: Any, + args: list[str], + *, + remoteaddress: str | None = None, + preamble: bytes = b"", ) -> Gateway: """Spawn ``args``, do the worker handshake, and attach a Trio session. Shared by the popen and ssh factories; ``args`` already encodes how the worker is launched (direct module, uv-provisioned, or wrapped in ssh). + ``preamble`` is streamed to the worker's stdin before the handshake (used to + ship a wheel to a remote that receives it with ``head -c``). """ import execnet @@ -569,6 +568,8 @@ async def _create_and_attach() -> Gateway: process = await open_popen_process(args) try: async_io = ProcessStreamsIO(process) + if preamble: + await async_io.write_all(preamble) ack = await async_io.read_exact(1) if ack != b"1": raise EOFError(f"bad bootstrap handshake: {ack!r}") @@ -615,30 +616,30 @@ def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: return _open_trio_gateway(group, spec, args) -def ssh_trio_args(spec: Any) -> list[str]: - """``ssh [-F cfg] ''`` for the Trio ssh path. +def ssh_trio_args(spec: Any) -> tuple[list[str], bytes]: + """``(ssh argv, stdin preamble)`` for the Trio ssh path. - The remote runs the uv-provisioned worker module; the worker command is - shell-quoted for the remote shell. + The remote runs the uv-provisioned worker; for a dev coordinator the + preamble carries the wheel bytes that the remote command receives. """ - import shlex - from . import _provision - remote_command = shlex.join(_provision.uv_worker_argv(spec)) + remote_command, preamble = _provision.ssh_remote_command(spec) args = ["ssh", "-C"] if getattr(spec, "ssh_config", None): args += ["-F", spec.ssh_config] assert spec.ssh is not None args += spec.ssh.split() args.append(remote_command) - return args + return args, preamble def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: """Create an ssh Gateway on the Trio IO path (uv-provisioned worker).""" - args = ssh_trio_args(spec) - return _open_trio_gateway(group, spec, args, remoteaddress=spec.ssh) + args, preamble = ssh_trio_args(spec) + return _open_trio_gateway( + group, spec, args, remoteaddress=spec.ssh, preamble=preamble + ) def should_use_trio_ssh(spec: Any) -> bool: diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 2c297589..f90db607 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -290,17 +290,17 @@ def _check_version(coordinator_version: str) -> None: def _main() -> None: - """Entry point for ``python -m execnet._trio_worker [ver]``. + """Entry point for ``python -m execnet._trio_worker ``. - The worker imports execnet + trio from the environment; no source is sent - over the wire to bootstrap it. + ```` is the coordinator's ``_provision.worker_cli_arg`` payload: + ``{"id", "execmodel", "coordinator_version"}``. The worker imports execnet + + trio from the environment; no source is sent over the wire to bootstrap it. """ - argv = sys.argv - worker_id = argv[1] if len(argv) > 1 else "worker" - execmodel = argv[2] if len(argv) > 2 else "thread" - if len(argv) > 3: - _check_version(argv[3]) - serve_popen_trio(id=worker_id, execmodel=execmodel) + import json + + config = json.loads(sys.argv[1]) + _check_version(config["coordinator_version"]) + serve_popen_trio(id=config["id"], execmodel=config["execmodel"]) if __name__ == "__main__": diff --git a/testing/test_gateway.py b/testing/test_gateway.py index f7f882ae..e123bbe2 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -404,10 +404,16 @@ def test_ssh_trio_args_include_config( from execnet import _provision from execnet import _trio_host - monkeypatch.setattr(_provision, "coordinator_requirement", lambda: "execnet") - args = _trio_host.ssh_trio_args(execnet.XSpec("ssh=xyz//ssh_config=qwe")) + monkeypatch.setattr( + _provision, "ssh_remote_command", lambda spec: ("worker-cmd", b"") + ) + args, preamble = _trio_host.ssh_trio_args( + execnet.XSpec("ssh=xyz//ssh_config=qwe") + ) assert args[args.index("-F") + 1] == "qwe" assert "xyz" in args + assert args[-1] == "worker-cmd" + assert preamble == b"" def test_sshaddress(self, gw: Gateway, specssh: execnet.XSpec) -> None: assert gw.remoteaddress == specssh.ssh diff --git a/testing/test_provision.py b/testing/test_provision.py new file mode 100644 index 00000000..8df8775a --- /dev/null +++ b/testing/test_provision.py @@ -0,0 +1,40 @@ +"""Unit tests for coordinator-side uv worker provisioning.""" + +from __future__ import annotations + +import json +import re + +import pytest + +import execnet +from execnet import _provision + +released = re.fullmatch(r"\d+\.\d+\.\d+", execnet.__version__) is not None + + +def test_worker_cli_arg_carries_config() -> None: + spec = execnet.XSpec("popen//id=gw5//execmodel=thread") + config = json.loads(_provision.worker_cli_arg(spec)) + assert config["id"] == "gw5-worker" + assert config["execmodel"] == "thread" + assert config["coordinator_version"] == execnet.__version__ + + +def test_ssh_remote_command_released(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(execnet, "__version__", "9.9.9") + spec = execnet.XSpec("ssh=host//id=gw0//execmodel=thread") + command, preamble = _provision.ssh_remote_command(spec) + assert "execnet==9.9.9" in command + assert "head -c" not in command # no shipping + assert preamble == b"" + + +@pytest.mark.skipif(released, reason="released execnet resolves from an index") +@pytest.mark.skipif(not _provision.uv_available(), reason="uv required to build wheel") +def test_ssh_remote_command_dev_ships_wheel() -> None: + spec = execnet.XSpec("ssh=host//id=gw0//execmodel=thread") + command, preamble = _provision.ssh_remote_command(spec) + assert "mktemp -d" in command + assert f"head -c {len(preamble)}" in command + assert preamble[:2] == b"PK" # a wheel is a zip archive From 21096da19f4a2be00290b81a32286502b8f8a501 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 09:48:12 +0200 Subject: [PATCH 10/91] feat: add execnet-socketserver console command Expose the socket server as a `[project.scripts]` entry point so it can be started directly, e.g. provisioned anywhere with `uvx --from execnet execnet-socketserver :8888`. Refactor the __main__ block into a main() with an argparse CLI (hostport + --once), and thread execmodel explicitly through startserver/exec_from_one_connection instead of relying on a module global (which main()'s local scope broke). Co-Authored-By: Claude Opus 4.8 --- doc/basics.rst | 4 +- pyproject.toml | 3 ++ src/execnet/script/socketserver.py | 43 +++++++++++++++----- testing/test_socketserver_cli.py | 63 ++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 10 deletions(-) create mode 100644 testing/test_socketserver_cli.py diff --git a/doc/basics.rst b/doc/basics.rst index 723de83f..a6b256b7 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -58,7 +58,9 @@ Examples for valid gateway specifications remotely sets an environment variable ``NAME`` to ``value``. * ``socket=192.168.1.4:8888`` specifies a Python Socket server - process that listens on ``192.168.1.4:8888`` + process that listens on ``192.168.1.4:8888``. Such a server can be + started with the ``execnet-socketserver`` console command, e.g. run + anywhere with ``uvx --from execnet execnet-socketserver :8888``. .. versionadded:: 1.5 diff --git a/pyproject.toml b/pyproject.toml index 853ba3d8..f419c31d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,9 @@ classifiers = [ "Topic :: System :: Networking", ] +[project.scripts] +execnet-socketserver = "execnet.script.socketserver:main" + [project.optional-dependencies] testing = [ "pre-commit", diff --git a/src/execnet/script/socketserver.py b/src/execnet/script/socketserver.py index fa98743c..43d82147 100644 --- a/src/execnet/script/socketserver.py +++ b/src/execnet/script/socketserver.py @@ -48,7 +48,7 @@ def print_(*args) -> None: ) -def exec_from_one_connection(serversock) -> None: +def exec_from_one_connection(serversock, execmodel: ExecModel) -> None: print_(progname, "Entering Accept loop", serversock.getsockname()) clientsock, address = serversock.accept() print_(progname, "got new connection from {} {}".format(*address)) @@ -89,12 +89,12 @@ def bind_and_listen(hostport: str | tuple[str, int], execmodel: ExecModel): return serversock -def startserver(serversock, loop: bool = False) -> None: +def startserver(serversock, execmodel: ExecModel, loop: bool = False) -> None: execute_path = os.getcwd() try: while 1: try: - exec_from_one_connection(serversock) + exec_from_one_connection(serversock, execmodel) except (KeyboardInterrupt, SystemExit): raise except BaseException as exc: @@ -112,15 +112,40 @@ def startserver(serversock, loop: bool = False) -> None: serversock.shutdown(2) -if __name__ == "__main__": - import sys +def main(argv: list[str] | None = None) -> None: + """Console entry point (``execnet-socketserver``). + + Bind a socket and serve gateway connections. Intended to be run directly, + e.g. provisioned on a host with ``uvx --from execnet execnet-socketserver``. + """ + import argparse - hostport = sys.argv[1] if len(sys.argv) > 1 else ":8888" from execnet.gateway_base import get_execmodel + parser = argparse.ArgumentParser( + prog="execnet-socketserver", + description="Serve execnet gateway connections over a socket.", + ) + parser.add_argument( + "hostport", + nargs="?", + default=":8888", + help="address to bind as HOST:PORT or :PORT (default: :8888)", + ) + parser.add_argument( + "--once", + action="store_true", + help="serve a single connection and exit instead of looping", + ) + args = parser.parse_args(argv) + execmodel = get_execmodel("thread") - serversock = bind_and_listen(hostport, execmodel) - startserver(serversock, loop=True) + serversock = bind_and_listen(args.hostport, execmodel) + startserver(serversock, execmodel, loop=not args.once) + + +if __name__ == "__main__": + main() elif __name__ == "__channelexec__": chan: Channel = globals()["channel"] @@ -130,4 +155,4 @@ def startserver(serversock, loop: bool = False) -> None: sock = bind_and_listen(bindname, execmodel) port = sock.getsockname() chan.send(port) - startserver(sock) + startserver(sock, execmodel) diff --git a/testing/test_socketserver_cli.py b/testing/test_socketserver_cli.py new file mode 100644 index 00000000..039803fa --- /dev/null +++ b/testing/test_socketserver_cli.py @@ -0,0 +1,63 @@ +"""Test the ``execnet-socketserver`` console entry point end to end.""" + +from __future__ import annotations + +import shutil +import socket +import subprocess +import time +from collections.abc import Iterator + +import pytest + +import execnet + +SERVER = shutil.which("execnet-socketserver") + +pytestmark = pytest.mark.skipif( + SERVER is None, reason="execnet-socketserver console script not installed" +) + + +def _free_port() -> int: + s = socket.socket() + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + s.close() + return port + + +@pytest.fixture +def socketserver_port() -> Iterator[int]: + assert SERVER is not None + port = _free_port() + proc = subprocess.Popen( + [SERVER, f"127.0.0.1:{port}"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + # loop mode: a throwaway probe just consumes one accept iteration + for _ in range(100): + try: + socket.create_connection(("127.0.0.1", port), timeout=0.2).close() + break + except OSError: + time.sleep(0.1) + else: + pytest.fail("execnet-socketserver did not start") + yield port + finally: + proc.kill() + proc.wait(timeout=5) + + +def test_socketserver_cli_roundtrip(socketserver_port: int) -> None: + group = execnet.Group() + try: + gw = group.makegateway(f"socket=127.0.0.1:{socketserver_port}//id=sock") + channel = gw.remote_exec("channel.send(channel.receive() + 1)") + channel.send(41) + assert channel.receive() == 42 + finally: + group.terminate(timeout=5.0) From 420693ac3d939af96929a2e8b433b5d0e2066ff1 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 10:53:35 +0200 Subject: [PATCH 11/91] feat: run direct socket gateways on Trio with subprocess workers Move the direct `socket=host:port` transport onto the Trio host: - The socketserver becomes a Trio TCP listener that spawns a `python -m execnet._trio_worker --socket-fd N` subprocess per connection (passing the accepted socket by fd) instead of exec'ing sent source inline. - The worker gains a socket serve mode: it adopts an inherited socket fd into a Trio SocketStream and serves the Message protocol over it. The worker CLI now always carries its config as args, so a future popen socketpair can reuse the same path instead of hijacking stdio. - The coordinator connects a Trio TCP stream, waits for the b"1" handshake, and attaches a Trio session (no local process). A failed connect raises HostNotFound. The worker serve setup is refactored into shared _build_worker_gateway / _run_worker helpers. `installvia` still uses the legacy path for now. Co-Authored-By: Claude Opus 4.8 --- src/execnet/_trio_host.py | 95 ++++++++++++++++++++++ src/execnet/_trio_worker.py | 124 ++++++++++++++++++++--------- src/execnet/multi.py | 2 + src/execnet/script/socketserver.py | 70 ++++++++++++++-- testing/test_socketserver_cli.py | 41 +++++----- 5 files changed, 263 insertions(+), 69 deletions(-) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index f9cf7a71..c58d0c34 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -137,6 +137,45 @@ async def aclose_write(self) -> None: await self._write.aclose() +class SocketStreamIO: + """Async IO over a single bidirectional Trio stream (a socket). + + ``read_exact`` never over-reads (``receive_some(k)`` returns at most ``k`` + bytes), so no cross-call buffering is needed. ``aclose`` on a Trio stream is + idempotent, so close-read and close-write both just close the socket. + """ + + def __init__(self, stream: trio.abc.Stream) -> None: + self._stream = stream + + async def read_exact(self, n: int) -> bytes: + return await read_exact_receive_stream(self._stream, n) + + async def write_all(self, data: bytes) -> None: + await self._stream.send_all(data) + + async def aclose_read(self) -> None: + await self._stream.aclose() + + async def aclose_write(self) -> None: + await self._stream.aclose() + + +async def adopt_socket(socket_fd: int) -> SocketStreamIO: + """Worker side: wrap an inherited socket fd and send the handshake. + + Runs on the Trio host loop. The coordinator waits for ``b"1"`` before + starting the Message protocol; the worker config comes from the CLI. + """ + import socket as _socket + + sock = _socket.socket(fileno=socket_fd) + stream = trio.SocketStream(trio.socket.from_stdlib_socket(sock)) + io = SocketStreamIO(stream) + await io.write_all(b"1") + return io + + class SyncIOHandle: """Sync IO facade for Group.terminate wait/kill/close_write.""" @@ -650,3 +689,59 @@ def should_use_trio_ssh(spec: Any) -> bool: return False execmodel = getattr(spec, "execmodel", None) return execmodel in (None, "thread", "main_thread_only") + + +def makegateway_socket_trio(group: Any, spec: Any) -> Gateway: + """Connect a Trio TCP stream to a running ``execnet-socketserver``. + + The server spawns the worker and synthesises its config, so the coordinator + just connects, waits for the worker's ``b"1"`` handshake, and attaches a Trio + session (no local process). + """ + import execnet + + from .gateway_bootstrap import HostNotFound + + host: TrioHost = group._ensure_trio_host() + assert spec.socket is not None + host_str, _, port_str = spec.socket.rpartition(":") + address = (host_str, int(port_str)) + + async def _create_and_attach() -> Gateway: + try: + stream = await trio.open_tcp_stream(*address) + except OSError as exc: + raise HostNotFound(spec.socket) from exc + io = SocketStreamIO(stream) + try: + ack = await io.read_exact(1) + if ack != b"1": + raise EOFError(f"bad socket handshake: {ack!r}") + except BaseException: + with trio.move_on_after(5): + await stream.aclose() + raise + + gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + session = await host.start_session(gw, io) + gw._attach_trio_session(session) + gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=spec.socket) + return gw + + return host.call(_create_and_attach) + + +def should_use_trio_socket(spec: Any) -> bool: + """Trio path for direct ``socket=host:port`` gateways. + + ``installvia`` (start a server through another gateway) still uses the legacy + path for now. + """ + if not trio_host_enabled(): + return False + if not getattr(spec, "socket", None): + return False + if getattr(spec, "installvia", None): + return False + execmodel = getattr(spec, "execmodel", None) + return execmodel in (None, "thread", "main_thread_only") diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index f90db607..5fc137fb 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -204,8 +204,59 @@ def kill(self) -> None: return +def _build_worker_gateway( + host: _trio_host.TrioHost, id: str, model: ExecModel +) -> tuple[WorkerGateway, TrioWorkerExec, bool]: + """Construct the WorkerGateway + Trio exec pool (no IO yet).""" + trace(f"creating workergateway on trio id={id!r}") + io_stub = _WorkerIOStub(model) + gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) + + main_thread_only = model.backend == "main_thread_only" + trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) + # Duck-type as WorkerPool for STATUS / _terminate_execution. + gateway._execpool = trio_exec # type: ignore[assignment] + gateway._trio_exec = trio_exec + gateway._executetask_complete = None + if main_thread_only: + gateway._executetask_complete = model.Event() + gateway._executetask_complete.set() + return gateway, trio_exec, main_thread_only + + +def _run_worker(host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel) -> None: + """Attach ``io`` as the gateway session and serve until shutdown.""" + gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model) + + async def _start() -> _trio_host.ProtocolSession: + return await host.start_session(gateway, io) + + session = host.call(_start) + gateway._attach_trio_session(session) + + try: + if main_thread_only: + trace("integrating as primary thread (trio worker)") + trio_exec.integrate_as_primary_thread() + gateway.join() + except KeyboardInterrupt: + # Match WorkerGateway.serve(): swallow in the worker. + trace("swallowing keyboardinterrupt, serve finished") + finally: + host.stop(timeout=5.0) + # Trio's to_thread cache uses non-daemon threads that would otherwise + # keep this disposable worker process alive after serve returns. + os._exit(0) + + +async def _make_fd_io(read_fd: int, write_fd: int) -> Any: + from . import _trio_host + + return _trio_host.FdStreamsIO(read_fd, write_fd) + + def serve_popen_trio(id: str, execmodel: str = "thread") -> None: - """Serve a WorkerGateway with Message IO + exec scheduling on Trio.""" + """Serve a WorkerGateway over the stdio pipes (popen / ssh worker).""" from . import _trio_host model = get_execmodel(execmodel) @@ -214,45 +265,27 @@ def serve_popen_trio(id: str, execmodel: str = "thread") -> None: # before starting the Message protocol. We are launched as a plain module # (``python -m execnet._trio_worker``), so nothing was sent to bootstrap us. os.write(write_fd, b"1") - # Keep the historic trace token so tests looking for workergateway still pass. - trace(f"creating workergateway on trio id={id!r}") host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") host.start() - try: - io_stub = _WorkerIOStub(model) - gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) - - main_thread_only = model.backend == "main_thread_only" - trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) - # Duck-type as WorkerPool for STATUS / _terminate_execution. - gateway._execpool = trio_exec # type: ignore[assignment] - gateway._trio_exec = trio_exec - gateway._executetask_complete = None - if main_thread_only: - gateway._executetask_complete = model.Event() - gateway._executetask_complete.set() + io = host.call(_make_fd_io, read_fd, write_fd) + _run_worker(host, io, id, model) - async def _start() -> _trio_host.ProtocolSession: - async_io = _trio_host.FdStreamsIO(read_fd, write_fd) - return await host.start_session(gateway, async_io) - session = host.call(_start) - gateway._attach_trio_session(session) +def serve_socket_trio(id: str, execmodel: str, socket_fd: int) -> None: + """Serve a WorkerGateway over an inherited socket fd. - try: - if main_thread_only: - trace("integrating as primary thread (trio worker)") - trio_exec.integrate_as_primary_thread() - gateway.join() - except KeyboardInterrupt: - # Match WorkerGateway.serve(): swallow in the worker. - trace("swallowing keyboardinterrupt, serve finished") - finally: - host.stop(timeout=5.0) - # Trio's to_thread cache uses non-daemon threads that would otherwise - # keep this disposable worker process alive after serve returns. - os._exit(0) + Used for the socketserver (an accepted TCP connection) and, in future, a + popen socketpair. The socket is adopted inside Trio and the handshake is + written on the host loop; config comes from the CLI. + """ + from . import _trio_host + + model = get_execmodel(execmodel) + host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") + host.start() + io = host.call(_trio_host.adopt_socket, socket_fd) + _run_worker(host, io, id, model) def _rough_version(version: str) -> tuple[int, ...]: @@ -290,17 +323,30 @@ def _check_version(coordinator_version: str) -> None: def _main() -> None: - """Entry point for ``python -m execnet._trio_worker ``. + """Entry point for ``python -m execnet._trio_worker [--socket-fd N]``. ```` is the coordinator's ``_provision.worker_cli_arg`` payload: - ``{"id", "execmodel", "coordinator_version"}``. The worker imports execnet + - trio from the environment; no source is sent over the wire to bootstrap it. + ``{"id", "execmodel", "coordinator_version"}``. With ``--socket-fd`` the + worker serves over that inherited socket (socketserver); otherwise over the + stdio pipes (popen / ssh). The worker imports execnet + trio from the + environment; no source is sent over the wire to bootstrap it. """ + import argparse import json - config = json.loads(sys.argv[1]) + parser = argparse.ArgumentParser(prog="execnet._trio_worker") + parser.add_argument("config", help="JSON worker config") + # Protocol transport: an inherited socket fd (socketserver, or a future popen + # socketpair); without it the worker serves over the stdio pipes (ssh). + parser.add_argument("--socket-fd", type=int, default=None) + ns = parser.parse_args() + + config = json.loads(ns.config) _check_version(config["coordinator_version"]) - serve_popen_trio(id=config["id"], execmodel=config["execmodel"]) + if ns.socket_fd is not None: + serve_socket_trio(config["id"], config["execmodel"], ns.socket_fd) + else: + serve_popen_trio(id=config["id"], execmodel=config["execmodel"]) if __name__ == "__main__": diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 29cea870..d64722ee 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -158,6 +158,8 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: gw = _trio_host.makegateway_popen_trio(self, spec) elif _trio_host.should_use_trio_ssh(spec): gw = _trio_host.makegateway_ssh_trio(self, spec) + elif _trio_host.should_use_trio_socket(spec): + gw = _trio_host.makegateway_socket_trio(self, spec) elif spec.via: assert not spec.socket master = self[spec.via] diff --git a/src/execnet/script/socketserver.py b/src/execnet/script/socketserver.py index 43d82147..ea580bcd 100644 --- a/src/execnet/script/socketserver.py +++ b/src/execnet/script/socketserver.py @@ -112,15 +112,74 @@ def startserver(serversock, execmodel: ExecModel, loop: bool = False) -> None: serversock.shutdown(2) +async def _trio_serve(hostport: str, once: bool) -> None: + """Trio TCP server that spawns a worker subprocess per connection. + + No code is executed inline: each accepted socket is handed (by fd) to a + fresh ``python -m execnet._trio_worker --socket-fd`` process which serves the + gateway over it. + """ + import itertools + import json + import subprocess + + import trio + + import execnet + + host, _, port_str = hostport.rpartition(":") + listeners = await trio.open_tcp_listeners(int(port_str), host=host or None) + addr = listeners[0].socket.getsockname() + # Report the bound address (port may be ephemeral) for callers to read. + print("execnet-socketserver listening on %s %s" % (addr[0], addr[1]), flush=True) + + counter = itertools.count() + + with trio.CancelScope() as scope: + + async def handler(stream: trio.SocketStream) -> None: + fd = stream.socket.fileno() + # Synthesise the worker config (socketserver workers default to the + # thread model, matching the legacy socket server). + config = json.dumps( + { + "id": "socketworker%d" % next(counter), + "execmodel": "thread", + "coordinator_version": execnet.__version__, + } + ) + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "execnet._trio_worker", + config, + "--socket-fd", + str(fd), + ], + pass_fds=[fd], + ) + # The child forked with a copy of the fd; release ours. + await stream.aclose() + if once: + scope.cancel() + return + # Reap the worker without blocking other connections. + await trio.to_thread.run_sync(proc.wait) + + await trio.serve_listeners(handler, listeners) + + def main(argv: list[str] | None = None) -> None: """Console entry point (``execnet-socketserver``). - Bind a socket and serve gateway connections. Intended to be run directly, - e.g. provisioned on a host with ``uvx --from execnet execnet-socketserver``. + Serve execnet gateway connections over a socket, spawning a worker + subprocess per connection. Intended to be run directly, e.g. provisioned on + a host with ``uvx --from execnet execnet-socketserver``. """ import argparse - from execnet.gateway_base import get_execmodel + import trio parser = argparse.ArgumentParser( prog="execnet-socketserver", @@ -138,10 +197,7 @@ def main(argv: list[str] | None = None) -> None: help="serve a single connection and exit instead of looping", ) args = parser.parse_args(argv) - - execmodel = get_execmodel("thread") - serversock = bind_and_listen(args.hostport, execmodel) - startserver(serversock, execmodel, loop=not args.once) + trio.run(_trio_serve, args.hostport, args.once) if __name__ == "__main__": diff --git a/testing/test_socketserver_cli.py b/testing/test_socketserver_cli.py index 039803fa..2b6d60dd 100644 --- a/testing/test_socketserver_cli.py +++ b/testing/test_socketserver_cli.py @@ -1,11 +1,14 @@ -"""Test the ``execnet-socketserver`` console entry point end to end.""" +"""Test the ``execnet-socketserver`` console entry point end to end. + +The Trio socketserver binds a port and spawns a ``python -m execnet._trio_worker`` +subprocess per connection (no inline code execution); the coordinator connects +over a Trio TCP stream. +""" from __future__ import annotations import shutil -import socket import subprocess -import time from collections.abc import Iterator import pytest @@ -19,33 +22,25 @@ ) -def _free_port() -> int: - s = socket.socket() - s.bind(("127.0.0.1", 0)) - port = s.getsockname()[1] - s.close() - return port - - @pytest.fixture def socketserver_port() -> Iterator[int]: assert SERVER is not None - port = _free_port() proc = subprocess.Popen( - [SERVER, f"127.0.0.1:{port}"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + [SERVER, ":0"], # ephemeral port; it prints the one it bound + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, ) try: - # loop mode: a throwaway probe just consumes one accept iteration - for _ in range(100): - try: - socket.create_connection(("127.0.0.1", port), timeout=0.2).close() + assert proc.stdout is not None + port = None + while True: + line = proc.stdout.readline() + if not line: + pytest.fail("execnet-socketserver exited before binding") + if "listening on" in line: + port = int(line.split()[-1]) break - except OSError: - time.sleep(0.1) - else: - pytest.fail("execnet-socketserver did not start") yield port finally: proc.kill() From b83ff3be6e62bdf829a56e5a79758dd5885f9c60 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 11:34:48 +0200 Subject: [PATCH 12/91] feat: migrate installvia to a GATEWAY_START_SOCKET protocol message Since execnet is now always installed on both sides, replace the remote_exec source-shipping used by `socket//installvia=` with a first-class protocol message. The coordinator sends GATEWAY_START_SOCKET (with the bind host) on a request channel; the via gateway's Trio host binds an ephemeral port, replies with the (host, port) on that channel, and serves the one connection by spawning a worker subprocess. The coordinator then connects to it over the Trio socket path. This makes the inline-exec socketserver dead, so drop it: the `execnet-socketserver` script is now only the Trio server + CLI, and the Windows service wrapper launches that. The legacy `__channelexec__` mode, `bind_and_listen`, `startserver`, and the inline `exec` are gone. Co-Authored-By: Claude Opus 4.8 --- src/execnet/_trio_host.py | 107 +++++++++++-- src/execnet/gateway_base.py | 11 ++ src/execnet/script/socketserver.py | 184 ++-------------------- src/execnet/script/socketserverservice.py | 7 +- 4 files changed, 124 insertions(+), 185 deletions(-) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index c58d0c34..ed78dd7d 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -7,6 +7,8 @@ from __future__ import annotations +import itertools +import json import os import queue import subprocess @@ -25,6 +27,8 @@ from .gateway_base import ExecModel from .gateway_base import GatewayReceivedTerminate from .gateway_base import Message +from .gateway_base import dumps_internal +from .gateway_base import loads_internal from .gateway_base import trace if TYPE_CHECKING: @@ -703,15 +707,21 @@ def makegateway_socket_trio(group: Any, spec: Any) -> Gateway: from .gateway_bootstrap import HostNotFound host: TrioHost = group._ensure_trio_host() - assert spec.socket is not None - host_str, _, port_str = spec.socket.rpartition(":") - address = (host_str, int(port_str)) + if getattr(spec, "installvia", None): + realhost, realport = start_socketserver_via(group[spec.installvia]) + address = (realhost, realport) + remoteaddress = "%s:%d" % (realhost, realport) + else: + assert spec.socket is not None + host_str, _, port_str = spec.socket.rpartition(":") + address = (host_str, int(port_str)) + remoteaddress = spec.socket async def _create_and_attach() -> Gateway: try: stream = await trio.open_tcp_stream(*address) except OSError as exc: - raise HostNotFound(spec.socket) from exc + raise HostNotFound(remoteaddress) from exc io = SocketStreamIO(stream) try: ack = await io.read_exact(1) @@ -725,23 +735,96 @@ async def _create_and_attach() -> Gateway: gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) session = await host.start_session(gw, io) gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=spec.socket) + gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) return gw return host.call(_create_and_attach) def should_use_trio_socket(spec: Any) -> bool: - """Trio path for direct ``socket=host:port`` gateways. - - ``installvia`` (start a server through another gateway) still uses the legacy - path for now. - """ + """Trio path for ``socket=host:port`` gateways, including ``installvia``.""" if not trio_host_enabled(): return False if not getattr(spec, "socket", None): return False - if getattr(spec, "installvia", None): - return False execmodel = getattr(spec, "execmodel", None) return execmodel in (None, "thread", "main_thread_only") + + +_socket_worker_counter = itertools.count() + + +def _spawn_socket_worker(fd: int) -> subprocess.Popen[bytes]: + """Spawn a worker subprocess serving over the inherited socket ``fd``.""" + import execnet + + config = json.dumps( + { + "id": "socketworker%d" % next(_socket_worker_counter), + "execmodel": "thread", + "coordinator_version": execnet.__version__, + } + ) + return subprocess.Popen( + [sys.executable, "-m", "execnet._trio_worker", config, "--socket-fd", str(fd)], + pass_fds=[fd], + ) + + +async def serve_socket_connection(stream: trio.SocketStream, *, reap: bool) -> None: + """Hand an accepted socket to a fresh worker subprocess (server side). + + ``reap`` waits for the worker (loop server); when false the worker outlives + this task (one-shot / installvia). + """ + proc = _spawn_socket_worker(stream.socket.fileno()) + # The child forked with a copy of the fd; release ours. + await stream.aclose() + if reap: + await trio.to_thread.run_sync(proc.wait) + + +async def _start_socket_and_reply( + gateway: BaseGateway, channelid: int, bind_host: str +) -> None: + """Bind an ephemeral port, reply with its address, then serve one connection. + + Runs as a task on the worker's Trio host (scheduled from the message + handler). The reply travels back on ``channelid`` like a STATUS reply. + """ + listeners = await trio.open_tcp_listeners(0, host=bind_host) + addr = listeners[0].socket.getsockname() + gateway._send(Message.CHANNEL_DATA, channelid, dumps_internal((addr[0], addr[1]))) + gateway._send(Message.CHANNEL_CLOSE, channelid) + + stream = await listeners[0].accept() + for listener in listeners: + await listener.aclose() + await serve_socket_connection(stream, reap=True) + + +def handle_start_socket(gateway: BaseGateway, channelid: int, data: bytes) -> None: + """Worker handler for ``Message.GATEWAY_START_SOCKET`` (on the host thread).""" + bind_host = loads_internal(data) + assert isinstance(bind_host, str) + host: TrioHost = gateway._trio_exec.host # type: ignore[attr-defined] + # The receiver runs on the host thread, so schedule the async work directly. + host.start_soon(_start_socket_and_reply, gateway, channelid, bind_host) + + +def start_socketserver_via( + via_gateway: Any, bind_host: str = "localhost" +) -> tuple[str, int]: + """Ask ``via_gateway`` (protocol message) to start a one-shot socket listener. + + Returns the ``(host, port)`` the coordinator should connect to. + """ + channel = via_gateway.newchannel() + via_gateway._send( + Message.GATEWAY_START_SOCKET, channel.id, dumps_internal(bind_host) + ) + realhost, realport = channel.receive() + channel.waitclose() + if not realhost or realhost in ("0.0.0.0", "::"): + realhost = "localhost" + return realhost, int(realport) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index c9933fe0..764f6a0a 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -548,6 +548,17 @@ def _channel_last_message(message: Message, gateway: BaseGateway) -> None: CHANNEL_LAST_MESSAGE = 7 _types[CHANNEL_LAST_MESSAGE] = ("CHANNEL_LAST_MESSAGE", _channel_last_message) + def _gateway_start_socket(message: Message, gateway: BaseGateway) -> None: + # Start a one-shot socketserver on this (Trio) gateway's host and reply + # with the bound (host, port) on the request channel. Handled natively + # instead of shipping source via remote_exec. + from . import _trio_host + + _trio_host.handle_start_socket(gateway, message.channelid, message.data) + + GATEWAY_START_SOCKET = 8 + _types[GATEWAY_START_SOCKET] = ("GATEWAY_START_SOCKET", _gateway_start_socket) + class GatewayReceivedTerminate(Exception): """Receiverthread got termination message.""" diff --git a/src/execnet/script/socketserver.py b/src/execnet/script/socketserver.py index ea580bcd..eb2a86ae 100644 --- a/src/execnet/script/socketserver.py +++ b/src/execnet/script/socketserver.py @@ -1,131 +1,19 @@ #! /usr/bin/env python -""" -start socket based minimal readline exec server - -it can exeuted in 2 modes of operation - -1. as normal script, that listens for new connections - -2. via existing_gateway.remote_exec (as imported module) +"""Trio socket server for execnet gateways. +Listens on a TCP port and hands each accepted connection (by fd) to a fresh +``python -m execnet._trio_worker`` subprocess that serves the gateway over it. +No code is executed inline. Run directly, e.g. provisioned anywhere with +``uvx --from execnet execnet-socketserver``. """ -# this part of the program only executes on the server side -# from __future__ import annotations -import os -import sys -from typing import TYPE_CHECKING - -try: - import fcntl -except ImportError: - fcntl = None # type: ignore[assignment] - -if TYPE_CHECKING: - from execnet.gateway_base import Channel - from execnet.gateway_base import ExecModel - -progname = "socket_readline_exec_server-1.2" - - -debug = 0 - -if debug: # and not os.isatty(sys.stdin.fileno()) - f = open("/tmp/execnet-socket-pyout.log", "w") - old = sys.stdout, sys.stderr - sys.stdout = sys.stderr = f - - -def print_(*args) -> None: - print(" ".join(str(arg) for arg in args)) - - -exec( - """def exec_(source, locs): - exec(source, locs)""" -) - - -def exec_from_one_connection(serversock, execmodel: ExecModel) -> None: - print_(progname, "Entering Accept loop", serversock.getsockname()) - clientsock, address = serversock.accept() - print_(progname, "got new connection from {} {}".format(*address)) - clientfile = clientsock.makefile("rb") - print_("reading line") - # rstrip so that we can use \r\n for telnet testing - source = clientfile.readline().rstrip() - clientfile.close() - g = {"clientsock": clientsock, "address": address, "execmodel": execmodel} - source = eval(source) - if source: - co = compile(source + "\n", "", "exec") - print_(progname, "compiled source, executing") - try: - exec_(co, g) # type: ignore[name-defined] # noqa: F821 - finally: - print_(progname, "finished executing code") - # background thread might hold a reference to this (!?) - # clientsock.close() - - -def bind_and_listen(hostport: str | tuple[str, int], execmodel: ExecModel): - socket = execmodel.socket - if isinstance(hostport, str): - host, port = hostport.split(":") - hostport = (host, int(port)) - serversock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - # set close-on-exec - if hasattr(fcntl, "FD_CLOEXEC"): - old = fcntl.fcntl(serversock.fileno(), fcntl.F_GETFD) - fcntl.fcntl(serversock.fileno(), fcntl.F_SETFD, old | fcntl.FD_CLOEXEC) - # allow the address to be reused in a reasonable amount of time - if os.name == "posix" and sys.platform != "cygwin": - serversock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - - serversock.bind(hostport) - serversock.listen(5) - return serversock - - -def startserver(serversock, execmodel: ExecModel, loop: bool = False) -> None: - execute_path = os.getcwd() - try: - while 1: - try: - exec_from_one_connection(serversock, execmodel) - except (KeyboardInterrupt, SystemExit): - raise - except BaseException as exc: - if debug: - import traceback - - traceback.print_exc() - else: - print_("got exception", exc) - os.chdir(execute_path) - if not loop: - break - finally: - print_("leaving socketserver execloop") - serversock.shutdown(2) - async def _trio_serve(hostport: str, once: bool) -> None: - """Trio TCP server that spawns a worker subprocess per connection. - - No code is executed inline: each accepted socket is handed (by fd) to a - fresh ``python -m execnet._trio_worker --socket-fd`` process which serves the - gateway over it. - """ - import itertools - import json - import subprocess - import trio - import execnet + from execnet import _trio_host host, _, port_str = hostport.rpartition(":") listeners = await trio.open_tcp_listeners(int(port_str), host=host or None) @@ -133,50 +21,22 @@ async def _trio_serve(hostport: str, once: bool) -> None: # Report the bound address (port may be ephemeral) for callers to read. print("execnet-socketserver listening on %s %s" % (addr[0], addr[1]), flush=True) - counter = itertools.count() + if once: + stream = await listeners[0].accept() + for listener in listeners: + await listener.aclose() + # The worker outlives this one-shot server. + await _trio_host.serve_socket_connection(stream, reap=False) + return - with trio.CancelScope() as scope: + async def handler(stream: trio.SocketStream) -> None: + await _trio_host.serve_socket_connection(stream, reap=True) - async def handler(stream: trio.SocketStream) -> None: - fd = stream.socket.fileno() - # Synthesise the worker config (socketserver workers default to the - # thread model, matching the legacy socket server). - config = json.dumps( - { - "id": "socketworker%d" % next(counter), - "execmodel": "thread", - "coordinator_version": execnet.__version__, - } - ) - proc = subprocess.Popen( - [ - sys.executable, - "-m", - "execnet._trio_worker", - config, - "--socket-fd", - str(fd), - ], - pass_fds=[fd], - ) - # The child forked with a copy of the fd; release ours. - await stream.aclose() - if once: - scope.cancel() - return - # Reap the worker without blocking other connections. - await trio.to_thread.run_sync(proc.wait) - - await trio.serve_listeners(handler, listeners) + await trio.serve_listeners(handler, listeners) def main(argv: list[str] | None = None) -> None: - """Console entry point (``execnet-socketserver``). - - Serve execnet gateway connections over a socket, spawning a worker - subprocess per connection. Intended to be run directly, e.g. provisioned on - a host with ``uvx --from execnet execnet-socketserver``. - """ + """Console entry point (``execnet-socketserver``).""" import argparse import trio @@ -202,13 +62,3 @@ def main(argv: list[str] | None = None) -> None: if __name__ == "__main__": main() - -elif __name__ == "__channelexec__": - chan: Channel = globals()["channel"] - execmodel = chan.gateway.execmodel - bindname = chan.receive() - assert isinstance(bindname, (str, tuple)) - sock = bind_and_listen(bindname, execmodel) - port = sock.getsockname() - chan.send(port) - startserver(sock, execmodel) diff --git a/src/execnet/script/socketserverservice.py b/src/execnet/script/socketserverservice.py index 18e375c4..1c991307 100644 --- a/src/execnet/script/socketserverservice.py +++ b/src/execnet/script/socketserverservice.py @@ -15,8 +15,6 @@ import win32service import win32serviceutil -from execnet.gateway_base import get_execmodel - from . import socketserver appname = "ExecNetSocketServer" @@ -65,12 +63,9 @@ def SvcDoRun(self) -> None: hostport = ":8888" print("Starting py.execnet SocketServer on %s" % hostport) - exec_model = get_execmodel("thread") - serversock = socketserver.bind_and_listen(hostport, exec_model) thread = threading.Thread( - target=socketserver.startserver, args=(serversock,), kwargs={"loop": True} + target=socketserver.main, args=([hostport],), daemon=True ) - thread.setDaemon(True) thread.start() # wait to be stopped or self.WAIT_TIME to pass From 62b746b705dece7e79484fcf9db57064aa049734 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 11:49:19 +0200 Subject: [PATCH 13/91] refactor: remove the legacy socket coordinator code With direct socket and installvia both on the Trio path, the legacy socket coordinator is dead. Delete gateway_socket.py (SocketIO, create_io, start_via), drop bootstrap_socket and the socket branch from gateway_bootstrap.bootstrap, and remove the now-unreachable `elif spec.socket:` arm from Group.makegateway. All socket gateways now go through _trio_host.makegateway_socket_trio. Co-Authored-By: Claude Opus 4.8 --- src/execnet/gateway_bootstrap.py | 22 ------- src/execnet/gateway_socket.py | 102 ------------------------------- src/execnet/multi.py | 5 -- 3 files changed, 129 deletions(-) delete mode 100644 src/execnet/gateway_socket.py diff --git a/src/execnet/gateway_bootstrap.py b/src/execnet/gateway_bootstrap.py index e9d7efe1..5707dff4 100644 --- a/src/execnet/gateway_bootstrap.py +++ b/src/execnet/gateway_bootstrap.py @@ -55,26 +55,6 @@ def bootstrap_exec(io: IO, spec: XSpec) -> None: raise HostNotFound(io.remoteaddress) from None -def bootstrap_socket(io: IO, id) -> None: - # XXX: switch to spec - from execnet.gateway_socket import SocketIO - - sendexec( - io, - inspect.getsource(gateway_base), - "import socket", - inspect.getsource(SocketIO), - "try: execmodel", - "except NameError:", - " execmodel = get_execmodel('thread')", - "io = SocketIO(clientsock, execmodel)", - "io.write('1'.encode('ascii'))", - "serve(io, id='%s-worker')" % id, - ) - s = io.read(1) - assert s == b"1" - - def sendexec(io: IO, *sources: str) -> None: source = "\n".join(sources) io.write((repr(source) + "\n").encode("utf-8")) @@ -88,8 +68,6 @@ def bootstrap(io: IO, spec: XSpec) -> execnet.Gateway: bootstrap_import(io, spec) elif spec.ssh or spec.vagrant_ssh: bootstrap_exec(io, spec) - elif spec.socket: - bootstrap_socket(io, spec) else: raise ValueError("unknown gateway type, can't bootstrap") gw = execnet.Gateway(io, spec) diff --git a/src/execnet/gateway_socket.py b/src/execnet/gateway_socket.py deleted file mode 100644 index be42f1ab..00000000 --- a/src/execnet/gateway_socket.py +++ /dev/null @@ -1,102 +0,0 @@ -from __future__ import annotations - -import sys -from typing import cast - -from execnet.gateway import Gateway -from execnet.gateway_base import ExecModel -from execnet.gateway_bootstrap import HostNotFound -from execnet.multi import Group -from execnet.xspec import XSpec - - -class SocketIO: - remoteaddress: str - - def __init__(self, sock, execmodel: ExecModel) -> None: - self.sock = sock - self.execmodel = execmodel - socket = execmodel.socket - try: - # IPTOS_LOWDELAY - sock.setsockopt(socket.SOL_IP, socket.IP_TOS, 0x10) - sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) - except (AttributeError, OSError): - sys.stderr.write("WARNING: cannot set socketoption") - - def read(self, numbytes: int) -> bytes: - "Read exactly 'bytes' bytes from the socket." - buf = b"" - while len(buf) < numbytes: - t = self.sock.recv(numbytes - len(buf)) - if not t: - raise EOFError - buf += t - return buf - - def write(self, data: bytes) -> None: - self.sock.sendall(data) - - def close_read(self) -> None: - try: - self.sock.shutdown(0) - except self.execmodel.socket.error: - pass - - def close_write(self) -> None: - try: - self.sock.shutdown(1) - except self.execmodel.socket.error: - pass - - def wait(self) -> None: - pass - - def kill(self) -> None: - pass - - -def start_via( - gateway: Gateway, hostport: tuple[str, int] | None = None -) -> tuple[str, int]: - """Instantiate a socketserver on the given gateway. - - Returns a host, port tuple. - """ - if hostport is None: - host, port = ("localhost", 0) - else: - host, port = hostport - - from execnet.script import socketserver - - # execute the above socketserverbootstrap on the other side - channel = gateway.remote_exec(socketserver) - channel.send((host, port)) - realhost, realport = cast("tuple[str, int]", channel.receive()) - # self._trace("new_remote received" - # "port=%r, hostname = %r" %(realport, hostname)) - if not realhost or realhost == "0.0.0.0": - realhost = "localhost" - return realhost, realport - - -def create_io(spec: XSpec, group: Group, execmodel: ExecModel) -> SocketIO: - assert spec.socket is not None - assert not spec.python, "socket: specifying python executables not yet supported" - gateway_id = spec.installvia - if gateway_id: - host, port = start_via(group[gateway_id]) - else: - host, port_str = spec.socket.split(":") - port = int(port_str) - - socket = execmodel.socket - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - io = SocketIO(sock, execmodel) - io.remoteaddress = "%s:%d" % (host, port) - try: - sock.connect((host, port)) - except execmodel.socket.gaierror as e: - raise HostNotFound() from e - return io diff --git a/src/execnet/multi.py b/src/execnet/multi.py index d64722ee..cc85d757 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -170,11 +170,6 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: elif spec.popen or spec.ssh or spec.vagrant_ssh: io = gateway_io.create_io(spec, execmodel=self.execmodel) gw = gateway_bootstrap.bootstrap(io, spec) - elif spec.socket: - from . import gateway_socket - - sio = gateway_socket.create_io(spec, self, execmodel=self.execmodel) - gw = gateway_bootstrap.bootstrap(sio, spec) else: raise ValueError(f"no gateway type found for {spec._spec!r}") gw.spec = spec From 19d620d976b4a677a7e2596378a0010bd0be6491 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 23 Jul 2026 23:34:36 +0200 Subject: [PATCH 14/91] feat: run popen via-gateways on Trio via GATEWAY_START_POPEN Move the common `popen//via=` proxy transport onto the Trio host with a new protocol message instead of remote_exec'ing gateway_io source. The coordinator sends GATEWAY_START_POPEN (with the sub worker config) on a request channel; the master's Trio host spawns a `python -m execnet._trio_worker` sub-worker and relays its Message protocol raw over that channel (stdin<-channel, stdout->channel). The coordinator wraps the channel as ChannelByteIO and runs a normal Trio session over it. ChannelByteIO is marked an interim hack: it tunnels sub frames as CHANNEL_DATA (double-framing) and should become a proper relayed transport. ssh/foreign-python via sub-gateways still use the legacy path. Co-Authored-By: Claude Opus 4.8 --- doc/implnotes.rst | 13 +++- src/execnet/_trio_host.py | 136 ++++++++++++++++++++++++++++++++++++ src/execnet/gateway_base.py | 10 +++ src/execnet/multi.py | 2 + 4 files changed, 159 insertions(+), 2 deletions(-) diff --git a/doc/implnotes.rst b/doc/implnotes.rst index 29df051c..ba1c8408 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -58,8 +58,17 @@ threads wait until the frame is written (so abrupt ``os._exit`` cannot drop queued data). Sends from the Trio host thread (receiver callbacks) only enqueue, to avoid deadlocking the writer task. -Disable with ``EXECNET_TRIO_HOST=0``. Other gateway types (``socket``, -``via``, greenlet execmodels) still use the legacy thread receiver and sync +``socket`` and ``via`` gateways run on the Trio host too. ``socket`` +connects a Trio TCP stream to an ``execnet-socketserver`` (itself a Trio +listener spawning ``python -m execnet._trio_worker --socket-fd`` subprocess +workers). Infrastructure that used to be driven by ``remote_exec``-ing +source is now expressed as native protocol messages handled on the target's +Trio host: ``GATEWAY_START_SOCKET`` (installvia -> bind a one-shot listener, +reply with its address) and ``GATEWAY_START_POPEN`` (``via`` -> spawn a popen +sub-worker and relay its protocol over the request channel). + +Disable with ``EXECNET_TRIO_HOST=0``. ssh/foreign-python ``via`` sub-gateways +and greenlet execmodels still use the legacy thread receiver and sync ``Popen`` path. Legacy thread model diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index ed78dd7d..e00f3de0 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -9,6 +9,7 @@ import itertools import json +import math import os import queue import subprocess @@ -165,6 +166,49 @@ async def aclose_write(self) -> None: await self._stream.aclose() +_CHANNEL_EOF = object() + + +class ChannelByteIO: + """Async byte IO tunnelled over a sync execnet ``Channel``. + + INTERIM HACK: the ``via`` transport currently runs the sub-gateway protocol + as raw bytes over a channel to the master, which double-frames it (sub frame + -> CHANNEL_DATA -> master frame). This bridges the sync ``Channel`` to the + async protocol; it should be replaced by a proper relayed transport rather + than tunnelling bytes through the channel layer. + + The channel callback (on the host loop) feeds an unbounded memory channel + that ``read_exact`` drains; writes ``channel.send`` raw frames. + """ + + def __init__(self, channel: Any) -> None: + self._channel = channel + self._send, self._recv = trio.open_memory_channel[Any](math.inf) + self._buf = bytearray() + channel.setcallback(self._send.send_nowait, endmarker=_CHANNEL_EOF) + + async def read_exact(self, n: int) -> bytes: + while len(self._buf) < n: + data = await self._recv.receive() + if data is _CHANNEL_EOF: + raise EOFError("channel closed") + assert isinstance(data, bytes) + self._buf += data + out = bytes(self._buf[:n]) + del self._buf[:n] + return out + + async def write_all(self, data: bytes) -> None: + self._channel.send(data) + + async def aclose_read(self) -> None: + return + + async def aclose_write(self) -> None: + self._channel.close() + + async def adopt_socket(socket_fd: int) -> SocketStreamIO: """Worker side: wrap an inherited socket fd and send the handshake. @@ -828,3 +872,95 @@ def start_socketserver_via( if not realhost or realhost in ("0.0.0.0", "::"): realhost = "localhost" return realhost, int(realport) + + +async def _start_popen_and_relay( + gateway: BaseGateway, channelid: int, worker_config: str +) -> None: + """Spawn a popen sub-worker and relay its Message protocol over the channel. + + Runs on the master's Trio host: bytes from the channel go to the sub's + stdin, and the sub's stdout goes back on the channel (the ``via`` transport). + """ + args = [sys.executable, "-u", "-m", "execnet._trio_worker", worker_config] + process = await open_popen_process(args) + channel = gateway._channelfactory.new(channelid) + send_ch, recv_ch = trio.open_memory_channel[Any](math.inf) + channel.setcallback(send_ch.send_nowait, endmarker=_CHANNEL_EOF) + + async def coordinator_to_sub() -> None: + assert process.stdin is not None + async for data in recv_ch: + if data is _CHANNEL_EOF: + break + await process.stdin.send_all(data) + with trio.move_on_after(5): + await process.stdin.aclose() + + async def sub_to_coordinator() -> None: + assert process.stdout is not None + while True: + data = await process.stdout.receive_some(65536) + if not data: + break + channel.send(data) + channel.close() + + try: + async with trio.open_nursery() as nursery: + nursery.start_soon(coordinator_to_sub) + nursery.start_soon(sub_to_coordinator) + finally: + with trio.move_on_after(5): + await process.wait() + + +def handle_start_popen(gateway: BaseGateway, channelid: int, data: bytes) -> None: + """Worker handler for ``Message.GATEWAY_START_POPEN`` (on the host thread).""" + worker_config = loads_internal(data) + assert isinstance(worker_config, str) + host: TrioHost = gateway._trio_exec.host # type: ignore[attr-defined] + host.start_soon(_start_popen_and_relay, gateway, channelid, worker_config) + + +def should_use_trio_via(spec: Any) -> bool: + """Trio path for ``popen//via=`` (a popen sub-gateway on the master). + + Only a same-interpreter popen sub is supported for now (no ssh/foreign sub). + """ + if not trio_host_enabled(): + return False + if not getattr(spec, "via", None): + return False + if getattr(spec, "socket", None) or getattr(spec, "ssh", None): + return False + if getattr(spec, "python", None): + return False + execmodel = getattr(spec, "execmodel", None) + return execmodel in (None, "thread", "main_thread_only") + + +def makegateway_via_trio(group: Any, spec: Any) -> Gateway: + """Create a ``via`` gateway: a popen sub relayed through the master gateway.""" + import execnet + + from . import _provision + + master = group[spec.via] + host: TrioHost = group._ensure_trio_host() + channel = master.newchannel() + worker_config = _provision.worker_cli_arg(spec) + master._send(Message.GATEWAY_START_POPEN, channel.id, dumps_internal(worker_config)) + + async def _create_and_attach() -> Gateway: + io = ChannelByteIO(channel) + ack = await io.read_exact(1) + if ack != b"1": + raise EOFError(f"bad via handshake: {ack!r}") + gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + session = await host.start_session(gw, io) + gw._attach_trio_session(session) + gw._io = SyncIOHandle(group.execmodel, session) + return gw + + return host.call(_create_and_attach) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 764f6a0a..679e8fd5 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -559,6 +559,16 @@ def _gateway_start_socket(message: Message, gateway: BaseGateway) -> None: GATEWAY_START_SOCKET = 8 _types[GATEWAY_START_SOCKET] = ("GATEWAY_START_SOCKET", _gateway_start_socket) + def _gateway_start_popen(message: Message, gateway: BaseGateway) -> None: + # Spawn a popen worker subprocess on this (Trio) gateway's host and relay + # its Message protocol over the request channel (the ``via`` transport). + from . import _trio_host + + _trio_host.handle_start_popen(gateway, message.channelid, message.data) + + GATEWAY_START_POPEN = 9 + _types[GATEWAY_START_POPEN] = ("GATEWAY_START_POPEN", _gateway_start_popen) + class GatewayReceivedTerminate(Exception): """Receiverthread got termination message.""" diff --git a/src/execnet/multi.py b/src/execnet/multi.py index cc85d757..3a9dc602 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -160,6 +160,8 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: gw = _trio_host.makegateway_ssh_trio(self, spec) elif _trio_host.should_use_trio_socket(spec): gw = _trio_host.makegateway_socket_trio(self, spec) + elif _trio_host.should_use_trio_via(spec): + gw = _trio_host.makegateway_via_trio(self, spec) elif spec.via: assert not spec.socket master = self[spec.via] From 92969c5a269a223a2d2edc8ef55e1c2abdb58b3e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 02:32:10 +0200 Subject: [PATCH 15/91] feat: spawn arbitrary via sub-gateways with GATEWAY_START_SUB Generalize the via relay message from popen-only to a spawn-request dict carrying the sub-spec essentials plus provisioning material: a released coordinator sends a pip requirement, a dev coordinator ships its wheel for the master to materialize into the local wheel cache. The master resolves the launch locally (direct module, uv-provisioned python=, or ssh with the wheel streamed as stdin preamble), so ssh= and python= sub specs now run on the Trio path. Also route ssh=...//via=... through the via path instead of opening a direct ssh connection, and close the request channel with an error on spawn/relay failure instead of crashing the host nursery. Co-Authored-By: Claude Fable 5 --- src/execnet/_provision.py | 182 +++++++++++++++++++++++++++++++----- src/execnet/_trio_host.py | 71 ++++++++------ src/execnet/gateway_base.py | 13 +-- testing/test_multi.py | 18 ++++ testing/test_ssh_local.py | 20 ++++ 5 files changed, 249 insertions(+), 55 deletions(-) diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index b24bfe8b..83fd1515 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -17,10 +17,12 @@ from __future__ import annotations import json +import os import re import shlex import shutil import subprocess +import sys import tempfile from functools import cache from pathlib import Path @@ -162,52 +164,62 @@ def worker_cli_arg(spec: Any) -> str: ) +def _worker_tokens(config: str) -> list[str]: + """``python -u -m execnet._trio_worker `` tokens. + + The literal ``python`` token is resolved by uv (inside the provisioned + environment) or the remote shell. + """ + return ["python", "-u", "-m", "execnet._trio_worker", config] + + def worker_module_tokens(spec: Any) -> list[str]: - """``python -u -m execnet._trio_worker `` tokens.""" - return ["python", "-u", "-m", "execnet._trio_worker", worker_cli_arg(spec)] + """``python -u -m execnet._trio_worker `` tokens for ``spec``.""" + return _worker_tokens(worker_cli_arg(spec)) -def _uv_prefix(spec: Any) -> list[str]: +def _uv_tokens(python: str | None) -> list[str]: # --no-project keeps the surrounding execnet checkout from being synced. prefix = ["uv", "run", "--no-project"] - if spec.python: - prefix += ["--python", spec.python] + if python: + prefix += ["--python", python] return prefix def uv_worker_argv(spec: Any) -> list[str]: """``uv run`` argv to launch the Trio worker locally (wheel path is local).""" return [ - *_uv_prefix(spec), + *_uv_tokens(spec.python), "--with", coordinator_requirement(), *worker_module_tokens(spec), ] -def ssh_remote_command(spec: Any) -> tuple[str, bytes]: - """Remote shell command + stdin preamble to launch the worker over ssh. +def _remote_shell_command( + python: str | None, + config: str, + *, + requirement: str | None = None, + wheel: Path | None = None, +) -> tuple[str, bytes]: + """Remote sh command + stdin preamble launching the worker via uv. - Released coordinator -> ``uv run --with execnet== …`` with no preamble. - Dev coordinator -> a POSIX-sh prelude that receives the wheel bytes from - stdin (``head -c N``) into a temp dir and ``exec``s uv against it; the wheel - bytes are returned as the preamble to stream before the Message protocol. + With ``requirement`` the remote installs from an index and no preamble is + needed. With ``wheel`` a POSIX-sh prelude receives the wheel bytes from + stdin (``head -c N``) into a temp dir and ``exec``s uv against it; the + wheel bytes are returned as the preamble to stream before the protocol. """ - import execnet + worker = _worker_tokens(config) + uv = _uv_tokens(python) + if wheel is None: + assert requirement is not None + return shlex.join([*uv, "--with", requirement, *worker]), b"" - version = execnet.__version__ - worker = worker_module_tokens(spec) - if _RELEASED_RE.match(version): - command = shlex.join( - [*_uv_prefix(spec), "--with", f"execnet=={version}", *worker] - ) - return command, b"" - - wheel = _build_wheel(version) data = wheel.read_bytes() # "$d/": expand the temp dir, concatenate the (quoted) wheel filename. remote_wheel = '"$d/"' + shlex.quote(wheel.name) - uv_run = " ".join(shlex.quote(token) for token in [*_uv_prefix(spec), "--with"]) + uv_run = " ".join(shlex.quote(token) for token in [*uv, "--with"]) worker_cmd = " ".join(shlex.quote(token) for token in worker) prelude = ( f"d=$(mktemp -d) && " @@ -215,3 +227,127 @@ def ssh_remote_command(spec: Any) -> tuple[str, bytes]: f"exec {uv_run} {remote_wheel} {worker_cmd}" ) return prelude, data + + +def ssh_remote_command(spec: Any) -> tuple[str, bytes]: + """Remote shell command + stdin preamble to launch the worker over ssh. + + Released coordinator -> ``uv run --with execnet== …`` with no preamble. + Dev coordinator -> wheel-shipping prelude (see ``_remote_shell_command``). + """ + import execnet + + version = execnet.__version__ + config = worker_cli_arg(spec) + if _RELEASED_RE.match(version): + return _remote_shell_command( + spec.python, config, requirement=f"execnet=={version}" + ) + return _remote_shell_command(spec.python, config, wheel=_build_wheel(version)) + + +def ssh_argv(ssh: str, ssh_config: str | None, remote_command: str) -> list[str]: + """``ssh`` client argv running ``remote_command`` on host ``ssh``.""" + args = ["ssh", "-C"] + if ssh_config: + args += ["-F", ssh_config] + args += ssh.split() + args.append(remote_command) + return args + + +def spawn_request(spec: Any) -> dict[str, Any]: + """Payload for ``GATEWAY_START_SUB``: ask a via master to spawn a sub-worker. + + Carries the sub-spec essentials plus provisioning material when the sub + may need it (ssh or foreign python): a released coordinator sends a pip + requirement; a dev coordinator ships its wheel bytes for the master to + materialize into its local wheel cache. + + TODO: the wheel is shipped eagerly because only the master can tell + whether the target interpreter already has execnet; a wheel-on-demand + round-trip would avoid the transfer in the common provisioned case. + """ + import execnet + + request: dict[str, Any] = { + "config": worker_cli_arg(spec), + "python": spec.python or None, + "ssh": spec.ssh or None, + "ssh_config": spec.ssh_config or None, + } + if spec.ssh or spec.python: + version = execnet.__version__ + if _RELEASED_RE.match(version): + request["requirement"] = f"execnet=={version}" + else: + wheel = _build_wheel(version) + request["wheel"] = (wheel.name, wheel.read_bytes()) + return request + + +def materialize_wheel(name: str, data: bytes) -> Path: + """Write shipped wheel bytes into the local wheel cache (idempotent).""" + target = _wheel_cache_dir() / name + if not target.exists(): + tmp = target.with_name(f"{target.name}.{os.getpid()}.tmp") + tmp.write_bytes(data) + tmp.replace(target) + return target + + +def _requested_requirement(request: dict[str, Any]) -> tuple[str | None, Path | None]: + """(uv requirement, local wheel path) from a spawn request's material.""" + requirement = request.get("requirement") + if isinstance(requirement, str): + return requirement, None + shipped = request.get("wheel") + if shipped is not None: + name, data = shipped + path = materialize_wheel(name, data) + return str(path), path + return None, None + + +def sub_spawn_argv(request: dict[str, Any]) -> tuple[list[str], bytes]: + """(argv, stdin preamble) spawning a requested sub-worker on this host. + + Handles a ``GATEWAY_START_SUB`` request on a via master: plain popen runs + this interpreter's worker module, a foreign ``python`` runs directly when + it already has execnet and is uv-provisioned otherwise, and ``ssh`` wraps + the remote uv command (streaming a shipped wheel as the preamble for dev + versions). + """ + from .gateway_io import shell_split_path + + config = request["config"] + assert isinstance(config, str) + python = request.get("python") + ssh = request.get("ssh") + if ssh: + assert isinstance(ssh, str) + requirement, wheel = _requested_requirement(request) + if requirement is None: + raise RuntimeError("ssh spawn request without provisioning material") + command, preamble = _remote_shell_command( + python, config, requirement=requirement, wheel=wheel + ) + return ssh_argv(ssh, request.get("ssh_config"), command), preamble + if python: + assert isinstance(python, str) + if target_has_execnet(python): + argv = [*shell_split_path(python), "-u", "-m", "execnet._trio_worker"] + return [*argv, config], b"" + requirement, _ = _requested_requirement(request) + if requirement is None or not uv_available(): + raise RuntimeError( + f"cannot provision sub-worker for python={python!r}: " + "uv and provisioning material required" + ) + return [ + *_uv_tokens(python), + "--with", + requirement, + *_worker_tokens(config), + ], b"" + return [sys.executable, "-u", "-m", "execnet._trio_worker", config], b"" diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index e00f3de0..709a8b64 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -17,6 +17,7 @@ import threading from collections.abc import Awaitable from collections.abc import Callable +from contextlib import suppress from typing import TYPE_CHECKING from typing import Any from typing import Protocol @@ -712,13 +713,8 @@ def ssh_trio_args(spec: Any) -> tuple[list[str], bytes]: from . import _provision remote_command, preamble = _provision.ssh_remote_command(spec) - args = ["ssh", "-C"] - if getattr(spec, "ssh_config", None): - args += ["-F", spec.ssh_config] assert spec.ssh is not None - args += spec.ssh.split() - args.append(remote_command) - return args, preamble + return _provision.ssh_argv(spec.ssh, spec.ssh_config, remote_command), preamble def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: @@ -730,11 +726,17 @@ def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: def should_use_trio_ssh(spec: Any) -> bool: - """Trio path for ssh gateways (worker provisioned on the remote via uv).""" + """Trio path for ssh gateways (worker provisioned on the remote via uv). + + ``ssh=…//via=…`` is not a direct ssh connection but a sub-gateway spawned + by the master; that goes through the via path instead. + """ if not trio_host_enabled(): return False if not getattr(spec, "ssh", None): return False + if getattr(spec, "via", None): + return False execmodel = getattr(spec, "execmodel", None) return execmodel in (None, "thread", "main_thread_only") @@ -874,22 +876,32 @@ def start_socketserver_via( return realhost, int(realport) -async def _start_popen_and_relay( - gateway: BaseGateway, channelid: int, worker_config: str +async def _start_sub_and_relay( + gateway: BaseGateway, channelid: int, request: dict[str, Any] ) -> None: - """Spawn a popen sub-worker and relay its Message protocol over the channel. + """Spawn a requested sub-worker and relay its Message protocol over the channel. Runs on the master's Trio host: bytes from the channel go to the sub's stdin, and the sub's stdout goes back on the channel (the ``via`` transport). + A stdin preamble (shipped wheel for a dev-version ssh sub) is streamed + before the relayed protocol bytes. """ - args = [sys.executable, "-u", "-m", "execnet._trio_worker", worker_config] - process = await open_popen_process(args) + from . import _provision + channel = gateway._channelfactory.new(channelid) + try: + args, preamble = _provision.sub_spawn_argv(request) + process = await open_popen_process(args) + except Exception as exc: + channel.close(f"could not spawn via sub-gateway: {exc}") + return send_ch, recv_ch = trio.open_memory_channel[Any](math.inf) channel.setcallback(send_ch.send_nowait, endmarker=_CHANNEL_EOF) async def coordinator_to_sub() -> None: assert process.stdin is not None + if preamble: + await process.stdin.send_all(preamble) async for data in recv_ch: if data is _CHANNEL_EOF: break @@ -910,38 +922,44 @@ async def sub_to_coordinator() -> None: async with trio.open_nursery() as nursery: nursery.start_soon(coordinator_to_sub) nursery.start_soon(sub_to_coordinator) + except Exception as exc: + # Do not let a relay failure crash the host nursery; surface it on + # the channel so the coordinator does not hang on the handshake. + gateway._trace("via sub relay failed:", exc) + with suppress(Exception): + channel.close(f"via sub-gateway relay failed: {exc}") finally: with trio.move_on_after(5): await process.wait() -def handle_start_popen(gateway: BaseGateway, channelid: int, data: bytes) -> None: - """Worker handler for ``Message.GATEWAY_START_POPEN`` (on the host thread).""" - worker_config = loads_internal(data) - assert isinstance(worker_config, str) +def handle_start_sub(gateway: BaseGateway, channelid: int, data: bytes) -> None: + """Worker handler for ``Message.GATEWAY_START_SUB`` (on the host thread).""" + request = loads_internal(data) + assert isinstance(request, dict) host: TrioHost = gateway._trio_exec.host # type: ignore[attr-defined] - host.start_soon(_start_popen_and_relay, gateway, channelid, worker_config) + host.start_soon(_start_sub_and_relay, gateway, channelid, request) def should_use_trio_via(spec: Any) -> bool: - """Trio path for ``popen//via=`` (a popen sub-gateway on the master). + """Trio path for ``via=`` sub-gateways (popen, foreign python, or ssh). - Only a same-interpreter popen sub is supported for now (no ssh/foreign sub). + The master spawns the sub-worker from a ``GATEWAY_START_SUB`` request and + relays its Message protocol. Socket subs go through ``installvia`` + instead; vagrant stays on the legacy path. """ if not trio_host_enabled(): return False if not getattr(spec, "via", None): return False - if getattr(spec, "socket", None) or getattr(spec, "ssh", None): - return False - if getattr(spec, "python", None): + if getattr(spec, "socket", None) or getattr(spec, "vagrant_ssh", None): return False execmodel = getattr(spec, "execmodel", None) return execmodel in (None, "thread", "main_thread_only") def makegateway_via_trio(group: Any, spec: Any) -> Gateway: - """Create a ``via`` gateway: a popen sub relayed through the master gateway.""" + """Create a ``via`` gateway: a sub-worker spawned by and relayed through the master.""" import execnet from . import _provision @@ -949,8 +967,9 @@ def makegateway_via_trio(group: Any, spec: Any) -> Gateway: master = group[spec.via] host: TrioHost = group._ensure_trio_host() channel = master.newchannel() - worker_config = _provision.worker_cli_arg(spec) - master._send(Message.GATEWAY_START_POPEN, channel.id, dumps_internal(worker_config)) + request = _provision.spawn_request(spec) + master._send(Message.GATEWAY_START_SUB, channel.id, dumps_internal(request)) + remoteaddress = f"{spec.ssh}[via {spec.via}]" if spec.ssh else None async def _create_and_attach() -> Gateway: io = ChannelByteIO(channel) @@ -960,7 +979,7 @@ async def _create_and_attach() -> Gateway: gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) session = await host.start_session(gw, io) gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session) + gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) return gw return host.call(_create_and_attach) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 679e8fd5..2736be96 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -559,15 +559,16 @@ def _gateway_start_socket(message: Message, gateway: BaseGateway) -> None: GATEWAY_START_SOCKET = 8 _types[GATEWAY_START_SOCKET] = ("GATEWAY_START_SOCKET", _gateway_start_socket) - def _gateway_start_popen(message: Message, gateway: BaseGateway) -> None: - # Spawn a popen worker subprocess on this (Trio) gateway's host and relay - # its Message protocol over the request channel (the ``via`` transport). + def _gateway_start_sub(message: Message, gateway: BaseGateway) -> None: + # Spawn a sub-gateway worker (popen, foreign python, or ssh) on this + # (Trio) gateway's host and relay its Message protocol over the request + # channel (the ``via`` transport). from . import _trio_host - _trio_host.handle_start_popen(gateway, message.channelid, message.data) + _trio_host.handle_start_sub(gateway, message.channelid, message.data) - GATEWAY_START_POPEN = 9 - _types[GATEWAY_START_POPEN] = ("GATEWAY_START_POPEN", _gateway_start_popen) + GATEWAY_START_SUB = 9 + _types[GATEWAY_START_SUB] = ("GATEWAY_START_SUB", _gateway_start_sub) class GatewayReceivedTerminate(Exception): diff --git a/testing/test_multi.py b/testing/test_multi.py index f3166503..e0390298 100644 --- a/testing/test_multi.py +++ b/testing/test_multi.py @@ -231,6 +231,24 @@ def test_terminate_with_proxying(self) -> None: group.makegateway("popen//via=master//id=worker") group.terminate(1.0) + def test_via_foreign_python(self) -> None: + # A python= sub-spec through a via master: the master resolves the + # interpreter locally (this interpreter has execnet, so the sub runs + # the worker module directly, no uv provisioning). + import sys + + group = Group() + try: + group.makegateway("popen//id=master") + gw = group.makegateway( + f"popen//python={sys.executable}//via=master//id=sub" + ) + channel = gw.remote_exec("channel.send(channel.receive() + 1)") + channel.send(41) + assert channel.receive() == 42 + finally: + group.terminate(1.0) + @pytest.mark.xfail(reason="active_count() has been broken for some time") def test_safe_terminate(execmodel: ExecModel) -> None: diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py index 72dea348..c21135d1 100644 --- a/testing/test_ssh_local.py +++ b/testing/test_ssh_local.py @@ -149,3 +149,23 @@ def test_ssh_roundtrip( assert channel.receive() == 42 finally: group.terminate(timeout=5.0) + + +def test_ssh_via_roundtrip(ssh_config: str) -> None: + """An ssh sub-gateway spawned by a popen master (GATEWAY_START_SUB relay). + + The master runs the ssh client; for a dev coordinator the wheel travels + coordinator -> master (in the spawn request) -> remote (ssh stdin preamble). + """ + group = execnet.Group() + try: + group.makegateway("popen//id=master") + gw = group.makegateway( + f"ssh=testhost//ssh_config={ssh_config}//python={sys.executable}" + "//via=master//id=sshvia" + ) + channel = gw.remote_exec("channel.send(channel.receive() + 1)") + channel.send(41) + assert channel.receive() == 42 + finally: + group.terminate(timeout=5.0) From 48d328c6d61f58a342fb6c5bbfe6f860c099d4ef Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 02:53:29 +0200 Subject: [PATCH 16/91] feat: run vagrant_ssh gateways on the Trio path vagrant_ssh= now launches the uv-provisioned worker through `vagrant ssh -- -C ` (mirroring the ssh argv), both as a direct gateway and as a via sub-gateway through GATEWAY_START_SUB. Co-Authored-By: Claude Fable 5 --- src/execnet/_provision.py | 31 ++++++++++++++++++++++++++----- src/execnet/_trio_host.py | 36 +++++++++++++++++++++++++++++++----- src/execnet/multi.py | 2 ++ testing/test_provision.py | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 10 deletions(-) diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index 83fd1515..c55fb431 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -256,6 +256,21 @@ def ssh_argv(ssh: str, ssh_config: str | None, remote_command: str) -> list[str] return args +def vagrant_ssh_argv( + machine: str, ssh_config: str | None, remote_command: str +) -> list[str]: + """``vagrant ssh`` argv running ``remote_command`` on the named VM. + + Everything after ``--`` is passed through to the underlying ssh client, + mirroring ``ssh_argv``. + """ + args = ["vagrant", "ssh", machine, "--", "-C"] + if ssh_config: + args += ["-F", ssh_config] + args.append(remote_command) + return args + + def spawn_request(spec: Any) -> dict[str, Any]: """Payload for ``GATEWAY_START_SUB``: ask a via master to spawn a sub-worker. @@ -274,9 +289,10 @@ def spawn_request(spec: Any) -> dict[str, Any]: "config": worker_cli_arg(spec), "python": spec.python or None, "ssh": spec.ssh or None, + "vagrant_ssh": spec.vagrant_ssh or None, "ssh_config": spec.ssh_config or None, } - if spec.ssh or spec.python: + if spec.ssh or spec.vagrant_ssh or spec.python: version = execnet.__version__ if _RELEASED_RE.match(version): request["requirement"] = f"execnet=={version}" @@ -324,15 +340,20 @@ def sub_spawn_argv(request: dict[str, Any]) -> tuple[list[str], bytes]: assert isinstance(config, str) python = request.get("python") ssh = request.get("ssh") - if ssh: - assert isinstance(ssh, str) + vagrant = request.get("vagrant_ssh") + if ssh or vagrant: requirement, wheel = _requested_requirement(request) if requirement is None: - raise RuntimeError("ssh spawn request without provisioning material") + raise RuntimeError("remote spawn request without provisioning material") command, preamble = _remote_shell_command( python, config, requirement=requirement, wheel=wheel ) - return ssh_argv(ssh, request.get("ssh_config"), command), preamble + ssh_config = request.get("ssh_config") + if ssh: + assert isinstance(ssh, str) + return ssh_argv(ssh, ssh_config, command), preamble + assert isinstance(vagrant, str) + return vagrant_ssh_argv(vagrant, ssh_config, command), preamble if python: assert isinstance(python, str) if target_has_execnet(python): diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 709a8b64..2a612996 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -725,6 +725,32 @@ def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: ) +def should_use_trio_vagrant(spec: Any) -> bool: + """Trio path for ``vagrant_ssh=`` gateways (uv-provisioned worker).""" + if not trio_host_enabled(): + return False + if not getattr(spec, "vagrant_ssh", None): + return False + if getattr(spec, "via", None): + return False + execmodel = getattr(spec, "execmodel", None) + return execmodel in (None, "thread", "main_thread_only") + + +def makegateway_vagrant_trio(group: Any, spec: Any) -> Gateway: + """Create a ``vagrant ssh``-wrapped Gateway on the Trio IO path.""" + from . import _provision + + remote_command, preamble = _provision.ssh_remote_command(spec) + assert spec.vagrant_ssh is not None + args = _provision.vagrant_ssh_argv( + spec.vagrant_ssh, spec.ssh_config, remote_command + ) + return _open_trio_gateway( + group, spec, args, remoteaddress=spec.vagrant_ssh, preamble=preamble + ) + + def should_use_trio_ssh(spec: Any) -> bool: """Trio path for ssh gateways (worker provisioned on the remote via uv). @@ -942,17 +968,16 @@ def handle_start_sub(gateway: BaseGateway, channelid: int, data: bytes) -> None: def should_use_trio_via(spec: Any) -> bool: - """Trio path for ``via=`` sub-gateways (popen, foreign python, or ssh). + """Trio path for ``via=`` sub-gateways (popen, python, ssh, vagrant). The master spawns the sub-worker from a ``GATEWAY_START_SUB`` request and - relays its Message protocol. Socket subs go through ``installvia`` - instead; vagrant stays on the legacy path. + relays its Message protocol. Socket subs go through ``installvia`` instead. """ if not trio_host_enabled(): return False if not getattr(spec, "via", None): return False - if getattr(spec, "socket", None) or getattr(spec, "vagrant_ssh", None): + if getattr(spec, "socket", None): return False execmodel = getattr(spec, "execmodel", None) return execmodel in (None, "thread", "main_thread_only") @@ -969,7 +994,8 @@ def makegateway_via_trio(group: Any, spec: Any) -> Gateway: channel = master.newchannel() request = _provision.spawn_request(spec) master._send(Message.GATEWAY_START_SUB, channel.id, dumps_internal(request)) - remoteaddress = f"{spec.ssh}[via {spec.via}]" if spec.ssh else None + remote = spec.ssh or spec.vagrant_ssh + remoteaddress = f"{remote}[via {spec.via}]" if remote else None async def _create_and_attach() -> Gateway: io = ChannelByteIO(channel) diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 3a9dc602..29b8bd6a 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -158,6 +158,8 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: gw = _trio_host.makegateway_popen_trio(self, spec) elif _trio_host.should_use_trio_ssh(spec): gw = _trio_host.makegateway_ssh_trio(self, spec) + elif _trio_host.should_use_trio_vagrant(spec): + gw = _trio_host.makegateway_vagrant_trio(self, spec) elif _trio_host.should_use_trio_socket(spec): gw = _trio_host.makegateway_socket_trio(self, spec) elif _trio_host.should_use_trio_via(spec): diff --git a/testing/test_provision.py b/testing/test_provision.py index 8df8775a..84e660e5 100644 --- a/testing/test_provision.py +++ b/testing/test_provision.py @@ -38,3 +38,39 @@ def test_ssh_remote_command_dev_ships_wheel() -> None: assert "mktemp -d" in command assert f"head -c {len(preamble)}" in command assert preamble[:2] == b"PK" # a wheel is a zip archive + + +def test_vagrant_ssh_argv() -> None: + argv = _provision.vagrant_ssh_argv("default", None, "run-worker") + assert argv == ["vagrant", "ssh", "default", "--", "-C", "run-worker"] + argv = _provision.vagrant_ssh_argv("default", "/tmp/cfg", "run-worker") + assert argv == [ + "vagrant", + "ssh", + "default", + "--", + "-C", + "-F", + "/tmp/cfg", + "run-worker", + ] + + +def test_sub_spawn_argv_plain_popen() -> None: + import sys + + argv, preamble = _provision.sub_spawn_argv({"config": "{}"}) + assert argv == [sys.executable, "-u", "-m", "execnet._trio_worker", "{}"] + assert preamble == b"" + + +def test_sub_spawn_argv_vagrant_released() -> None: + request = { + "config": "{}", + "vagrant_ssh": "default", + "requirement": "execnet==9.9.9", + } + argv, preamble = _provision.sub_spawn_argv(request) + assert argv[:5] == ["vagrant", "ssh", "default", "--", "-C"] + assert "execnet==9.9.9" in argv[-1] + assert preamble == b"" From 3d1d31efb6be1fc56df527d2e4cae8bd4e67e50e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 09:48:52 +0200 Subject: [PATCH 17/91] refactor!: remove the legacy bootstrap stack and EXECNET_TRIO_HOST hatch The Trio host is now the only IO path. Delete gateway_io.py (ProxyIO, Popen2IOMaster, source bootstrap lines) and gateway_bootstrap.py, the Popen2IO sync pipe IO, the thread receiver, and WorkerGateway.serve(); makegateway dispatches directly on the spec. shell_split_path moves to _provision, and HostNotFound moves to gateway_base and now subclasses ConnectionError so generic OSError handling catches unreachable hosts. Co-Authored-By: Claude Fable 5 --- doc/implnotes.rst | 31 +--- src/execnet/__init__.py | 2 +- src/execnet/_provision.py | 15 +- src/execnet/_trio_host.py | 102 +------------ src/execnet/_trio_worker.py | 2 +- src/execnet/gateway.py | 23 +-- src/execnet/gateway_base.py | 190 +++-------------------- src/execnet/gateway_bootstrap.py | 74 --------- src/execnet/gateway_io.py | 255 ------------------------------- src/execnet/multi.py | 28 +--- testing/test_basics.py | 148 ++++-------------- testing/test_gateway.py | 26 +--- testing/test_ssh_local.py | 8 +- testing/test_xspec.py | 26 ++-- 14 files changed, 111 insertions(+), 819 deletions(-) delete mode 100644 src/execnet/gateway_bootstrap.py delete mode 100644 src/execnet/gateway_io.py diff --git a/doc/implnotes.rst b/doc/implnotes.rst index ba1c8408..eed92232 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -64,26 +64,11 @@ listener spawning ``python -m execnet._trio_worker --socket-fd`` subprocess workers). Infrastructure that used to be driven by ``remote_exec``-ing source is now expressed as native protocol messages handled on the target's Trio host: ``GATEWAY_START_SOCKET`` (installvia -> bind a one-shot listener, -reply with its address) and ``GATEWAY_START_POPEN`` (``via`` -> spawn a popen -sub-worker and relay its protocol over the request channel). - -Disable with ``EXECNET_TRIO_HOST=0``. ssh/foreign-python ``via`` sub-gateways -and greenlet execmodels still use the legacy thread receiver and sync -``Popen`` path. - -Legacy thread model -------------------- - -After bootstrapping, ``BaseGateway`` opens a receiver thread which -accepts encoded messages and triggers actions to interpret them. -Sending of channel data items happens directly through -write operations to InputOutput objects so there is no -separate send thread. - -Code execution messages are scheduled on a WorkerPool. -On the worker, ``serve()`` integrates the main thread as the -primary executor when using the ``thread`` / ``main_thread_only`` -models. - -The receiver thread terminates if the remote side sends -a gateway termination message or if the IO-connection drops. +reply with its address) and ``GATEWAY_START_SUB`` (``via`` -> spawn a +sub-worker — popen, foreign python, ssh, or vagrant — and relay its protocol +over the request channel). + +The Trio host is the only IO path; the legacy thread receiver and the +source-shipping bootstrap have been removed. The receiver task terminates +when the remote side sends a gateway termination message or the +IO-connection drops. diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 1edf3254..cb91cb66 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -12,6 +12,7 @@ from .gateway_base import Channel from .gateway_base import DataFormatError from .gateway_base import DumpError +from .gateway_base import HostNotFound from .gateway_base import LoadError from .gateway_base import RemoteError from .gateway_base import TimeoutError @@ -19,7 +20,6 @@ from .gateway_base import dumps from .gateway_base import load from .gateway_base import loads -from .gateway_bootstrap import HostNotFound from .multi import Group from .multi import MultiChannel from .multi import default_group diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index c55fb431..b1c16335 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -36,6 +36,17 @@ def uv_available() -> bool: return shutil.which("uv") is not None +def shell_split_path(path: str) -> list[str]: + """Split a ``python=`` value into argv tokens with shell lexing. + + Takes care to handle Windows' ``\\`` correctly. + """ + if sys.platform.startswith("win"): + # replace \\ by / otherwise shlex will strip them out + path = path.replace("\\", "/") + return shlex.split(path) + + @cache def target_has_execnet(python: str) -> bool: """Whether interpreter ``python`` can already import execnet + trio. @@ -43,8 +54,6 @@ def target_has_execnet(python: str) -> bool: When true the worker can be launched directly on that interpreter (preserving ``sys.executable``); otherwise it must be uv-provisioned. """ - from .gateway_io import shell_split_path - argv = [*shell_split_path(python), "-c", "import execnet, trio"] try: completed = subprocess.run(argv, capture_output=True, timeout=30, check=False) @@ -334,8 +343,6 @@ def sub_spawn_argv(request: dict[str, Any]) -> tuple[list[str], bytes]: the remote uv command (streaming a shipped wheel as the preamble for dev versions). """ - from .gateway_io import shell_split_path - config = request["config"] assert isinstance(config, str) python = request.get("python") diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 2a612996..59f6b47f 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -10,7 +10,6 @@ import itertools import json import math -import os import queue import subprocess import sys @@ -40,37 +39,6 @@ T = TypeVar("T") _CLOSE_WRITE = object() -_ENABLED_ENV = "EXECNET_TRIO_HOST" - - -def trio_host_enabled() -> bool: - """Return whether the Trio IO path should be used when applicable.""" - value = os.environ.get(_ENABLED_ENV, "1").strip().lower() - return value not in ("0", "false", "no", "off") - - -def should_use_trio_popen(spec: Any) -> bool: - """Trio path for local popen. - - Same-interpreter popen launches the worker module directly; a foreign - interpreter (``python=``) is provisioned via ``uv`` and only taken when - ``uv`` is available (otherwise the legacy source-copy path handles it). - """ - if not trio_host_enabled(): - return False - if not getattr(spec, "popen", False): - return False - if getattr(spec, "via", None): - return False - execmodel = getattr(spec, "execmodel", None) - if execmodel not in (None, "thread", "main_thread_only"): - return False - if getattr(spec, "python", None): - from . import _provision - - # Direct launch if the interpreter already has execnet; else uv-provision. - return _provision.target_has_execnet(spec.python) or _provision.uv_available() - return True class AsyncByteIO(Protocol): @@ -467,7 +435,7 @@ async def _finish(self) -> None: self._wake_writer() gateway._trace("[trio-receiver] finishing channels") gateway._channelfactory._finished_receiving() - # Unblock WorkerGateway.serve()/join before heavy exec-pool shutdown + # Unblock the worker's join() before heavy exec-pool shutdown # so the primary thread is not waiting on _done while terminate waits # on the primary thread draining work. self._done.set() @@ -593,9 +561,7 @@ def popen_module_args(spec: Any) -> list[str]: from . import _provision if getattr(spec, "python", None): - from .gateway_io import shell_split_path - - interpreter = shell_split_path(spec.python) + interpreter = _provision.shell_split_path(spec.python) else: interpreter = [sys.executable] @@ -648,7 +614,7 @@ def _open_trio_gateway( """ import execnet - from .gateway_bootstrap import HostNotFound + from .gateway_base import HostNotFound host: TrioHost = group._ensure_trio_host() @@ -677,7 +643,7 @@ async def _create_and_attach() -> Gateway: await process.wait() raise - gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + gw = execnet.Gateway(_TempIO(group.execmodel), spec) session = await host.start_session(gw, async_io, process=process) gw._attach_trio_session(session) gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) @@ -725,18 +691,6 @@ def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: ) -def should_use_trio_vagrant(spec: Any) -> bool: - """Trio path for ``vagrant_ssh=`` gateways (uv-provisioned worker).""" - if not trio_host_enabled(): - return False - if not getattr(spec, "vagrant_ssh", None): - return False - if getattr(spec, "via", None): - return False - execmodel = getattr(spec, "execmodel", None) - return execmodel in (None, "thread", "main_thread_only") - - def makegateway_vagrant_trio(group: Any, spec: Any) -> Gateway: """Create a ``vagrant ssh``-wrapped Gateway on the Trio IO path.""" from . import _provision @@ -751,22 +705,6 @@ def makegateway_vagrant_trio(group: Any, spec: Any) -> Gateway: ) -def should_use_trio_ssh(spec: Any) -> bool: - """Trio path for ssh gateways (worker provisioned on the remote via uv). - - ``ssh=…//via=…`` is not a direct ssh connection but a sub-gateway spawned - by the master; that goes through the via path instead. - """ - if not trio_host_enabled(): - return False - if not getattr(spec, "ssh", None): - return False - if getattr(spec, "via", None): - return False - execmodel = getattr(spec, "execmodel", None) - return execmodel in (None, "thread", "main_thread_only") - - def makegateway_socket_trio(group: Any, spec: Any) -> Gateway: """Connect a Trio TCP stream to a running ``execnet-socketserver``. @@ -776,7 +714,7 @@ def makegateway_socket_trio(group: Any, spec: Any) -> Gateway: """ import execnet - from .gateway_bootstrap import HostNotFound + from .gateway_base import HostNotFound host: TrioHost = group._ensure_trio_host() if getattr(spec, "installvia", None): @@ -804,7 +742,7 @@ async def _create_and_attach() -> Gateway: await stream.aclose() raise - gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + gw = execnet.Gateway(_TempIO(group.execmodel), spec) session = await host.start_session(gw, io) gw._attach_trio_session(session) gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) @@ -813,16 +751,6 @@ async def _create_and_attach() -> Gateway: return host.call(_create_and_attach) -def should_use_trio_socket(spec: Any) -> bool: - """Trio path for ``socket=host:port`` gateways, including ``installvia``.""" - if not trio_host_enabled(): - return False - if not getattr(spec, "socket", None): - return False - execmodel = getattr(spec, "execmodel", None) - return execmodel in (None, "thread", "main_thread_only") - - _socket_worker_counter = itertools.count() @@ -967,22 +895,6 @@ def handle_start_sub(gateway: BaseGateway, channelid: int, data: bytes) -> None: host.start_soon(_start_sub_and_relay, gateway, channelid, request) -def should_use_trio_via(spec: Any) -> bool: - """Trio path for ``via=`` sub-gateways (popen, python, ssh, vagrant). - - The master spawns the sub-worker from a ``GATEWAY_START_SUB`` request and - relays its Message protocol. Socket subs go through ``installvia`` instead. - """ - if not trio_host_enabled(): - return False - if not getattr(spec, "via", None): - return False - if getattr(spec, "socket", None): - return False - execmodel = getattr(spec, "execmodel", None) - return execmodel in (None, "thread", "main_thread_only") - - def makegateway_via_trio(group: Any, spec: Any) -> Gateway: """Create a ``via`` gateway: a sub-worker spawned by and relayed through the master.""" import execnet @@ -1002,7 +914,7 @@ async def _create_and_attach() -> Gateway: ack = await io.read_exact(1) if ack != b"1": raise EOFError(f"bad via handshake: {ack!r}") - gw = execnet.Gateway(_TempIO(group.execmodel), spec, defer_receive=True) + gw = execnet.Gateway(_TempIO(group.execmodel), spec) session = await host.start_session(gw, io) gw._attach_trio_session(session) gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 5fc137fb..9e1f1be7 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -215,7 +215,7 @@ def _build_worker_gateway( main_thread_only = model.backend == "main_thread_only" trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) # Duck-type as WorkerPool for STATUS / _terminate_execution. - gateway._execpool = trio_exec # type: ignore[assignment] + gateway._execpool = trio_exec gateway._trio_exec = trio_exec gateway._executetask_complete = None if main_thread_only: diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index 6e336d6f..593bc6fe 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -26,21 +26,14 @@ class Gateway(gateway_base.BaseGateway): _group: Group - def __init__( - self, - io: IO, - spec: XSpec, - *, - trio_session: object | None = None, - defer_receive: bool = False, - ) -> None: - """:private:""" + def __init__(self, io: IO, spec: XSpec) -> None: + """:private: + + The Trio session doing the Message IO is attached separately via + ``_attach_trio_session`` once the connection is established. + """ super().__init__(io=io, id=spec.id, _startcount=1) self.spec = spec - if trio_session is not None: - self._attach_trio_session(trio_session) - elif not defer_receive: - self._initreceive() @property def remoteaddress(self) -> str: @@ -102,9 +95,7 @@ def _rinfo(self, update: bool = False) -> RInfo: def hasreceiver(self) -> bool: """Whether gateway is able to receive data.""" session = self._trio_session - if session is not None: - return bool(session.is_alive()) - return self._receivepool.active_count() > 0 + return session is not None and bool(session.is_alive()) def remote_status(self) -> RemoteStatus: """Obtain information about the remote execution status.""" diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 2736be96..0260b416 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1,4 +1,4 @@ -"""Base execnet gateway code send to the other side for bootstrapping. +"""Core gateway, channel and serialization code shared by coordinator and worker. :copyright: 2004-2015 :authors: @@ -389,48 +389,6 @@ def trace(*msg: object) -> None: notrace = trace = lambda *msg: None -class Popen2IO: - error = (IOError, OSError, EOFError) - - def __init__(self, outfile, infile, execmodel: ExecModel) -> None: - # we need raw byte streams - self.outfile, self.infile = outfile, infile - if sys.platform == "win32": - import msvcrt - - try: - msvcrt.setmode(infile.fileno(), os.O_BINARY) - msvcrt.setmode(outfile.fileno(), os.O_BINARY) - except (AttributeError, OSError): - pass - self._read = getattr(infile, "buffer", infile).read - self._write = getattr(outfile, "buffer", outfile).write - self.execmodel = execmodel - - def read(self, numbytes: int) -> bytes: - """Read exactly 'numbytes' bytes from the pipe.""" - # a file in non-blocking mode may return less bytes, so we loop - buf = b"" - while numbytes > len(buf): - data = self._read(numbytes - len(buf)) - if not data: - raise EOFError("expected %d bytes, got %d" % (numbytes, len(buf))) - buf += data - return buf - - def write(self, data: bytes) -> None: - """Write out all data bytes.""" - assert isinstance(data, bytes) - self._write(data) - self.outfile.flush() - - def close_read(self) -> None: - self.infile.close() - - def close_write(self) -> None: - self.outfile.close() - - class Message: """Encapsulates Messages and their wire protocol.""" @@ -572,7 +530,11 @@ def _gateway_start_sub(message: Message, gateway: BaseGateway) -> None: class GatewayReceivedTerminate(Exception): - """Receiverthread got termination message.""" + """Receiver got a gateway termination message.""" + + +class HostNotFound(ConnectionError): + """The remote side of a gateway could not be reached.""" def geterrortext( @@ -1043,6 +1005,8 @@ class BaseGateway: _sysex = sysex id = "" _trio_session: Any = None + # Set by the receiver on EOF without a prior termination message. + _error: BaseException | None = None def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel @@ -1054,7 +1018,6 @@ def __init__(self, io: IO, id, _startcount: int = 2) -> None: # globals may be NONE at process-termination self.__trace = trace self._geterrortext = geterrortext - self._receivepool = WorkerPool(self.execmodel) self._trio_session = None def _trace(self, *msg: object) -> None: @@ -1064,43 +1027,6 @@ def _attach_trio_session(self, session: Any) -> None: """Attach a Trio ProtocolSession for Message IO (no receiver thread).""" self._trio_session = session - def _initreceive(self) -> None: - if self._trio_session is not None: - return - self._receivepool.spawn(self._thread_receiver) - - def _thread_receiver(self) -> None: - def log(*msg: object) -> None: - self._trace("[receiver-thread]", *msg) - - log("RECEIVERTHREAD: starting to run") - io = self._io - try: - while 1: - msg = Message.from_io(io) - log("received", msg) - with self._receivelock: - msg.received(self) - del msg - except (KeyboardInterrupt, GatewayReceivedTerminate): - pass - except EOFError as exc: - log("EOF without prior gateway termination message") - self._error = exc - except Exception as exc: - log(self._geterrortext(exc)) - log("finishing receiving thread") - # wake up and terminate any execution waiting to receive - self._channelfactory._finished_receiving() - log("terminating execution") - self._terminate_execution() - log("closing read") - self._io.close_read() - log("closing write") - self._io.close_write() - log("terminating our receive pseudo pool") - self._receivepool.trigger_shutdown() - def _terminate_execution(self) -> None: pass @@ -1136,41 +1062,25 @@ def newchannel(self) -> Channel: return self._channelfactory.new() def join(self, timeout: float | None = None) -> None: - """Wait for receiverthread to terminate.""" - self._trace("waiting for receiver thread to finish") + """Wait for the receiver (Trio session) to terminate.""" + self._trace("waiting for receiver to finish") session = self._trio_session if session is not None: session.wait_done(timeout) - return - self._receivepool.waitall(timeout) class WorkerGateway(BaseGateway): _trio_exec: Any = None + # The exec pool (a TrioWorkerExec duck-typed as WorkerPool for STATUS). + _execpool: Any = None + _executetask_complete: Event | None = None def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: - trio_exec = getattr(self, "_trio_exec", None) - if trio_exec is not None: - trio_exec.schedule(channel, sourcetask) + trio_exec = self._trio_exec + if trio_exec is None: + channel.close("execution disallowed") return - - if self._execpool.execmodel.backend == "main_thread_only": - assert self._executetask_complete is not None - # It's necessary to wait for a short time in order to ensure - # that we do not report a false-positive deadlock error, since - # channel close does not elicit a response that would provide - # a guarantee to remote_exec callers that the previous task - # has released the main thread. If the timeout expires then it - # should be practically impossible to report a false-positive. - if not self._executetask_complete.wait(timeout=1): - channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) - return - # It's only safe to clear here because the above wait proves - # that there is not a previous task about to set it again. - self._executetask_complete.clear() - - sourcetask_ = loads_internal(sourcetask) - self._execpool.spawn(self.executetask, (channel, sourcetask_)) + trio_exec.schedule(channel, sourcetask) def _terminate_execution(self) -> None: # called from receiverthread @@ -1192,31 +1102,6 @@ def _terminate_execution(self) -> None: ) os._exit(1) - def serve(self) -> None: - def trace(msg: str) -> None: - self._trace("[serve] " + msg) - - hasprimary = self.execmodel.backend in ("thread", "main_thread_only") - self._execpool = WorkerPool(self.execmodel, hasprimary=hasprimary) - self._executetask_complete = None - if self.execmodel.backend == "main_thread_only": - self._executetask_complete = self.execmodel.Event() - # Initialize state to indicate that there is no previous task - # executing so that we don't need a separate flag to track this. - self._executetask_complete.set() - trace("spawning receiver thread") - self._initreceive() - try: - if hasprimary: - # this will return when we are in shutdown - trace("integrating as primary thread") - self._execpool.integrate_as_primary_thread() - trace("joining receiver thread") - self.join() - except KeyboardInterrupt: - # in the worker we can't really do anything sensible - trace("swallowing keyboardinterrupt, serve finished") - def executetask( self, item: tuple[Channel, tuple[str, str | None, str | None, dict[str, object]]], @@ -1701,44 +1586,3 @@ def save_frozenset(self, s: frozenset[object]) -> None: def save_Channel(self, channel: Channel) -> None: self._write(opcode.CHANNEL) self._write_int4(channel.id) - - -def init_popen_io(execmodel: ExecModel) -> Popen2IO: - if not hasattr(os, "dup"): # jython - io = Popen2IO(sys.stdout, sys.stdin, execmodel) - import tempfile - - sys.stdin = tempfile.TemporaryFile("r") - sys.stdout = tempfile.TemporaryFile("w") - else: - try: - devnull = os.devnull - except AttributeError: - devnull = "NUL" if os.name == "nt" else "/dev/null" - # stdin - stdin = execmodel.fdopen(os.dup(0), "r", 1) - fd = os.open(devnull, os.O_RDONLY) - os.dup2(fd, 0) - os.close(fd) - - # stdout - stdout = execmodel.fdopen(os.dup(1), "w", 1) - fd = os.open(devnull, os.O_WRONLY) - os.dup2(fd, 1) - - # stderr for win32 - if os.name == "nt": - sys.stderr = execmodel.fdopen(os.dup(2), "w", 1) - os.dup2(fd, 2) - os.close(fd) - io = Popen2IO(stdout, stdin, execmodel) - # Use closefd=False since 0 and 1 are shared with - # sys.__stdin__ and sys.__stdout__. - sys.stdin = execmodel.fdopen(0, "r", 1, closefd=False) - sys.stdout = execmodel.fdopen(1, "w", 1, closefd=False) - return io - - -def serve(io: IO, id) -> None: - trace(f"creating workergateway on {io!r}") - WorkerGateway(io=io, id=id, _startcount=2).serve() diff --git a/src/execnet/gateway_bootstrap.py b/src/execnet/gateway_bootstrap.py deleted file mode 100644 index 5707dff4..00000000 --- a/src/execnet/gateway_bootstrap.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Code to initialize the remote side of a gateway once the IO is created.""" - -from __future__ import annotations - -import inspect -import os - -import execnet - -from . import gateway_base -from .gateway_base import IO -from .xspec import XSpec - -importdir = os.path.dirname(os.path.dirname(execnet.__file__)) - - -class HostNotFound(Exception): - pass - - -def bootstrap_import(io: IO, spec: XSpec) -> None: - # Only insert the importdir into the path if we must. This prevents - # bugs where backports expect to be shadowed by the standard library on - # newer versions of python but would instead shadow the standard library. - sendexec( - io, - "import sys", - "if %r not in sys.path:" % importdir, - " sys.path.insert(0, %r)" % importdir, - "from execnet.gateway_base import serve, init_popen_io, get_execmodel", - "sys.stdout.write('1')", - "sys.stdout.flush()", - "execmodel = get_execmodel(%r)" % spec.execmodel, - "serve(init_popen_io(execmodel), id='%s-worker')" % spec.id, - ) - s = io.read(1) - assert s == b"1", repr(s) - - -def bootstrap_exec(io: IO, spec: XSpec) -> None: - try: - sendexec( - io, - inspect.getsource(gateway_base), - "execmodel = get_execmodel(%r)" % spec.execmodel, - "io = init_popen_io(execmodel)", - "io.write('1'.encode('ascii'))", - "serve(io, id='%s-worker')" % spec.id, - ) - s = io.read(1) - assert s == b"1" - except EOFError: - ret = io.wait() - if ret == 255 and hasattr(io, "remoteaddress"): - raise HostNotFound(io.remoteaddress) from None - - -def sendexec(io: IO, *sources: str) -> None: - source = "\n".join(sources) - io.write((repr(source) + "\n").encode("utf-8")) - - -def bootstrap(io: IO, spec: XSpec) -> execnet.Gateway: - if spec.popen: - if spec.via or spec.python: - bootstrap_exec(io, spec) - else: - bootstrap_import(io, spec) - elif spec.ssh or spec.vagrant_ssh: - bootstrap_exec(io, spec) - else: - raise ValueError("unknown gateway type, can't bootstrap") - gw = execnet.Gateway(io, spec) - return gw diff --git a/src/execnet/gateway_io.py b/src/execnet/gateway_io.py deleted file mode 100644 index 21285ab4..00000000 --- a/src/execnet/gateway_io.py +++ /dev/null @@ -1,255 +0,0 @@ -"""execnet IO initialization code. - -Creates IO instances used for gateway IO. -""" - -from __future__ import annotations - -import shlex -import sys -from typing import TYPE_CHECKING -from typing import cast - -if TYPE_CHECKING: - from execnet.gateway_base import Channel - from execnet.gateway_base import ExecModel - from execnet.xspec import XSpec - -try: - from execnet.gateway_base import Message - from execnet.gateway_base import Popen2IO -except ImportError: - from __main__ import Message # type: ignore[no-redef] - from __main__ import Popen2IO # type: ignore[no-redef] - -from functools import partial - - -class Popen2IOMaster(Popen2IO): - # Set externally, for some specs only. - remoteaddress: str - - def __init__(self, args, execmodel: ExecModel) -> None: - PIPE = execmodel.subprocess.PIPE - self.popen = p = execmodel.subprocess.Popen(args, stdout=PIPE, stdin=PIPE) - super().__init__(p.stdin, p.stdout, execmodel=execmodel) - - def wait(self) -> int | None: - try: - return self.popen.wait() # type: ignore[no-any-return] - except OSError: - return None - - def kill(self) -> None: - try: - self.popen.kill() - except OSError as e: - sys.stderr.write("ERROR killing: %s\n" % e) - sys.stderr.flush() - - -popen_bootstrapline = "import sys;exec(eval(sys.stdin.readline()))" - - -def shell_split_path(path: str) -> list[str]: - """ - Use shell lexer to split the given path into a list of components, - taking care to handle Windows' '\' correctly. - """ - if sys.platform.startswith("win"): - # replace \\ by / otherwise shlex will strip them out - path = path.replace("\\", "/") - return shlex.split(path) - - -def popen_args(spec: XSpec) -> list[str]: - args = shell_split_path(spec.python) if spec.python else [sys.executable] - args.append("-u") - if spec.dont_write_bytecode: - args.append("-B") - args.extend(["-c", popen_bootstrapline]) - return args - - -def ssh_args(spec: XSpec) -> list[str]: - # NOTE: If changing this, you need to sync those changes to vagrant_args - # as well, or, take some time to further refactor the commonalities of - # ssh_args and vagrant_args. - remotepython = spec.python or "python" - args = ["ssh", "-C"] - if spec.ssh_config is not None: - args.extend(["-F", str(spec.ssh_config)]) - - assert spec.ssh is not None - args.extend(spec.ssh.split()) - remotecmd = f'{remotepython} -c "{popen_bootstrapline}"' - args.append(remotecmd) - return args - - -def vagrant_ssh_args(spec: XSpec) -> list[str]: - # This is the vagrant-wrapped version of SSH. Unfortunately the - # command lines are incompatible to just channel through ssh_args - # due to ordering/templating issues. - # NOTE: This should be kept in sync with the ssh_args behaviour. - # spec.vagrant is identical to spec.ssh in that they both carry - # the remote host "address". - assert spec.vagrant_ssh is not None - remotepython = spec.python or "python" - args = ["vagrant", "ssh", spec.vagrant_ssh, "--", "-C"] - if spec.ssh_config is not None: - args.extend(["-F", str(spec.ssh_config)]) - remotecmd = f'{remotepython} -c "{popen_bootstrapline}"' - args.extend([remotecmd]) - return args - - -def create_io(spec: XSpec, execmodel: ExecModel) -> Popen2IOMaster: - if spec.popen: - args = popen_args(spec) - return Popen2IOMaster(args, execmodel) - if spec.ssh: - args = ssh_args(spec) - io = Popen2IOMaster(args, execmodel) - io.remoteaddress = spec.ssh - return io - if spec.vagrant_ssh: - args = vagrant_ssh_args(spec) - io = Popen2IOMaster(args, execmodel) - io.remoteaddress = spec.vagrant_ssh - return io - assert False - - -# -# Proxy Gateway handling code -# -# master: proxy initiator -# forwarder: forwards between master and sub -# sub: sub process that is proxied to the initiator - -RIO_KILL = 1 -RIO_WAIT = 2 -RIO_REMOTEADDRESS = 3 -RIO_CLOSE_WRITE = 4 - - -class ProxyIO: - """A Proxy IO object allows to instantiate a Gateway - through another "via" gateway. - - A master:ProxyIO object provides an IO object effectively connected to the - sub via the forwarder. To achieve this, master:ProxyIO interacts with - forwarder:serve_proxy_io() which itself instantiates and interacts with the - sub. - """ - - def __init__(self, proxy_channel: Channel, execmodel: ExecModel) -> None: - # after exchanging the control channel we use proxy_channel - # for messaging IO - self.controlchan = proxy_channel.gateway.newchannel() - proxy_channel.send(self.controlchan) - self.iochan = proxy_channel - self.iochan_file = self.iochan.makefile("r") - self.execmodel = execmodel - - def read(self, nbytes: int) -> bytes: - # TODO(typing): The IO protocol requires bytes here but ChannelFileRead - # returns str. - return self.iochan_file.read(nbytes) # type: ignore[return-value] - - def write(self, data: bytes) -> None: - self.iochan.send(data) - - def _controll(self, event: int) -> object: - self.controlchan.send(event) - return self.controlchan.receive() - - def close_write(self) -> None: - self._controll(RIO_CLOSE_WRITE) - - def close_read(self) -> None: - raise NotImplementedError() - - def kill(self) -> None: - self._controll(RIO_KILL) - - def wait(self) -> int | None: - response = self._controll(RIO_WAIT) - assert response is None or isinstance(response, int) - return response - - @property - def remoteaddress(self) -> str: - response = self._controll(RIO_REMOTEADDRESS) - assert isinstance(response, str) - return response - - def __repr__(self) -> str: - return f"" - - -class PseudoSpec: - def __init__(self, vars) -> None: - self.__dict__.update(vars) - - def __getattr__(self, name: str) -> None: - return None - - -def serve_proxy_io(proxy_channelX: Channel) -> None: - execmodel = proxy_channelX.gateway.execmodel - log = partial( - proxy_channelX.gateway._trace, "serve_proxy_io:%s" % proxy_channelX.id - ) - spec = cast("XSpec", PseudoSpec(proxy_channelX.receive())) - # create sub IO object which we will proxy back to our proxy initiator - sub_io = create_io(spec, execmodel) - control_chan = cast("Channel", proxy_channelX.receive()) - log("got control chan", control_chan) - - # read data from master, forward it to the sub - # XXX writing might block, thus blocking the receiver thread - def forward_to_sub(data: bytes) -> None: - log("forward data to sub, size %s" % len(data)) - sub_io.write(data) - - proxy_channelX.setcallback(forward_to_sub) - - def control(data: int) -> None: - if data == RIO_WAIT: - control_chan.send(sub_io.wait()) - elif data == RIO_KILL: - sub_io.kill() - control_chan.send(None) - elif data == RIO_REMOTEADDRESS: - control_chan.send(sub_io.remoteaddress) - elif data == RIO_CLOSE_WRITE: - sub_io.close_write() - control_chan.send(None) - - control_chan.setcallback(control) - - # write data to the master coming from the sub - forward_to_master_file = proxy_channelX.makefile("w") - - # read bootstrap byte from sub, send it on to master - log("reading bootstrap byte from sub", spec.id) - initial = sub_io.read(1) - assert initial == b"1", initial - log("forwarding bootstrap byte from sub", spec.id) - forward_to_master_file.write(initial) - - # enter message forwarding loop - while True: - try: - message = Message.from_io(sub_io) - except EOFError: - log("EOF from sub, terminating proxying loop", spec.id) - break - message.to_io(forward_to_master_file) - # proxy_channelX will be closed from remote_exec's finalization code - - -if __name__ == "__channelexec__": - serve_proxy_io(channel) # type: ignore[name-defined] # noqa:F821 diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 29b8bd6a..c4892995 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -20,8 +20,6 @@ from typing import TypeAlias from typing import overload -from . import gateway_bootstrap -from . import gateway_io from .gateway_base import Channel from .gateway_base import ExecModel from .gateway_base import WorkerPool @@ -154,26 +152,16 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: spec.execmodel = self.remote_execmodel.backend from . import _trio_host - if _trio_host.should_use_trio_popen(spec): - gw = _trio_host.makegateway_popen_trio(self, spec) - elif _trio_host.should_use_trio_ssh(spec): - gw = _trio_host.makegateway_ssh_trio(self, spec) - elif _trio_host.should_use_trio_vagrant(spec): - gw = _trio_host.makegateway_vagrant_trio(self, spec) - elif _trio_host.should_use_trio_socket(spec): + if spec.socket: gw = _trio_host.makegateway_socket_trio(self, spec) - elif _trio_host.should_use_trio_via(spec): - gw = _trio_host.makegateway_via_trio(self, spec) elif spec.via: - assert not spec.socket - master = self[spec.via] - proxy_channel = master.remote_exec(gateway_io) - proxy_channel.send(vars(spec)) - proxy_io_master = gateway_io.ProxyIO(proxy_channel, self.execmodel) - gw = gateway_bootstrap.bootstrap(proxy_io_master, spec) - elif spec.popen or spec.ssh or spec.vagrant_ssh: - io = gateway_io.create_io(spec, execmodel=self.execmodel) - gw = gateway_bootstrap.bootstrap(io, spec) + gw = _trio_host.makegateway_via_trio(self, spec) + elif spec.ssh: + gw = _trio_host.makegateway_ssh_trio(self, spec) + elif spec.vagrant_ssh: + gw = _trio_host.makegateway_vagrant_trio(self, spec) + elif spec.popen: + gw = _trio_host.makegateway_popen_trio(self, spec) else: raise ValueError(f"no gateway type found for {spec._spec!r}") gw.spec = spec diff --git a/testing/test_basics.py b/testing/test_basics.py index 1756ec34..d4ffd5d8 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -17,11 +17,9 @@ import execnet from execnet import gateway from execnet import gateway_base -from execnet import gateway_io from execnet.gateway_base import ChannelFactory from execnet.gateway_base import ExecModel from execnet.gateway_base import Message -from execnet.gateway_base import Popen2IO skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), @@ -68,81 +66,29 @@ def test_errors_on_execnet() -> None: assert hasattr(execnet, "DataFormatError") -def test_subprocess_interaction(anypython: str) -> None: - line = gateway_io.popen_bootstrapline - compile(line, "xyz", "exec") - args = [str(anypython), "-c", line] - popen = subprocess.Popen( - args, - bufsize=0, - universal_newlines=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - - assert popen.stdin is not None - assert popen.stdout is not None - - def send(line: str) -> None: - assert popen.stdin is not None - popen.stdin.write(line) - popen.stdin.flush() - - def receive() -> str: - assert popen.stdout is not None - return popen.stdout.readline() - - try: - source = inspect.getsource(read_write_loop) + "read_write_loop()" - send(repr(source) + "\n") - s = receive() - assert s == "ok\n" - send("hello\n") - s = receive() - assert s == "received: hello\n" - send("world\n") - s = receive() - assert s == "received: world\n" - send("\n") # terminate loop - finally: - popen.stdin.close() - popen.stdout.close() - popen.wait() +IO_MESSAGE_EXTRA_SOURCE = """ +from io import BytesIO +class BufIO: + def __init__(self): + self.buf = BytesIO() -def read_write_loop() -> None: - sys.stdout.write("ok\n") - sys.stdout.flush() - while 1: - try: - line = sys.stdin.readline() - if not line.strip(): - break - sys.stdout.write("received: %s" % line) - sys.stdout.flush() - except (OSError, EOFError): - break + def write(self, data): + self.buf.write(data) + def read(self, numbytes): + data = self.buf.read(numbytes) + if len(data) < numbytes: + raise EOFError("expected %d bytes" % numbytes) + return data -IO_MESSAGE_EXTRA_SOURCE = """ -import sys -backend = sys.argv[1] -from io import BytesIO -import tempfile -temp_out = BytesIO() -temp_in = BytesIO() -io = Popen2IO(temp_out, temp_in, get_execmodel(backend)) for i, handler in enumerate(Message._types): print ("checking", i, handler) for data in "hello", "hello".encode('ascii'): + io = BufIO() msg1 = Message(i, i, dumps(data)) msg1.to_io(io) - x = io.outfile.getvalue() - io.outfile.truncate(0) - io.outfile.seek(0) - io.infile.seek(0) - io.infile.write(x) - io.infile.seek(0) + io.buf.seek(0) msg2 = Message.from_io(io) assert msg1.channelid == msg2.channelid, (msg1, msg2) assert msg1.data == msg2.data, (msg1.data, msg2.data) @@ -177,44 +123,12 @@ def checker(anypython: str, tmp_path: Path) -> Checker: return Checker(python=anypython, path=tmp_path) -def test_io_message(checker: Checker, execmodel: ExecModel) -> None: - out = checker.run_check( - inspect.getsource(gateway_base) + IO_MESSAGE_EXTRA_SOURCE, execmodel.backend - ) +def test_io_message(checker: Checker) -> None: + out = checker.run_check(inspect.getsource(gateway_base) + IO_MESSAGE_EXTRA_SOURCE) print(out.stdout) assert "all passed" in out.stdout -def test_popen_io(checker: Checker, execmodel: ExecModel) -> None: - out = checker.run_check( - inspect.getsource(gateway_base) - + f""" -io = init_popen_io(get_execmodel({execmodel.backend!r})) -io.write(b"hello") -s = io.read(1) -assert s == b"x" -""", - input="x", - ) - print(out.stderr) - assert "hello" in out.stdout - - -def test_popen_io_readloop(execmodel: ExecModel) -> None: - sio = BytesIO(b"test") - io = Popen2IO(sio, sio, execmodel) - real_read = io._read - - def newread(numbytes: int) -> bytes: - if numbytes > 1: - numbytes = numbytes - 1 - return real_read(numbytes) # type: ignore[no-any-return] - - io._read = newread - result = io.read(3) - assert result == b"tes" - - def test_rinfo_source(checker: Checker) -> None: out = checker.run_check( f""" @@ -252,19 +166,20 @@ class Arg(Exception): @pytest.mark.skipif("not hasattr(os, 'dup')") -def test_stdouterrin_setnull( - execmodel: ExecModel, capfd: pytest.CaptureFixture[str] -) -> None: - # Backup and restore stdin state, and rely on capfd to handle - # this for stdout and stderr. +def test_stdouterrin_setnull(capfd: pytest.CaptureFixture[str]) -> None: + # _prepare_protocol_fds dups the stdio fds for the Message protocol and + # points fd 0/1 at devnull; writes/reads on the original fds must go + # nowhere. Back up and restore the real fds around the call. + from execnet import _trio_worker + orig_stdin = sys.stdin - orig_stdin_fd = os.dup(0) + orig_stdout = sys.stdout + orig_fd0 = os.dup(0) + orig_fd1 = os.dup(1) try: - # The returned Popen2IO instance can be garbage collected - # prematurely since we don't hold a reference here, but we - # tolerate this because it is intended to leave behind a - # sane state afterwards. - gateway_base.init_popen_io(execmodel) + read_fd, write_fd = _trio_worker._prepare_protocol_fds() + os.close(read_fd) + os.close(write_fd) os.write(1, b"hello") os.read(0, 1) out, err = capfd.readouterr() @@ -272,8 +187,11 @@ def test_stdouterrin_setnull( assert not err finally: sys.stdin = orig_stdin - os.dup2(orig_stdin_fd, 0) - os.close(orig_stdin_fd) + sys.stdout = orig_stdout + os.dup2(orig_fd0, 0) + os.dup2(orig_fd1, 1) + os.close(orig_fd0) + os.close(orig_fd1) class PseudoChannel: diff --git a/testing/test_gateway.py b/testing/test_gateway.py index e123bbe2..c3c87e4a 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -16,7 +16,6 @@ import execnet from execnet import gateway_base -from execnet import gateway_io from execnet.gateway import Gateway TESTTIMEOUT = 10.0 # seconds @@ -381,23 +380,6 @@ def test_socket_gw_host_not_found(makegateway: Callable[[str], Gateway]) -> None class TestSshPopenGateway: gwtype = "ssh" - def test_sshconfig_config_parsing( - self, monkeypatch: pytest.MonkeyPatch, makegateway: Callable[[str], Gateway] - ) -> None: - # white-box test of the legacy Popen2IOMaster arg construction - monkeypatch.setenv("EXECNET_TRIO_HOST", "0") - l = [] - monkeypatch.setattr( - gateway_io, "Popen2IOMaster", lambda *args, **kwargs: l.append(args[0]) - ) - with pytest.raises(AttributeError): - makegateway("ssh=xyz//ssh_config=qwe") - - assert len(l) == 1 - popen_args = l[0] - i = popen_args.index("-F") - assert popen_args[i + 1] == "qwe" - def test_ssh_trio_args_include_config( self, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -549,9 +531,11 @@ def test_no_tracing_by_default(self): ], ) def test_popen_args(spec: str, expected_args: list[str]) -> None: - expected_args = [*expected_args, "-u", "-c", gateway_io.popen_bootstrapline] - args = gateway_io.popen_args(execnet.XSpec(spec)) - assert args == expected_args + from execnet import _trio_host + + args = _trio_host.popen_module_args(execnet.XSpec(spec + "//id=gw0")) + assert args[: len(expected_args)] == expected_args + assert args[len(expected_args) :][:3] == ["-u", "-m", "execnet._trio_worker"] def test_assert_main_thread_only( diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py index c21135d1..086ce877 100644 --- a/testing/test_ssh_local.py +++ b/testing/test_ssh_local.py @@ -133,12 +133,8 @@ def ssh_config(ssh_server: SSHServerThread, tmp_path) -> str: return path -@pytest.mark.parametrize("trio_host", ["1", "0"], ids=["trio", "legacy"]) -def test_ssh_roundtrip( - ssh_config: str, monkeypatch: pytest.MonkeyPatch, trio_host: str -) -> None: - # trio=1 provisions the worker over ssh with uv; legacy=0 source-copies it. - monkeypatch.setenv("EXECNET_TRIO_HOST", trio_host) +def test_ssh_roundtrip(ssh_config: str) -> None: + # The worker is provisioned over ssh with uv. group = execnet.Group() try: gw = group.makegateway( diff --git a/testing/test_xspec.py b/testing/test_xspec.py index 5240d72d..f9681f55 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -11,10 +11,8 @@ import execnet from execnet import XSpec +from execnet import _provision from execnet.gateway import Gateway -from execnet.gateway_io import popen_args -from execnet.gateway_io import ssh_args -from execnet.gateway_io import vagrant_ssh_args skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), @@ -69,22 +67,20 @@ def test_execmodel(self) -> None: def test_ssh_options_and_config(self) -> None: spec = XSpec("ssh=-p 22100 user@host//python=python3") - spec.ssh_config = "/home/user/ssh_config" - assert ssh_args(spec)[:6] == ["ssh", "-C", "-F", spec.ssh_config, "-p", "22100"] + args = _provision.ssh_argv("-p 22100 user@host", "/home/user/ssh_config", "cmd") + assert args[:6] == ["ssh", "-C", "-F", "/home/user/ssh_config", "-p", "22100"] + assert spec.ssh is not None def test_vagrant_options(self) -> None: - spec = XSpec("vagrant_ssh=default//python=python3") - assert vagrant_ssh_args(spec)[:-1] == ["vagrant", "ssh", "default", "--", "-C"] + args = _provision.vagrant_ssh_argv("default", None, "cmd") + assert args[:-1] == ["vagrant", "ssh", "default", "--", "-C"] def test_popen_with_sudo_python(self) -> None: - spec = XSpec("popen//python=sudo python3") - assert popen_args(spec) == [ - "sudo", - "python3", - "-u", - "-c", - "import sys;exec(eval(sys.stdin.readline()))", - ] + from execnet import _trio_host + + spec = XSpec("popen//python=sudo python3//id=gw0") + args = _trio_host.popen_module_args(spec) + assert args[:5] == ["sudo", "python3", "-u", "-m", "execnet._trio_worker"] def test_env(self) -> None: xspec = XSpec("popen//env:NAME=value1") From 0a24039959a02ea96679fe863bc9b9437d0d7abf Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 11:11:22 +0200 Subject: [PATCH 18/91] refactor: unify transports on StapledStream with sans-IO frame decoding Phase B.1+B.2 of the Trio port: - add gateway_base.FrameDecoder, an incremental sans-IO decoder for the 9-byte-header Message framing (feed arbitrary chunks, complete messages come out; close() flags mid-frame EOF) - replace the hand-rolled AsyncByteIO wrappers (ProcessStreamsIO, FdStreamsIO, SocketStreamIO) with trio.StapledStream / plain trio.SocketStream behind a neutral ByteStream protocol (send_all/receive_some/send_eof/aclose) that a future anyio backend can satisfy structurally; the via tunnel keeps its interim bridge as ChannelByteStream - ProtocolSession's reader becomes the uniform receive_some+feed loop; exact reads survive only as the one-byte handshake ack - route worker exec requests through a single FIFO pump task: batched frame decoding removed the per-message awaits that had accidentally serialized main_thread_only exec admission, so admission now happens explicitly in message-arrival order instead of racing tasks - give pre-commit's mypy the trio dependency so trio types are real; drop now-redundant casts Co-Authored-By: Claude Fable 5 --- .pre-commit-config.yaml | 1 + src/execnet/_trio_host.py | 211 +++++++++++++----------------------- src/execnet/_trio_worker.py | 45 ++++---- src/execnet/gateway_base.py | 36 ++++++ testing/test_basics.py | 84 ++++++++++++++ 5 files changed, 223 insertions(+), 154 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 37ccd44d..f7a9f86c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,4 +31,5 @@ repos: - id: mypy additional_dependencies: - pytest + - trio - types-pywin32 diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 59f6b47f..1a8dd197 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -21,11 +21,11 @@ from typing import Any from typing import Protocol from typing import TypeVar -from typing import cast import trio from .gateway_base import ExecModel +from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate from .gateway_base import Message from .gateway_base import dumps_internal @@ -41,144 +41,95 @@ _CLOSE_WRITE = object() -class AsyncByteIO(Protocol): - async def read_exact(self, n: int) -> bytes: ... +RECEIVE_CHUNK = 65536 - async def write_all(self, data: bytes) -> None: ... - async def aclose_read(self) -> None: ... +class ByteStream(Protocol): + """Neutral bidirectional byte-stream protocol for gateway transports. - async def aclose_write(self) -> None: ... - - -async def read_exact_receive_stream(stream: trio.abc.ReceiveStream, n: int) -> bytes: - buf = bytearray() - while len(buf) < n: - chunk = await stream.receive_some(n - len(buf)) - if not chunk: - raise EOFError("expected %d bytes, got %d" % (n, len(buf))) - buf += chunk - return bytes(buf) - - -async def read_message(io: AsyncByteIO) -> Message: - header = await io.read_exact(9) - msgtype, channel, payload = Message.from_header(header) - data = await io.read_exact(payload) if payload else b"" - return Message.from_parts(msgtype, channel, data) - - -class ProcessStreamsIO: - """Async IO over a Trio Process stdin/stdout pair.""" - - def __init__(self, process: trio.Process) -> None: - assert process.stdin is not None - assert process.stdout is not None - self.process = process - self._stdin = process.stdin - self._stdout = process.stdout - - async def read_exact(self, n: int) -> bytes: - return await read_exact_receive_stream(self._stdout, n) - - async def write_all(self, data: bytes) -> None: - await self._stdin.send_all(data) - - async def aclose_read(self) -> None: - await self._stdout.aclose() - - async def aclose_write(self) -> None: - await self._stdin.aclose() - - -class FdStreamsIO: - """Async IO over OS file descriptors (worker stdio pipes).""" - - def __init__(self, read_fd: int, write_fd: int) -> None: - self._read = trio.lowlevel.FdStream(read_fd) - self._write = trio.lowlevel.FdStream(write_fd) - - async def read_exact(self, n: int) -> bytes: - return await read_exact_receive_stream(self._read, n) - - async def write_all(self, data: bytes) -> None: - await self._write.send_all(data) + ``trio.StapledStream`` (process/fd pipe pairs) and ``trio.SocketStream`` + satisfy this structurally; a future anyio backend's byte streams use the + same four names. ``send_eof`` signals write-EOF to the peer (half-close + for sockets; for pipe pairs trio falls back to closing the send half). + """ - async def aclose_read(self) -> None: - await self._read.aclose() + async def send_all(self, data: bytes) -> None: ... - async def aclose_write(self) -> None: - await self._write.aclose() + async def receive_some(self, max_bytes: int | None = None) -> bytes: ... + async def send_eof(self) -> None: ... -class SocketStreamIO: - """Async IO over a single bidirectional Trio stream (a socket). + async def aclose(self) -> None: ... - ``read_exact`` never over-reads (``receive_some(k)`` returns at most ``k`` - bytes), so no cross-call buffering is needed. ``aclose`` on a Trio stream is - idempotent, so close-read and close-write both just close the socket. - """ - def __init__(self, stream: trio.abc.Stream) -> None: - self._stream = stream +def staple_process_stream(process: trio.Process) -> ByteStream: + """One bidirectional stream over a Trio Process stdin/stdout pair.""" + assert process.stdin is not None + assert process.stdout is not None + return trio.StapledStream(process.stdin, process.stdout) - async def read_exact(self, n: int) -> bytes: - return await read_exact_receive_stream(self._stream, n) - async def write_all(self, data: bytes) -> None: - await self._stream.send_all(data) +def staple_fd_stream(read_fd: int, write_fd: int) -> ByteStream: + """One bidirectional stream over OS pipe fds (worker stdio pipes).""" + return trio.StapledStream( + trio.lowlevel.FdStream(write_fd), trio.lowlevel.FdStream(read_fd) + ) - async def aclose_read(self) -> None: - await self._stream.aclose() - async def aclose_write(self) -> None: - await self._stream.aclose() +async def read_handshake_ack(stream: ByteStream, what: str) -> None: + """Wait for the worker's single ``b"1"`` ready byte.""" + ack = await stream.receive_some(1) + if ack != b"1": + raise EOFError(f"bad {what} handshake: {ack!r}") _CHANNEL_EOF = object() -class ChannelByteIO: - """Async byte IO tunnelled over a sync execnet ``Channel``. +class ChannelByteStream: + """``ByteStream`` tunnelled over a sync execnet ``Channel``. INTERIM HACK: the ``via`` transport currently runs the sub-gateway protocol as raw bytes over a channel to the master, which double-frames it (sub frame -> CHANNEL_DATA -> master frame). This bridges the sync ``Channel`` to the - async protocol; it should be replaced by a proper relayed transport rather - than tunnelling bytes through the channel layer. + async protocol; it dissolves once low-level raw channels exist (Phase B.4). The channel callback (on the host loop) feeds an unbounded memory channel - that ``read_exact`` drains; writes ``channel.send`` raw frames. + that ``receive_some`` drains; writes ``channel.send`` raw bytes. """ def __init__(self, channel: Any) -> None: self._channel = channel self._send, self._recv = trio.open_memory_channel[Any](math.inf) self._buf = bytearray() + self._eof = False channel.setcallback(self._send.send_nowait, endmarker=_CHANNEL_EOF) - async def read_exact(self, n: int) -> bytes: - while len(self._buf) < n: + async def send_all(self, data: bytes) -> None: + self._channel.send(data) + + async def receive_some(self, max_bytes: int | None = None) -> bytes: + if not self._buf and not self._eof: data = await self._recv.receive() if data is _CHANNEL_EOF: - raise EOFError("channel closed") - assert isinstance(data, bytes) - self._buf += data - out = bytes(self._buf[:n]) - del self._buf[:n] + self._eof = True + else: + assert isinstance(data, bytes) + self._buf += data + if max_bytes is None: + max_bytes = len(self._buf) + out = bytes(self._buf[:max_bytes]) + del self._buf[:max_bytes] return out - async def write_all(self, data: bytes) -> None: - self._channel.send(data) - - async def aclose_read(self) -> None: - return + async def send_eof(self) -> None: + self._channel.close() - async def aclose_write(self) -> None: + async def aclose(self) -> None: self._channel.close() -async def adopt_socket(socket_fd: int) -> SocketStreamIO: +async def adopt_socket(socket_fd: int) -> trio.SocketStream: """Worker side: wrap an inherited socket fd and send the handshake. Runs on the Trio host loop. The coordinator waits for ``b"1"`` before @@ -188,9 +139,8 @@ async def adopt_socket(socket_fd: int) -> SocketStreamIO: sock = _socket.socket(fileno=socket_fd) stream = trio.SocketStream(trio.socket.from_stdlib_socket(sock)) - io = SocketStreamIO(stream) - await io.write_all(b"1") - return io + await stream.send_all(b"1") + return stream class SyncIOHandle: @@ -235,7 +185,7 @@ class ProtocolSession: def __init__( self, gateway: BaseGateway, - io: AsyncByteIO, + io: ByteStream, *, process: trio.Process | None = None, host: TrioHost, @@ -361,13 +311,17 @@ def log(*msg: object) -> None: gateway._trace("[trio-receiver]", *msg) log("RECEIVER: starting") + decoder = FrameDecoder() try: while True: - msg = await read_message(self.io) - log("received", msg) - with gateway._receivelock: - msg.received(gateway) - del msg + data = await self.io.receive_some(RECEIVE_CHUNK) + if not data: + decoder.close() # raises EOFError on a mid-frame EOF + raise EOFError("clean EOF") + for msg in decoder.feed(data): + log("received", msg) + with gateway._receivelock: + msg.received(gateway) except GatewayReceivedTerminate: log("GATEWAY_TERMINATE") except EOFError as exc: @@ -401,15 +355,15 @@ async def _writer_handle_item(self, item: object) -> bool: """Handle one outbound queue item. Return False when writer should stop.""" if item is _CLOSE_WRITE: try: - await self.io.aclose_write() + await self.io.send_eof() except Exception as exc: - self.gateway._trace("aclose_write failed", exc) + self.gateway._trace("send_eof failed", exc) return False assert isinstance(item, tuple) blob, done, errors = item assert isinstance(blob, bytes) try: - await self.io.write_all(blob) + await self.io.send_all(blob) except Exception as exc: self.gateway._trace("write failed", exc) errors.append(exc) @@ -445,11 +399,7 @@ async def _finish(self) -> None: gateway._terminate_execution, abandon_on_cancel=True ) try: - await self.io.aclose_read() - except Exception: - pass - try: - await self.io.aclose_write() + await self.io.aclose() except Exception: pass @@ -496,14 +446,12 @@ async def _main(self) -> None: def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: if self._token is None: raise RuntimeError("TrioHost is not running") - return cast("T", trio.from_thread.run(async_fn, *args, trio_token=self._token)) + return trio.from_thread.run(async_fn, *args, trio_token=self._token) def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: if self._token is None: raise RuntimeError("TrioHost is not running") - return cast( - "T", trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) - ) + return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: """Schedule a task on the root nursery (must be called on the host thread).""" @@ -516,7 +464,7 @@ def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: async def start_session( self, gateway: BaseGateway, - io: AsyncByteIO, + io: ByteStream, *, process: trio.Process | None = None, ) -> ProtocolSession: @@ -621,12 +569,10 @@ def _open_trio_gateway( async def _create_and_attach() -> Gateway: process = await open_popen_process(args) try: - async_io = ProcessStreamsIO(process) + async_io = staple_process_stream(process) if preamble: - await async_io.write_all(preamble) - ack = await async_io.read_exact(1) - if ack != b"1": - raise EOFError(f"bad bootstrap handshake: {ack!r}") + await async_io.send_all(preamble) + await read_handshake_ack(async_io, "bootstrap") except EOFError: with trio.move_on_after(5): code = await process.wait() @@ -732,18 +678,15 @@ async def _create_and_attach() -> Gateway: stream = await trio.open_tcp_stream(*address) except OSError as exc: raise HostNotFound(remoteaddress) from exc - io = SocketStreamIO(stream) try: - ack = await io.read_exact(1) - if ack != b"1": - raise EOFError(f"bad socket handshake: {ack!r}") + await read_handshake_ack(stream, "socket") except BaseException: with trio.move_on_after(5): await stream.aclose() raise gw = execnet.Gateway(_TempIO(group.execmodel), spec) - session = await host.start_session(gw, io) + session = await host.start_session(gw, stream) gw._attach_trio_session(session) gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) return gw @@ -910,10 +853,8 @@ def makegateway_via_trio(group: Any, spec: Any) -> Gateway: remoteaddress = f"{remote}[via {spec.via}]" if remote else None async def _create_and_attach() -> Gateway: - io = ChannelByteIO(channel) - ack = await io.read_exact(1) - if ack != b"1": - raise EOFError(f"bad via handshake: {ack!r}") + io = ChannelByteStream(channel) + await read_handshake_ack(io, "via") gw = execnet.Gateway(_TempIO(group.execmodel), spec) session = await host.start_session(gw, io) gw._attach_trio_session(session) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 9e1f1be7..5fc9bceb 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -2,6 +2,7 @@ from __future__ import annotations +import functools import os import queue import sys @@ -51,9 +52,14 @@ def __init__( tuple[Channel, ExecItem, threading.Event] | None ] = queue.SimpleQueue() self._primary_wake = threading.Event() - # Serialize main_thread_only admission (wait+clear) so two tasks cannot - # both observe the idle Event before either clears it. - self._admit_lock = trio.Lock() + # Exec requests flow through a single pump task so admission happens + # strictly in message-arrival order (trio task scheduling order is + # deliberately unordered, so per-request tasks would race for the + # main_thread_only slot). + self._pending_send: trio.MemorySendChannel[tuple[Channel, ExecItem]] + self._pending_recv: trio.MemoryReceiveChannel[tuple[Channel, ExecItem]] + self._pending_send, self._pending_recv = trio.open_memory_channel(float("inf")) + self._pump_started = False def active_count(self) -> int: with self._lock: @@ -82,24 +88,25 @@ def schedule(self, channel: Channel, sourcetask: bytes) -> None: channel.close("execution disallowed") return # Already on the Trio host thread (Message handler). - self.host.start_soon(self._run_exec, channel, item) - - async def _run_exec(self, channel: Channel, item: ExecItem) -> None: - if self.main_thread_only: - complete = self.gateway._executetask_complete - assert complete is not None - - def _wait_slot() -> bool: - return complete.wait(timeout=1) - - async with self._admit_lock: - if not await trio.to_thread.run_sync( - _wait_slot, abandon_on_cancel=True - ): + if not self._pump_started: + self._pump_started = True + self.host.start_soon(self._pump) + self._pending_send.send_nowait((channel, item)) + + async def _pump(self) -> None: + """Admit queued exec requests in FIFO order, then run each as a task.""" + async for channel, item in self._pending_recv: + if self.main_thread_only: + complete = self.gateway._executetask_complete + assert complete is not None + wait_slot = functools.partial(complete.wait, timeout=1) + if not await trio.to_thread.run_sync(wait_slot, abandon_on_cancel=True): channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) - return + continue complete.clear() + self.host.start_soon(self._run_exec, channel, item) + async def _run_exec(self, channel: Channel, item: ExecItem) -> None: self._track_start() try: if self.main_thread_only: @@ -252,7 +259,7 @@ async def _start() -> _trio_host.ProtocolSession: async def _make_fd_io(read_fd: int, write_fd: int) -> Any: from . import _trio_host - return _trio_host.FdStreamsIO(read_fd, write_fd) + return _trio_host.staple_fd_stream(read_fd, write_fd) def serve_popen_trio(id: str, execmodel: str = "thread") -> None: diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 0260b416..d36c7542 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -529,6 +529,42 @@ def _gateway_start_sub(message: Message, gateway: BaseGateway) -> None: _types[GATEWAY_START_SUB] = ("GATEWAY_START_SUB", _gateway_start_sub) +class FrameDecoder: + """Incremental decoder for the 9-byte-header Message framing. + + ``feed(data)`` accepts arbitrary byte chunks and yields every complete + Message; partial frames buffer internally until more bytes arrive. + Pure computation — no IO, no awaits, no knowledge of streams — so + receivers only ever stream bytes in (``receive_some`` loops) and the + decoder owns framing. + """ + + def __init__(self) -> None: + self._buffer = bytearray() + + def feed(self, data: bytes) -> Iterator[Message]: + self._buffer += data + return self._parse() + + def _parse(self) -> Iterator[Message]: + while len(self._buffer) >= 9: + msgtype, channelid, payload_len = Message.from_header( + bytes(self._buffer[:9]) + ) + if len(self._buffer) < 9 + payload_len: + return + payload = bytes(self._buffer[9 : 9 + payload_len]) + del self._buffer[: 9 + payload_len] + yield Message(msgtype, channelid, payload) + + def close(self) -> None: + """Signal EOF; raises EOFError if the stream ended mid-frame.""" + if self._buffer: + raise EOFError( + "connection closed mid-frame (%d buffered bytes)" % len(self._buffer) + ) + + class GatewayReceivedTerminate(Exception): """Receiver got a gateway termination message.""" diff --git a/testing/test_basics.py b/testing/test_basics.py index d4ffd5d8..473cd741 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -236,6 +236,90 @@ def test_wire_protocol(self) -> None: assert isinstance(repr(msg), str) +class TestFrameDecoder: + def _messages(self) -> list[Message]: + return [ + Message(Message.CHANNEL_DATA, 1, b"x" * 20), + Message(Message.STATUS, 42, b""), + Message(Message.CHANNEL_DATA, 7, b"y"), + ] + + def test_single_feed_yields_all(self) -> None: + decoder = gateway_base.FrameDecoder() + blob = b"".join(m.pack() for m in self._messages()) + got = list(decoder.feed(blob)) + assert [(m.msgcode, m.channelid, m.data) for m in got] == [ + (m.msgcode, m.channelid, m.data) for m in self._messages() + ] + decoder.close() + + @pytest.mark.parametrize("chunksize", [1, 2, 3, 8, 9, 10, 13]) + def test_adversarial_chunk_splits(self, chunksize: int) -> None: + decoder = gateway_base.FrameDecoder() + blob = b"".join(m.pack() for m in self._messages()) + got: list[Message] = [] + for start in range(0, len(blob), chunksize): + got.extend(decoder.feed(blob[start : start + chunksize])) + assert [(m.msgcode, m.channelid, m.data) for m in got] == [ + (m.msgcode, m.channelid, m.data) for m in self._messages() + ] + decoder.close() + + def test_close_mid_frame_raises(self) -> None: + decoder = gateway_base.FrameDecoder() + blob = Message(Message.CHANNEL_DATA, 1, b"hello").pack() + assert list(decoder.feed(blob[:-2])) == [] + with pytest.raises(EOFError, match="mid-frame"): + decoder.close() + + def test_feed_buffers_even_when_not_iterated(self) -> None: + decoder = gateway_base.FrameDecoder() + blob = Message(Message.CHANNEL_DATA, 5, b"data").pack() + decoder.feed(blob[:4]) # result deliberately not iterated + (msg,) = decoder.feed(blob[4:]) + assert (msg.msgcode, msg.channelid, msg.data) == ( + Message.CHANNEL_DATA, + 5, + b"data", + ) + + def test_memory_stream_roundtrip(self) -> None: + """Protocol-level: frames sent over a trio memory stream pair arrive + intact through the receive_some + FrameDecoder loop.""" + import trio + import trio.testing + + messages = self._messages() + + async def main() -> list[Message]: + ours, theirs = trio.testing.memory_stream_pair() + received: list[Message] = [] + + async def sender() -> None: + for m in messages: + await theirs.send_all(m.pack()) + await theirs.send_eof() + + async def receiver() -> None: + decoder = gateway_base.FrameDecoder() + while True: + data = await ours.receive_some(4096) + if not data: + decoder.close() + break + received.extend(decoder.feed(data)) + + async with trio.open_nursery() as nursery: + nursery.start_soon(sender) + nursery.start_soon(receiver) + return received + + received = trio.run(main) + assert [(m.msgcode, m.channelid, m.data) for m in received] == [ + (m.msgcode, m.channelid, m.data) for m in messages + ] + + class TestPureChannel: @pytest.fixture def fac(self, execmodel: ExecModel) -> ChannelFactory: From f271da5e379dd23392b39ddcdcc0792bc6fc43c1 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 12:19:03 +0200 Subject: [PATCH 19/91] refactor: extract the LoopPortal cross-thread primitive Phase B.3 of the Trio port: one portal module replaces the three ad-hoc cross-thread bridges. - new execnet/portal.py: LoopPortal (trio token holder with run/run_sync/post/is_loop_thread; post = run_sync_soon, strict FIFO) and SyncReceiver (loop-to-thread queue whose get() stays KeyboardInterrupt-interruptible on the main thread) - TrioHost owns a LoopPortal; call/call_sync/is_host_thread delegate - ProtocolSession's outbound queue becomes an unbounded trio memory channel; every send is posted through the portal so loop callbacks and foreign threads share one FIFO, and the writer task is a plain async-for -- the recreated-Event wake dance is gone. Blocking and close semantics are unchanged: non-loop threads still wait for the OS write (120s timeout), closed sends still raise OSError, and a finished loop maps trio.RunFinishedError to the same OSError. - TrioWorkerExec's main-thread exec handoff uses SyncReceiver Because each end only needs the other loop's token, the same primitive will serve two-loop setups (facade host loop + user loop) in B.5/B.6. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_host.py | 85 ++++++++++++++------------------ src/execnet/_trio_worker.py | 23 +++------ src/execnet/portal.py | 96 +++++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 66 deletions(-) create mode 100644 src/execnet/portal.py diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 1a8dd197..cd20a0f2 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -10,7 +10,6 @@ import itertools import json import math -import queue import subprocess import sys import threading @@ -31,6 +30,7 @@ from .gateway_base import dumps_internal from .gateway_base import loads_internal from .gateway_base import trace +from .portal import LoopPortal if TYPE_CHECKING: from .gateway import Gateway @@ -194,14 +194,24 @@ def __init__( self.io = io self.process = process self.host = host - self._outbound: queue.SimpleQueue[object] = queue.SimpleQueue() - self._wake: trio.Event | None = None + self._outbound_send: trio.MemorySendChannel[object] + self._outbound_recv: trio.MemoryReceiveChannel[object] + self._outbound_send, self._outbound_recv = trio.open_memory_channel(math.inf) self._done = threading.Event() self._process_exitcode: int | None = None self._process_done = threading.Event() self._send_closed = False self._lock = threading.Lock() + def _post_outbound(self, item: object) -> None: + """Schedule ``item`` onto the outbound channel. + + Goes through the portal even from the host thread so every send — + loop callbacks and foreign threads alike — lands in one global FIFO + order. Raises ``trio.RunFinishedError`` after loop shutdown. + """ + self.host.portal.post(self._outbound_send.send_nowait, item) + def enqueue_message(self, message: Message) -> None: """Enqueue a frame; wait until written when safe to block. @@ -211,14 +221,16 @@ def enqueue_message(self, message: Message) -> None: The Trio host thread (receiver callbacks) must not wait — that would deadlock the writer task on the same event loop. """ - wait = not self.host.is_host_thread() + wait = not self.host.portal.is_loop_thread() done = threading.Event() if wait else None errors: list[BaseException] = [] with self._lock: if self._send_closed or self._done.is_set(): raise OSError("cannot send (already closed?)") - self._outbound.put((message.pack(), done, errors)) - self._wake_writer() + try: + self._post_outbound((message.pack(), done, errors)) + except trio.RunFinishedError: + raise OSError("cannot send (already closed?)") from None if done is None: return if not done.wait(timeout=120.0): @@ -226,25 +238,13 @@ def enqueue_message(self, message: Message) -> None: if errors: raise OSError("cannot send (already closed?)") from errors[0] - def _wake_writer(self) -> None: - wake = self._wake - if wake is None: - return - if self.host.is_host_thread(): - wake.set() - else: - try: - self.host.call_sync(wake.set) - except Exception: - pass - def request_close_write(self) -> None: with self._lock: if self._send_closed: return self._send_closed = True - self._outbound.put(_CLOSE_WRITE) - self._wake_writer() + with suppress(trio.RunFinishedError): + self._post_outbound(_CLOSE_WRITE) def request_close_read(self) -> None: # Reader observes EOF / cancel; nothing required from callers. @@ -332,22 +332,7 @@ def log(*msg: object) -> None: log("finishing receiver") async def _writer(self) -> None: - self._wake = trio.Event() - while True: - while True: - try: - item = self._outbound.get_nowait() - except queue.Empty: - break - if not await self._writer_handle_item(item): - return - # Reset wake before re-check to avoid losing a notification. - self._wake = trio.Event() - try: - item = self._outbound.get_nowait() - except queue.Empty: - await self._wake.wait() - continue + async for item in self._outbound_recv: if not await self._writer_handle_item(item): return @@ -385,8 +370,8 @@ async def _finish(self) -> None: gateway = self.gateway with self._lock: self._send_closed = True - self._outbound.put(_CLOSE_WRITE) - self._wake_writer() + with suppress(trio.RunFinishedError): + self._post_outbound(_CLOSE_WRITE) gateway._trace("[trio-receiver] finishing channels") gateway._channelfactory._finished_receiving() # Unblock the worker's join() before heavy exec-pool shutdown @@ -410,7 +395,7 @@ class TrioHost: def __init__(self, name: str = "execnet-trio-host") -> None: self._name = name self._thread: threading.Thread | None = None - self._token: trio.lowlevel.TrioToken | None = None + self._portal: LoopPortal | None = None self._nursery: trio.Nursery | None = None self._ready = threading.Event() self._shutdown: trio.Event | None = None @@ -425,14 +410,20 @@ def start(self) -> None: raise RuntimeError("TrioHost failed to start") self._started = True + @property + def portal(self) -> LoopPortal: + if self._portal is None: + raise RuntimeError("TrioHost is not running") + return self._portal + def is_host_thread(self) -> bool: - return self._thread is not None and threading.current_thread() is self._thread + return self._portal is not None and self._portal.is_loop_thread() def _run(self) -> None: trio.run(self._main) async def _main(self) -> None: - self._token = trio.lowlevel.current_trio_token() + self._portal = LoopPortal() self._shutdown = trio.Event() try: async with trio.open_nursery() as nursery: @@ -444,14 +435,10 @@ async def _main(self) -> None: self._nursery = None def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: - if self._token is None: - raise RuntimeError("TrioHost is not running") - return trio.from_thread.run(async_fn, *args, trio_token=self._token) + return self.portal.run(async_fn, *args) def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: - if self._token is None: - raise RuntimeError("TrioHost is not running") - return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + return self.portal.run_sync(sync_fn, *args) def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: """Schedule a task on the root nursery (must be called on the host thread).""" @@ -475,7 +462,7 @@ async def start_session( return session def stop(self, timeout: float | None = 5.0) -> None: - if not self._started or self._token is None or self._shutdown is None: + if not self._started or self._portal is None or self._shutdown is None: return def _set() -> None: @@ -483,7 +470,7 @@ def _set() -> None: self._shutdown.set() try: - trio.from_thread.run_sync(_set, trio_token=self._token) + self._portal.run_sync(_set) except Exception: pass if self._thread is not None: diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 5fc9bceb..bb71b027 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -4,7 +4,6 @@ import functools import os -import queue import sys import threading from typing import TYPE_CHECKING @@ -17,6 +16,7 @@ from .gateway_base import get_execmodel from .gateway_base import loads_internal from .gateway_base import trace +from .portal import SyncReceiver if TYPE_CHECKING: from . import _trio_host @@ -48,10 +48,9 @@ def __init__( self._shutting_down = False self._idle = threading.Event() self._idle.set() - self._primary_q: queue.SimpleQueue[ + self._primary: SyncReceiver[ tuple[Channel, ExecItem, threading.Event] | None - ] = queue.SimpleQueue() - self._primary_wake = threading.Event() + ] = SyncReceiver() # Exec requests flow through a single pump task so admission happens # strictly in message-arrival order (trio task scheduling order is # deliberately unordered, so per-request tasks would race for the @@ -111,8 +110,7 @@ async def _run_exec(self, channel: Channel, item: ExecItem) -> None: try: if self.main_thread_only: done = threading.Event() - self._primary_q.put((channel, item, done)) - self._primary_wake.set() + self._primary.put((channel, item, done)) await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) else: await trio.to_thread.run_sync( @@ -126,15 +124,7 @@ async def _run_exec(self, channel: Channel, item: ExecItem) -> None: def integrate_as_primary_thread(self) -> None: """Block the main thread running main_thread_only exec tasks.""" while True: - self._primary_wake.wait() - try: - task = self._primary_q.get_nowait() - except queue.Empty: - self._primary_wake.clear() - try: - task = self._primary_q.get_nowait() - except queue.Empty: - continue + task = self._primary.get() if task is None: break channel, item, done = task @@ -146,8 +136,7 @@ def integrate_as_primary_thread(self) -> None: def trigger_shutdown(self) -> None: with self._lock: self._shutting_down = True - self._primary_q.put(None) - self._primary_wake.set() + self._primary.put(None) def waitall(self, timeout: float | None = None) -> bool: return self._idle.wait(timeout) diff --git a/src/execnet/portal.py b/src/execnet/portal.py new file mode 100644 index 00000000..9804ca45 --- /dev/null +++ b/src/execnet/portal.py @@ -0,0 +1,96 @@ +"""Cross-thread / cross-loop communication primitives. + +A :class:`LoopPortal` is a handle to a running trio loop that foreign +threads use to run functions on the loop or push work into it. Because +each direction only needs the *receiving* loop's token, two trio loops in +two threads can communicate by holding each other's portal (the sync +facade's host loop and a user loop; the worker loop and the process main +thread). + +:class:`SyncReceiver` covers the opposite direction: a plain thread +(typically the process main thread) receiving items produced on a loop, +in a way that stays interruptible by KeyboardInterrupt. +""" + +from __future__ import annotations + +import queue +import threading +from collections.abc import Awaitable +from collections.abc import Callable +from typing import Any +from typing import Generic +from typing import TypeVar + +import trio + +T = TypeVar("T") + + +class LoopPortal: + """Handle to a running trio loop, usable from foreign threads. + + Must be constructed on the loop's own thread (it captures the current + trio token). + """ + + def __init__(self) -> None: + self._token = trio.lowlevel.current_trio_token() + + def is_loop_thread(self) -> bool: + """Whether the calling thread is running this portal's loop.""" + try: + return trio.lowlevel.current_trio_token() is self._token + except RuntimeError: + return False + + def run(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: + """Run ``await async_fn(*args)`` on the loop, blocking this thread.""" + return trio.from_thread.run(async_fn, *args, trio_token=self._token) + + def run_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: + """Run ``sync_fn(*args)`` on the loop, blocking this thread.""" + return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) + + def post(self, sync_fn: Callable[..., object], *args: Any) -> None: + """Schedule ``sync_fn(*args)`` on the loop without waiting. + + Thread-safe and callable from the loop thread itself; all posts run + in strict FIFO order (``TrioToken.run_sync_soon``). Raises + ``trio.RunFinishedError`` once the loop has shut down. + """ + self._token.run_sync_soon(sync_fn, *args) + + +class SyncReceiver(Generic[T]): + """Receive items on a plain thread, KeyboardInterrupt-friendly. + + ``queue.SimpleQueue.get`` parks in a C-level lock acquire that shields + KeyboardInterrupt on the main thread; ``threading.Event.wait`` does not. + So producers put into an unbounded queue and set a wake event, and + :meth:`get` waits on the event and drains the queue. + """ + + def __init__(self) -> None: + self._items: queue.SimpleQueue[T] = queue.SimpleQueue() + self._wake = threading.Event() + + def put(self, item: T) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + self._items.put(item) + self._wake.set() + + def get(self) -> T: + """Block until an item is available (interruptible on main thread).""" + while True: + self._wake.wait() + try: + return self._items.get_nowait() + except queue.Empty: + # Re-check after clearing so a put between get_nowait and + # clear cannot be lost. + self._wake.clear() + try: + return self._items.get_nowait() + except queue.Empty: + continue From 0eb6cc531774148003de81ef1e84a6174d3eb804 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 14:45:43 +0200 Subject: [PATCH 20/91] feat: add the trio-native AsyncGateway core with low-level RawChannels Phase B.4 starts the async-native inversion: a new _trio_gateway module hosts AsyncGateway, whose single serve task reads the framed Message protocol off a ByteStream (receive_some + sans-IO FrameDecoder) and dispatches inline -- no receiver thread, no receive lock -- plus a writer task draining an unbounded outbound queue. RawChannel is the low-level half of the two-level channel model: id-routed raw byte payloads with the sync Channel close semantics (CHANNEL_CLOSE both ways, CHANNEL_LAST_MESSAGE as write-EOF leaving the peer sendonly, CHANNEL_CLOSE_ERROR surfacing as RemoteError). Errors keep the execnet contract -- OSError on closed sends, EOFError/RemoteError on receive -- so no trio exception types leak into the API. The ByteStream protocol and RECEIVE_CHUNK move here from _trio_host so the async core sits at the bottom of the dependency stack. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 398 +++++++++++++++++++++++++++++++++++ src/execnet/_trio_host.py | 24 +-- testing/test_trio_gateway.py | 202 ++++++++++++++++++ 3 files changed, 602 insertions(+), 22 deletions(-) create mode 100644 src/execnet/_trio_gateway.py create mode 100644 testing/test_trio_gateway.py diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py new file mode 100644 index 00000000..67e103ed --- /dev/null +++ b/src/execnet/_trio_gateway.py @@ -0,0 +1,398 @@ +"""Trio-native gateway core: async dispatch loop and low-level raw channels. + +Async-first counterpart of the sync machinery in ``gateway_base``: an +:class:`AsyncGateway` owns a :class:`ByteStream` and runs a single dispatch +task (stream -> ``FrameDecoder`` -> route). Message handlers execute inline +on that task, so there is no receiver thread and no receive lock. + +Two-level channel model: + +* :class:`RawChannel` (this module) -- id-routed raw byte payload streams + over the gateway: no serialization, no strconfig, no callbacks. + ``CHANNEL_DATA`` payloads route to the channel verbatim; the layer on top + decides what the bytes mean. +* ``AsyncChannel`` -- the serialized object API layered on a RawChannel. + +The code deliberately sticks to idioms an anyio backend can mirror later: +a neutral ``ByteStream`` protocol, the sans-IO ``FrameDecoder``, and +unbounded memory channels. +""" + +from __future__ import annotations + +import math +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from contextlib import suppress +from typing import Protocol + +import trio + +from .gateway_base import FrameDecoder +from .gateway_base import GatewayReceivedTerminate +from .gateway_base import Message +from .gateway_base import RemoteError +from .gateway_base import Unserializer +from .gateway_base import dumps_internal +from .gateway_base import loads_internal +from .gateway_base import trace + +RECEIVE_CHUNK = 65536 + + +class ByteStream(Protocol): + """Neutral bidirectional byte-stream protocol for gateway transports. + + ``trio.StapledStream`` (process/fd pipe pairs) and ``trio.SocketStream`` + satisfy this structurally; a future anyio backend's byte streams use the + same four names. ``send_eof`` signals write-EOF to the peer (half-close + for sockets; for pipe pairs trio falls back to closing the send half). + """ + + async def send_all(self, data: bytes) -> None: ... + + async def receive_some(self, max_bytes: int | None = None) -> bytes: ... + + async def send_eof(self) -> None: ... + + async def aclose(self) -> None: ... + + +class RawChannel: + """Low-level id-routed byte payload stream over an :class:`AsyncGateway`. + + Payload boundaries are preserved: every :meth:`send_bytes` arrives as + one :meth:`receive_bytes` result on the peer. No serialization and no + flow control beyond the gateway's outbound queue. + + Close semantics mirror the sync ``Channel`` state machine: + + * :meth:`aclose` closes both directions (``CHANNEL_CLOSE`` / + ``CHANNEL_CLOSE_ERROR`` to the peer). + * :meth:`send_eof` only ends our payload stream (``CHANNEL_LAST_MESSAGE``); + the peer drains, hits EOF, and may keep sending to us. + """ + + _strconfig: tuple[bool, bool] | None = None + + def __init__(self, gateway: AsyncGateway, id: int) -> None: + self.gateway = gateway + self.id = id + self._closed = False # no more sends (local aclose or remote close) + self._sent_eof = False + self._remote_closed = False + self._remote_error: RemoteError | None = None + self._payload_send, self._payloads = trio.open_memory_channel[bytes](math.inf) + + def __repr__(self) -> str: + state = "closed" if self._closed else "open" + return f"" + + async def send_bytes(self, data: bytes) -> None: + """Send one payload; the peer receives it as a single item. + + OSError is raised when the channel or gateway is closed, matching + the sync ``Channel.send`` contract. + """ + if self._closed or self._sent_eof: + raise OSError(f"cannot send to {self!r}") + await self.gateway._send(Message.CHANNEL_DATA, self.id, data) + + async def receive_bytes(self) -> bytes: + """Receive the next payload. + + Raises EOFError once the peer closed or sent EOF and all payloads + are drained; a peer close-with-error raises that ``RemoteError``. + """ + try: + return await self._payloads.receive() + except (trio.EndOfChannel, trio.ClosedResourceError): + raise self._pending_error() from None + + async def send_eof(self) -> None: + """Signal that no more payloads follow (peer keeps its send side).""" + if self._closed or self._sent_eof: + raise OSError(f"cannot send EOF to {self!r}") + self._sent_eof = True + await self.gateway._send(Message.CHANNEL_LAST_MESSAGE, self.id) + + async def aclose(self, error: str | None = None) -> None: + """Close both directions; ``error`` reaches the peer as a RemoteError.""" + if self._closed: + await trio.lowlevel.checkpoint() + return + self._closed = True + self._payload_send.close() + self.gateway._forget_channel(self.id) + if not self._remote_closed: + # A peer-initiated close needs no reply; a dead gateway is + # already as closed as it gets. + with suppress(OSError): + if error is not None: + await self.gateway._send( + Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(error) + ) + else: + await self.gateway._send(Message.CHANNEL_CLOSE, self.id) + + def __aiter__(self) -> RawChannel: + return self + + async def __anext__(self) -> bytes: + try: + return await self.receive_bytes() + except EOFError: + raise StopAsyncIteration from None + + def _pending_error(self) -> BaseException: + return ( + self._remote_error + or self.gateway._error + or EOFError(f"raw channel {self.id} closed") + ) + + # dispatch-loop internals (inline on the gateway's serve task) + + def _feed(self, data: bytes) -> None: + try: + self._payload_send.send_nowait(data) + except (trio.BrokenResourceError, trio.ClosedResourceError): + pass # locally closed: drop, like the sync channel + + def _close_from_remote( + self, error: RemoteError | None, *, sendonly: bool + ) -> None: + if error is not None: + self._remote_error = error + self._remote_closed = True + if not sendonly: + self._closed = True + self.gateway._forget_channel(self.id) + self._payload_send.close() + + +class AsyncGateway: + """Async-native gateway: the framed Message protocol over a ByteStream. + + Serving (``serve_gateway`` or ``nursery.start(gateway._serve)``) runs a + reader task that dispatches messages inline and a writer task draining + an unbounded outbound queue -- sends never block on the peer. + + ``_startcount`` follows the sync convention: locally allocated channel + ids step by two, coordinators from 1 (odd) and workers from 2 (even), + so the two peers never collide. + """ + + _error: BaseException | None = None + + def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None: + self._stream = stream + self.id = id + self._channels: dict[int, RawChannel] = {} + self._count = _startcount + self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) + self._outbound_send, self._outbound = trio.open_memory_channel[bytes](math.inf) + self._closed = False + self._serve_started = False + self._writer_done = trio.Event() + self._done = trio.Event() + + def __repr__(self) -> str: + state = "closed" if self._closed else "open" + return f"" + + @property + def closed(self) -> bool: + return self._closed + + async def wait_closed(self) -> None: + """Wait until serving has fully shut down.""" + await self._done.wait() + + def _trace(self, *msg: object) -> None: + trace(self.id, *msg) + + def open_raw_channel(self, id: int | None = None) -> RawChannel: + """Return the raw channel for ``id``, allocating a fresh id if None. + + An explicit id attaches to a channel the peer references (e.g. an id + received in a request payload); the same object is returned if the + dispatch loop already routed data to it. + """ + if self._closed: + raise OSError(f"connection already closed: {self!r}") + if id is None: + id = self._count + self._count += 2 + return self._channel_for(id) + + async def terminate(self) -> None: + """Send GATEWAY_TERMINATE to the peer, then close this side.""" + if not self._closed: + with suppress(OSError): + await self._send(Message.GATEWAY_TERMINATE) + await self.aclose() + + async def aclose(self) -> None: + """Flush queued frames, close the stream, and wait for shutdown.""" + if not self._closed: + self._closed = True + self._outbound_send.close() + with trio.move_on_after(5): + await self._writer_done.wait() + with trio.CancelScope(shield=True): + with suppress(Exception): + await self._stream.aclose() + if self._serve_started: + await self._done.wait() + else: + self._finish_channels() + + async def _serve( + self, task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED + ) -> None: + """Run the reader and writer until EOF, termination, or ``aclose``.""" + self._serve_started = True + try: + async with trio.open_nursery() as nursery: + nursery.start_soon(self._writer) + task_status.started() + await self._reader() + # Reader is done: let the writer flush queued frames + # (close replies), then stop it. + self._closed = True + self._outbound_send.close() + with trio.move_on_after(5): + await self._writer_done.wait() + nursery.cancel_scope.cancel() + finally: + self._closed = True + self._outbound_send.close() + self._finish_channels() + with trio.CancelScope(shield=True): + with suppress(Exception): + await self._stream.aclose() + self._done.set() + + async def _reader(self) -> None: + decoder = FrameDecoder() + try: + while True: + try: + data = await self._stream.receive_some(RECEIVE_CHUNK) + except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + if not self._closed: + self._error = exc + return + if not data: + decoder.close() # raises EOFError on a mid-frame EOF + raise EOFError("connection closed (no gateway termination)") + for message in decoder.feed(data): + self._trace("received", message) + self._dispatch(message) + except GatewayReceivedTerminate: + self._trace("received GATEWAY_TERMINATE") + except EOFError as exc: + self._trace("EOF without prior gateway termination message") + self._error = exc + + async def _writer(self) -> None: + try: + async for frame in self._outbound: + await self._stream.send_all(frame) + except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + self._trace("writer failed", exc) + if self._error is None: + self._error = exc + self._outbound_send.close() # fail future sends fast + else: + # Queue closed and drained: signal write-EOF to the peer. + with suppress(Exception): + await self._stream.send_eof() + finally: + self._writer_done.set() + + def _dispatch(self, message: Message) -> None: + """Route one message; runs inline on the serve task.""" + code = message.msgcode + channelid = message.channelid + if code == Message.CHANNEL_DATA: + self._channel_for(channelid)._feed(message.data) + elif code == Message.CHANNEL_CLOSE: + self._channel_for(channelid)._close_from_remote(None, sendonly=False) + elif code == Message.CHANNEL_CLOSE_ERROR: + error_message = loads_internal(message.data) + assert isinstance(error_message, str) + self._channel_for(channelid)._close_from_remote( + RemoteError(error_message), sendonly=False + ) + elif code == Message.CHANNEL_LAST_MESSAGE: + self._channel_for(channelid)._close_from_remote(None, sendonly=True) + elif code == Message.GATEWAY_TERMINATE: + raise GatewayReceivedTerminate(self) + elif code == Message.STATUS: + status = { + "numchannels": len(self._channels), + "numexecuting": 0, + "execmodel": "trio", + } + self._send_nowait(Message.CHANNEL_DATA, channelid, dumps_internal(status)) + self._send_nowait(Message.CHANNEL_CLOSE, channelid) + elif code == Message.RECONFIGURE: + data = loads_internal(message.data) + assert isinstance(data, tuple) + if channelid == 0: + self._strconfig = data + else: + # picked up by the serialized channel layer + self._channel_for(channelid)._strconfig = data + else: + # CHANNEL_EXEC / GATEWAY_START_*: not served by the async core + self._trace("rejecting unsupported message", message) + self._send_nowait( + Message.CHANNEL_CLOSE_ERROR, + channelid, + dumps_internal(f"unsupported message on async gateway: {message!r}"), + ) + + def _channel_for(self, id: int) -> RawChannel: + try: + return self._channels[id] + except KeyError: + channel = self._channels[id] = RawChannel(self, id) + return channel + + def _forget_channel(self, id: int) -> None: + self._channels.pop(id, None) + + async def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: + # The queue is unbounded, so this never waits on the peer -- but it + # is a real checkpoint and raises once the gateway is closed. + try: + await self._outbound_send.send(Message(msgcode, channelid, data).pack()) + except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + raise OSError("cannot send (already closed?)") from exc + + def _send_nowait(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: + """Enqueue a frame from a dispatch handler (sync, inline on the loop).""" + with suppress(trio.BrokenResourceError, trio.ClosedResourceError): + self._outbound_send.send_nowait(Message(msgcode, channelid, data).pack()) + + def _finish_channels(self) -> None: + for channel in list(self._channels.values()): + channel._close_from_remote(None, sendonly=True) + self._channels.clear() + + +@asynccontextmanager +async def serve_gateway( + stream: ByteStream, *, id: str, _startcount: int = 1 +) -> AsyncIterator[AsyncGateway]: + """Serve an :class:`AsyncGateway` over ``stream`` for the ``with`` body.""" + gateway = AsyncGateway(stream, id=id, _startcount=_startcount) + async with trio.open_nursery() as nursery: + await nursery.start(gateway._serve) + try: + yield gateway + finally: + await gateway.aclose() diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index cd20a0f2..b09dca15 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -18,11 +18,12 @@ from contextlib import suppress from typing import TYPE_CHECKING from typing import Any -from typing import Protocol from typing import TypeVar import trio +from ._trio_gateway import RECEIVE_CHUNK +from ._trio_gateway import ByteStream from .gateway_base import ExecModel from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate @@ -41,27 +42,6 @@ _CLOSE_WRITE = object() -RECEIVE_CHUNK = 65536 - - -class ByteStream(Protocol): - """Neutral bidirectional byte-stream protocol for gateway transports. - - ``trio.StapledStream`` (process/fd pipe pairs) and ``trio.SocketStream`` - satisfy this structurally; a future anyio backend's byte streams use the - same four names. ``send_eof`` signals write-EOF to the peer (half-close - for sockets; for pipe pairs trio falls back to closing the send half). - """ - - async def send_all(self, data: bytes) -> None: ... - - async def receive_some(self, max_bytes: int | None = None) -> bytes: ... - - async def send_eof(self) -> None: ... - - async def aclose(self) -> None: ... - - def staple_process_stream(process: trio.Process) -> ByteStream: """One bidirectional stream over a Trio Process stdin/stdout pair.""" assert process.stdin is not None diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py new file mode 100644 index 00000000..872c35a5 --- /dev/null +++ b/testing/test_trio_gateway.py @@ -0,0 +1,202 @@ +"""Protocol tests for the trio-native async gateway core (RawChannel level). + +Two AsyncGateways are wired together over an in-memory stream pair (the +transport harness, as in ``TestFrameDecoder``); the assertions cover execnet +semantics: payload routing by channel id, the close/EOF/sendonly state +machine, RemoteError propagation, and gateway termination. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +import pytest +import trio +import trio.testing + +from execnet._trio_gateway import AsyncGateway +from execnet.gateway_base import Message +from execnet.gateway_base import RemoteError +from execnet.gateway_base import dumps_internal +from execnet.gateway_base import loads_internal + + +@asynccontextmanager +async def gateway_pair() -> AsyncIterator[tuple[AsyncGateway, AsyncGateway]]: + left_stream, right_stream = trio.testing.memory_stream_pair() + async with trio.open_nursery() as nursery: + left = AsyncGateway(left_stream, id="left", _startcount=1) + right = AsyncGateway(right_stream, id="right", _startcount=2) + await nursery.start(left._serve) + await nursery.start(right._serve) + try: + yield left, right + finally: + await left.aclose() + await right.aclose() + + +def test_payload_boundaries_are_preserved() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + await sender.send_bytes(b"first payload") + await sender.send_bytes(b"second") + assert await receiver.receive_bytes() == b"first payload" + assert await receiver.receive_bytes() == b"second" + + trio.run(main) + + +def test_send_eof_makes_peer_sendonly() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + await sender.send_bytes(b"data") + await sender.send_eof() + assert await receiver.receive_bytes() == b"data" + with pytest.raises(EOFError): + await receiver.receive_bytes() + # the receiver of an EOF may still send back + await receiver.send_bytes(b"reply") + assert await sender.receive_bytes() == b"reply" + with pytest.raises(OSError, match="cannot send"): + await sender.send_bytes(b"after eof") + + trio.run(main) + + +def test_close_drains_then_blocks_both_directions() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + await sender.send_bytes(b"x") + await sender.aclose() + # payloads sent before the close still drain + assert await receiver.receive_bytes() == b"x" + with pytest.raises(EOFError): + await receiver.receive_bytes() + with pytest.raises(OSError, match="cannot send"): + await receiver.send_bytes(b"y") + with pytest.raises(OSError, match="cannot send"): + await sender.send_bytes(b"z") + + trio.run(main) + + +def test_close_with_error_raises_remote_error() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + await sender.aclose(error="boom happened") + with pytest.raises(RemoteError, match="boom happened"): + await receiver.receive_bytes() + + trio.run(main) + + +def test_async_iteration_yields_payloads_until_eof() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_raw_channel() + receiver = right.open_raw_channel(sender.id) + payloads = [b"a", b"bb", b"ccc"] + for payload in payloads: + await sender.send_bytes(payload) + await sender.send_eof() + assert [data async for data in receiver] == payloads + + trio.run(main) + + +def test_terminate_closes_peer_cleanly() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + channel = right.open_raw_channel() + await left.terminate() + await right.wait_closed() + assert right._error is None + with pytest.raises(EOFError): + await channel.receive_bytes() + + trio.run(main) + + +def test_status_reply_travels_on_raw_channel() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + channel = left.open_raw_channel() + await left._send(Message.STATUS, channel.id) + status = loads_internal(await channel.receive_bytes()) + assert status["execmodel"] == "trio" + assert status["numexecuting"] == 0 + # the peer closes the status channel after the reply + with pytest.raises(EOFError): + await channel.receive_bytes() + + trio.run(main) + + +def test_unsupported_message_is_rejected_with_remote_error() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + channel = left.open_raw_channel() + await left._send( + Message.CHANNEL_EXEC, + channel.id, + dumps_internal(("code", None, None, {})), + ) + with pytest.raises(RemoteError, match="unsupported message"): + await channel.receive_bytes() + + trio.run(main) + + +def test_send_after_gateway_close_raises() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + channel = left.open_raw_channel() + await left.aclose() + with pytest.raises(OSError, match="cannot send"): + await channel.send_bytes(b"x") + with pytest.raises(OSError, match="already closed"): + left.open_raw_channel() + + trio.run(main) + + +def test_peer_disappearing_surfaces_eof_error() -> None: + async def main() -> None: + left_stream, right_stream = trio.testing.memory_stream_pair() + async with trio.open_nursery() as nursery: + right = AsyncGateway(right_stream, id="right", _startcount=2) + await nursery.start(right._serve) + channel = right.open_raw_channel() + # peer vanishes without a termination message + await left_stream.aclose() + await right.wait_closed() + with pytest.raises(EOFError): + await channel.receive_bytes() + + trio.run(main) + + +def test_mid_frame_eof_is_an_error() -> None: + async def main() -> None: + left_stream, right_stream = trio.testing.memory_stream_pair() + async with trio.open_nursery() as nursery: + right = AsyncGateway(right_stream, id="right", _startcount=2) + await nursery.start(right._serve) + frame = Message(Message.CHANNEL_DATA, 1, b"payload").pack() + await left_stream.send_all(frame[:5]) + await left_stream.aclose() + await right.wait_closed() + assert isinstance(right._error, EOFError) + assert "mid-frame" in str(right._error) + + trio.run(main) From ed5f99ffa66d942b16a969e0d2e2c87c2fb85b9d Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 15:06:32 +0200 Subject: [PATCH 21/91] feat: layer the serialized AsyncChannel API on RawChannel The high level of the two-level channel model: AsyncChannel wraps a RawChannel with dumps/loads per item, per-channel strconfig (RECONFIGURE travels the wire, both ends coerce on load), async iteration, receive timeouts via trio.fail_after surfacing execnet's TimeoutError, and wait_closed mirroring the sync waitclose contract (reraise RemoteError). Channel objects serialize over the wire: a duck-typed save_AsyncChannel emits the existing CHANNEL opcode and AsyncGateway grows a factory adapter the Unserializer resolves ids through, so channels received inside items attach to the local gateway like sync channels do. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 144 ++++++++++++++++++++++++++++++++--- src/execnet/gateway_base.py | 6 ++ testing/test_trio_gateway.py | 89 +++++++++++++++++++++- 3 files changed, 227 insertions(+), 12 deletions(-) diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 67e103ed..aefb0162 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -24,6 +24,7 @@ from collections.abc import AsyncIterator from contextlib import asynccontextmanager from contextlib import suppress +from typing import Any from typing import Protocol import trio @@ -32,6 +33,7 @@ from .gateway_base import GatewayReceivedTerminate from .gateway_base import Message from .gateway_base import RemoteError +from .gateway_base import TimeoutError from .gateway_base import Unserializer from .gateway_base import dumps_internal from .gateway_base import loads_internal @@ -81,6 +83,7 @@ def __init__(self, gateway: AsyncGateway, id: int) -> None: self._closed = False # no more sends (local aclose or remote close) self._sent_eof = False self._remote_closed = False + self._receive_closed = trio.Event() # no more payloads will arrive self._remote_error: RemoteError | None = None self._payload_send, self._payloads = trio.open_memory_channel[bytes](math.inf) @@ -123,6 +126,7 @@ async def aclose(self, error: str | None = None) -> None: return self._closed = True self._payload_send.close() + self._receive_closed.set() self.gateway._forget_channel(self.id) if not self._remote_closed: # A peer-initiated close needs no reply; a dead gateway is @@ -159,18 +163,129 @@ def _feed(self, data: bytes) -> None: except (trio.BrokenResourceError, trio.ClosedResourceError): pass # locally closed: drop, like the sync channel - def _close_from_remote( - self, error: RemoteError | None, *, sendonly: bool - ) -> None: + def _close_from_remote(self, error: RemoteError | None, *, sendonly: bool) -> None: if error is not None: self._remote_error = error self._remote_closed = True + self._receive_closed.set() if not sendonly: self._closed = True self.gateway._forget_channel(self.id) self._payload_send.close() +class AsyncChannel: + """Serialized object API over a :class:`RawChannel`. + + Every payload is one dumps/loads-serialized item; close/EOF semantics + and error propagation come from the raw layer. Channels are + async-iterable, and :meth:`receive` supports the familiar execnet + timeout (raising ``TimeoutError``). + + Channel objects themselves serialize: sending an AsyncChannel inside an + item transfers a reference the peer receives as its own AsyncChannel + for the same id (the wire CHANNEL opcode, as with sync channels). + """ + + RemoteError = RemoteError + TimeoutError = TimeoutError + + def __init__(self, raw: RawChannel) -> None: + self._raw = raw + self.gateway = raw.gateway + self.id = raw.id + + def __repr__(self) -> str: + flag = "closed" if self.isclosed() else "open" + return f"" + + def isclosed(self) -> bool: + """Return True if the channel is closed for sending.""" + return self._raw._closed + + async def send(self, item: object) -> None: + """Serialize ``item`` and send it to the other side. + + The item must be a simple Python type; OSError is raised when the + channel or gateway is closed. + """ + if self.isclosed(): + raise OSError(f"cannot send to {self!r}") + await self._raw.send_bytes(dumps_internal(item)) + + async def receive(self, timeout: float | None = None) -> Any: + """Receive the next item sent from the other side. + + Raises EOFError once the peer closed or sent EOF, a RemoteError for + a peer close-with-error, and TimeoutError if no item arrived within + ``timeout`` seconds. + """ + if timeout is None: + data = await self._raw.receive_bytes() + else: + try: + with trio.fail_after(timeout): + data = await self._raw.receive_bytes() + except trio.TooSlowError: + raise TimeoutError("no item after %r seconds" % timeout) from None + return loads_internal(data, self) + + async def send_eof(self) -> None: + """Signal that no more items follow (peer keeps its send side).""" + await self._raw.send_eof() + + async def aclose(self, error: str | None = None) -> None: + """Close the channel; ``error`` reaches the peer as a RemoteError.""" + await self._raw.aclose(error) + + async def wait_closed(self) -> None: + """Wait until the peer closed or sent EOF; reraise remote errors.""" + await self._raw._receive_closed.wait() + error = self._raw._remote_error or self.gateway._error + if error is not None: + raise error + + async def reconfigure( + self, py2str_as_py3str: bool = True, py3str_as_py2str: bool = False + ) -> None: + """Set the string coercion for both ends of this channel.""" + strconfig = (py2str_as_py3str, py3str_as_py2str) + self._raw._strconfig = strconfig + await self.gateway._send( + Message.RECONFIGURE, self.id, dumps_internal(strconfig) + ) + + def __aiter__(self) -> AsyncChannel: + return self + + async def __anext__(self) -> Any: + try: + return await self.receive() + except EOFError: + raise StopAsyncIteration from None + + # Unserializer duck-type: loads_internal(data, self) reads _strconfig + # and _channelfactory off the object to resolve CHANNEL opcodes. + + @property + def _strconfig(self) -> tuple[bool, bool]: + return self._raw._strconfig or self.gateway._strconfig + + @property + def _channelfactory(self) -> _AsyncChannelFactory: + return self.gateway._channelfactory + + +class _AsyncChannelFactory: + """Duck-typed factory for the Unserializer CHANNEL opcode (``.new(id)``).""" + + def __init__(self, gateway: AsyncGateway) -> None: + self.gateway = gateway + + def new(self, id: int) -> AsyncChannel: + return self.gateway.open_channel(id) + + class AsyncGateway: """Async-native gateway: the framed Message protocol over a ByteStream. @@ -189,6 +304,8 @@ def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None self._stream = stream self.id = id self._channels: dict[int, RawChannel] = {} + self._async_channels: dict[int, AsyncChannel] = {} + self._channelfactory = _AsyncChannelFactory(self) self._count = _startcount self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) self._outbound_send, self._outbound = trio.open_memory_channel[bytes](math.inf) @@ -226,6 +343,15 @@ def open_raw_channel(self, id: int | None = None) -> RawChannel: self._count += 2 return self._channel_for(id) + def open_channel(self, id: int | None = None) -> AsyncChannel: + """Return the serialized channel for ``id``, allocating one if None.""" + raw = self.open_raw_channel(id) + try: + return self._async_channels[raw.id] + except KeyError: + channel = self._async_channels[raw.id] = AsyncChannel(raw) + return channel + async def terminate(self) -> None: """Send GATEWAY_TERMINATE to the peer, then close this side.""" if not self._closed: @@ -240,9 +366,8 @@ async def aclose(self) -> None: self._outbound_send.close() with trio.move_on_after(5): await self._writer_done.wait() - with trio.CancelScope(shield=True): - with suppress(Exception): - await self._stream.aclose() + with trio.CancelScope(shield=True), suppress(Exception): + await self._stream.aclose() if self._serve_started: await self._done.wait() else: @@ -269,9 +394,8 @@ async def _serve( self._closed = True self._outbound_send.close() self._finish_channels() - with trio.CancelScope(shield=True): - with suppress(Exception): - await self._stream.aclose() + with trio.CancelScope(shield=True), suppress(Exception): + await self._stream.aclose() self._done.set() async def _reader(self) -> None: @@ -364,6 +488,7 @@ def _channel_for(self, id: int) -> RawChannel: def _forget_channel(self, id: int) -> None: self._channels.pop(id, None) + self._async_channels.pop(id, None) async def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: # The queue is unbounded, so this never waits on the peer -- but it @@ -382,6 +507,7 @@ def _finish_channels(self) -> None: for channel in list(self._channels.values()): channel._close_from_remote(None, sendonly=True) self._channels.clear() + self._async_channels.clear() @asynccontextmanager diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index d36c7542..9d39cce0 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1622,3 +1622,9 @@ def save_frozenset(self, s: frozenset[object]) -> None: def save_Channel(self, channel: Channel) -> None: self._write(opcode.CHANNEL) self._write_int4(channel.id) + + def save_AsyncChannel(self, channel: Any) -> None: + # trio-native channel (execnet._trio_gateway); same wire opcode, + # duck-typed here to avoid importing the async core. + self._write(opcode.CHANNEL) + self._write_int4(channel.id) diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index 872c35a5..d95622af 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -15,6 +15,7 @@ import trio import trio.testing +from execnet import gateway_base from execnet._trio_gateway import AsyncGateway from execnet.gateway_base import Message from execnet.gateway_base import RemoteError @@ -129,7 +130,7 @@ async def main() -> None: def test_status_reply_travels_on_raw_channel() -> None: async def main() -> None: - async with gateway_pair() as (left, right): + async with gateway_pair() as (left, _right): channel = left.open_raw_channel() await left._send(Message.STATUS, channel.id) status = loads_internal(await channel.receive_bytes()) @@ -144,7 +145,7 @@ async def main() -> None: def test_unsupported_message_is_rejected_with_remote_error() -> None: async def main() -> None: - async with gateway_pair() as (left, right): + async with gateway_pair() as (left, _right): channel = left.open_raw_channel() await left._send( Message.CHANNEL_EXEC, @@ -159,7 +160,7 @@ async def main() -> None: def test_send_after_gateway_close_raises() -> None: async def main() -> None: - async with gateway_pair() as (left, right): + async with gateway_pair() as (left, _right): channel = left.open_raw_channel() await left.aclose() with pytest.raises(OSError, match="cannot send"): @@ -200,3 +201,85 @@ async def main() -> None: assert "mid-frame" in str(right._error) trio.run(main) + + +def test_channel_serializes_builtin_items() -> None: + items = [42, "text", b"bytes", [1, 2], ("a", 1), {"key": [True, None]}, {1, 2}] + + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_channel() + receiver = right.open_channel(sender.id) + for item in items: + await sender.send(item) + await sender.send_eof() + assert [item async for item in receiver] == items + + trio.run(main) + + +def test_channel_receive_timeout() -> None: + async def main() -> None: + async with gateway_pair() as (left, _right): + channel = left.open_channel() + with pytest.raises(gateway_base.TimeoutError): + await channel.receive(timeout=0.05) + + trio.run(main) + + +def test_channel_close_with_error_and_wait_closed() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_channel() + receiver = right.open_channel(sender.id) + await sender.aclose(error="exec exploded") + with pytest.raises(RemoteError, match="exec exploded"): + await receiver.receive() + with pytest.raises(RemoteError, match="exec exploded"): + await receiver.wait_closed() + + trio.run(main) + + +def test_channel_wait_closed_on_clean_eof() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_channel() + receiver = right.open_channel(sender.id) + await sender.send(1) + await sender.send_eof() + await receiver.wait_closed() + # items sent before the EOF still drain after waitclose + assert await receiver.receive() == 1 + + trio.run(main) + + +def test_channel_objects_travel_over_the_wire() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + carrier = left.open_channel() + right_carrier = right.open_channel(carrier.id) + extra = left.open_channel() + await carrier.send({"reply-to": extra}) + received = await right_carrier.receive() + remote_extra = received["reply-to"] + assert remote_extra.id == extra.id + await remote_extra.send("over the transferred channel") + assert await extra.receive() == "over the transferred channel" + + trio.run(main) + + +def test_channel_reconfigure_string_coercion() -> None: + async def main() -> None: + async with gateway_pair() as (left, right): + sender = left.open_channel() + receiver = right.open_channel(sender.id) + await sender.reconfigure(py3str_as_py2str=True) + await receiver.send("text") + # our side now loads py3 strings as bytes + assert await sender.receive() == b"text" + + trio.run(main) From 80ad5dfade4a14bedcece250af1530fcaacafa8e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 15:35:21 +0200 Subject: [PATCH 22/91] feat: run async popen gateways with remote_exec inside the user's trio run open_popen_gateway spawns a _trio_worker subprocess, does the handshake, and serves an AsyncGateway directly in the caller's nursery -- the first end-to-end trio-native path with no host thread. AsyncGateway grows remote_exec accepting the same source kinds (string / pure function / module) as the sync API; exec-finish close, RemoteError propagation, and concurrent execs all flow through the raw/serialized channel layers. Source normalization moves to _exec_source (shared by both coordinators; gateway.py re-exports the old names for its tests), and the transport helpers (staple_*, handshake, popen argv) move down into _trio_gateway so the async core has no dependency on the sync host machinery. Co-Authored-By: Claude Fable 5 --- src/execnet/_exec_source.py | 87 ++++++++++++++++++++++++ src/execnet/_trio_gateway.py | 128 +++++++++++++++++++++++++++++++++++ src/execnet/_trio_host.py | 64 ++---------------- src/execnet/_trio_worker.py | 4 +- src/execnet/gateway.py | 81 ++++------------------ testing/test_gateway.py | 4 +- testing/test_trio_gateway.py | 68 +++++++++++++++++++ testing/test_xspec.py | 4 +- 8 files changed, 307 insertions(+), 133 deletions(-) create mode 100644 src/execnet/_exec_source.py diff --git a/src/execnet/_exec_source.py b/src/execnet/_exec_source.py new file mode 100644 index 00000000..10c55795 --- /dev/null +++ b/src/execnet/_exec_source.py @@ -0,0 +1,87 @@ +"""Normalize ``remote_exec`` sources (string / function / module) to code. + +Shared by the sync coordinator ``Gateway`` and the trio-native +``AsyncGateway`` so both accept the same source kinds with identical +restrictions (pure functions taking ``channel`` first, no closures, no +non-builtin globals). +""" + +from __future__ import annotations + +import inspect +import linecache +import textwrap +import types +from collections.abc import Callable + + +def normalize_exec_source( + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + kwargs: dict[str, object], +) -> tuple[str, str | None, str | None]: + """Return ``(source, file_name, call_name)`` for a CHANNEL_EXEC payload.""" + call_name = None + file_name = None + if isinstance(source, types.ModuleType): + file_name = inspect.getsourcefile(source) + linecache.updatecache(file_name) # type: ignore[arg-type] + source = inspect.getsource(source) + elif isinstance(source, types.FunctionType): + call_name = source.__name__ + file_name = inspect.getsourcefile(source) + source = _source_of_function(source) + else: + source = textwrap.dedent(str(source)) + + if not call_name and kwargs: + raise TypeError("can't pass kwargs to non-function remote_exec") + return source, file_name, call_name + + +def _find_non_builtin_globals(source: str, codeobj: types.CodeType) -> list[str]: + import ast + import builtins + + vars = dict.fromkeys(codeobj.co_varnames) + return [ + node.id + for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.Name) + and node.id not in vars + and node.id not in builtins.__dict__ + ] + + +def _source_of_function(function: types.FunctionType | Callable[..., object]) -> str: + if function.__name__ == "": + raise ValueError("can't evaluate lambda functions'") + # XXX: we dont check before remote instantiation + # if arguments are used properly + try: + sig = inspect.getfullargspec(function) + except AttributeError: + args = inspect.getargspec(function)[0] + else: + args = sig.args + if not args or args[0] != "channel": + raise ValueError("expected first function argument to be `channel`") + + closure = function.__closure__ + codeobj = function.__code__ + + if closure is not None: + raise ValueError("functions with closures can't be passed") + + try: + source = inspect.getsource(function) + except OSError as e: + raise ValueError("can't find source file for %s" % function) from e + + source = textwrap.dedent(source) # just for inner functions + + used_globals = _find_non_builtin_globals(source, codeobj) + if used_globals: + raise ValueError("the use of non-builtin globals isn't supported", used_globals) + + leading_ws = "\n" * (codeobj.co_firstlineno - 1) + return leading_ws + source diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index aefb0162..f51eec9d 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -21,7 +21,11 @@ from __future__ import annotations import math +import subprocess +import sys +import types from collections.abc import AsyncIterator +from collections.abc import Callable from contextlib import asynccontextmanager from contextlib import suppress from typing import Any @@ -29,6 +33,7 @@ import trio +from ._exec_source import normalize_exec_source from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate from .gateway_base import Message @@ -60,6 +65,69 @@ async def send_eof(self) -> None: ... async def aclose(self) -> None: ... +def staple_process_stream(process: trio.Process) -> ByteStream: + """One bidirectional stream over a Trio Process stdin/stdout pair.""" + assert process.stdin is not None + assert process.stdout is not None + return trio.StapledStream(process.stdin, process.stdout) + + +def staple_fd_stream(read_fd: int, write_fd: int) -> ByteStream: + """One bidirectional stream over OS pipe fds (worker stdio pipes).""" + return trio.StapledStream( + trio.lowlevel.FdStream(write_fd), trio.lowlevel.FdStream(read_fd) + ) + + +async def read_handshake_ack(stream: ByteStream, what: str) -> None: + """Wait for the worker's single ``b"1"`` ready byte.""" + ack = await stream.receive_some(1) + if ack != b"1": + raise EOFError(f"bad {what} handshake: {ack!r}") + + +async def open_popen_process(args: list[str]) -> trio.Process: + return await trio.lowlevel.open_process( + args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + +def popen_module_args(spec: Any) -> list[str]: + """Launch the Trio worker as a module: ``python -m execnet._trio_worker``. + + No source is sent over the wire; the worker imports the installed execnet + + trio. Used for same-interpreter popen and for a ``python=`` interpreter that + already has execnet (so ``sys.executable`` stays that interpreter). + """ + from . import _provision + + if getattr(spec, "python", None): + interpreter = _provision.shell_split_path(spec.python) + else: + interpreter = [sys.executable] + + args = [*interpreter, "-u"] + if getattr(spec, "dont_write_bytecode", False): + args.append("-B") + args += ["-m", "execnet._trio_worker", _provision.worker_cli_arg(spec)] + return args + + +def popen_worker_argv(spec: Any) -> list[str]: + """Argv for a popen worker: direct module launch, or uv-provisioned. + + A bare ``python=`` interpreter without execnet gets execnet + trio + provisioned via ``uv``; otherwise the worker module is launched directly. + """ + from . import _provision + + if spec.python and not _provision.target_has_execnet(spec.python): + return _provision.uv_worker_argv(spec) + return popen_module_args(spec) + + class RawChannel: """Low-level id-routed byte payload stream over an :class:`AsyncGateway`. @@ -352,6 +420,27 @@ def open_channel(self, id: int | None = None) -> AsyncChannel: channel = self._async_channels[raw.id] = AsyncChannel(raw) return channel + async def remote_exec( + self, + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + **kwargs: object, + ) -> AsyncChannel: + """Connect a new channel to remote execution of ``source``. + + Accepts the same source kinds as the sync ``Gateway.remote_exec``: + a source string, a pure function called with ``channel`` and + ``**kwargs``, or a module. The remote end closes the channel when + execution finishes. + """ + source, file_name, call_name = normalize_exec_source(source, kwargs) + channel = self.open_channel() + await self._send( + Message.CHANNEL_EXEC, + channel.id, + dumps_internal((source, file_name, call_name, kwargs)), + ) + return channel + async def terminate(self) -> None: """Send GATEWAY_TERMINATE to the peer, then close this side.""" if not self._closed: @@ -522,3 +611,42 @@ async def serve_gateway( yield gateway finally: await gateway.aclose() + + +@asynccontextmanager +async def open_popen_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGateway]: + """Spawn a popen worker and serve an AsyncGateway over its stdio. + + Runs inside the caller's own trio run -- no host thread involved. On + exit the worker is asked to terminate (GATEWAY_TERMINATE), and killed + if it has not exited within a bounded grace period. + """ + from .xspec import XSpec + + if not isinstance(spec, XSpec): + spec = XSpec(spec) + if spec.execmodel is None: + # the sync Group normally stamps this before spawning + spec.execmodel = "thread" + process = await open_popen_process(popen_worker_argv(spec)) + try: + stream = staple_process_stream(process) + await read_handshake_ack(stream, "popen") + except BaseException: + with trio.CancelScope(shield=True), trio.move_on_after(5): + process.kill() + await process.wait() + raise + try: + async with serve_gateway(stream, id=spec.id or "popen-async") as gateway: + try: + yield gateway + finally: + await gateway.terminate() + finally: + with trio.CancelScope(shield=True): + with trio.move_on_after(5): + await process.wait() + if process.returncode is None: + process.kill() + await process.wait() diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index b09dca15..78be2c71 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -24,6 +24,10 @@ from ._trio_gateway import RECEIVE_CHUNK from ._trio_gateway import ByteStream +from ._trio_gateway import open_popen_process +from ._trio_gateway import popen_worker_argv +from ._trio_gateway import read_handshake_ack +from ._trio_gateway import staple_process_stream from .gateway_base import ExecModel from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate @@ -42,27 +46,6 @@ _CLOSE_WRITE = object() -def staple_process_stream(process: trio.Process) -> ByteStream: - """One bidirectional stream over a Trio Process stdin/stdout pair.""" - assert process.stdin is not None - assert process.stdout is not None - return trio.StapledStream(process.stdin, process.stdout) - - -def staple_fd_stream(read_fd: int, write_fd: int) -> ByteStream: - """One bidirectional stream over OS pipe fds (worker stdio pipes).""" - return trio.StapledStream( - trio.lowlevel.FdStream(write_fd), trio.lowlevel.FdStream(read_fd) - ) - - -async def read_handshake_ack(stream: ByteStream, what: str) -> None: - """Wait for the worker's single ``b"1"`` ready byte.""" - ack = await stream.receive_some(1) - if ack != b"1": - raise EOFError(f"bad {what} handshake: {ack!r}") - - _CHANNEL_EOF = object() @@ -458,35 +441,6 @@ def _set() -> None: self._started = False -async def open_popen_process(args: list[str]) -> trio.Process: - return await trio.lowlevel.open_process( - args, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - - -def popen_module_args(spec: Any) -> list[str]: - """Launch the Trio worker as a module: ``python -m execnet._trio_worker``. - - No source is sent over the wire; the worker imports the installed execnet + - trio. Used for same-interpreter popen and for a ``python=`` interpreter that - already has execnet (so ``sys.executable`` stays that interpreter). - """ - from . import _provision - - if getattr(spec, "python", None): - interpreter = _provision.shell_split_path(spec.python) - else: - interpreter = [sys.executable] - - args = [*interpreter, "-u"] - if getattr(spec, "dont_write_bytecode", False): - args.append("-B") - args += ["-m", "execnet._trio_worker", _provision.worker_cli_arg(spec)] - return args - - class _TempIO: """Placeholder IO used only while constructing a Trio-backed Gateway.""" @@ -572,15 +526,7 @@ def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: a foreign interpreter (``python=``) is provisioned via ``uv``. Either way the worker imports execnet + trio; nothing is sent over the wire to bootstrap it. """ - from . import _provision - - if spec.python and not _provision.target_has_execnet(spec.python): - # bare interpreter: provision execnet + trio via uv - args = _provision.uv_worker_argv(spec) - else: - # same interpreter, or a python= that already has execnet - args = popen_module_args(spec) - return _open_trio_gateway(group, spec, args) + return _open_trio_gateway(group, spec, popen_worker_argv(spec)) def ssh_trio_args(spec: Any) -> tuple[list[str], bytes]: diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index bb71b027..01b0186c 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -246,9 +246,9 @@ async def _start() -> _trio_host.ProtocolSession: async def _make_fd_io(read_fd: int, write_fd: int) -> Any: - from . import _trio_host + from . import _trio_gateway - return _trio_host.staple_fd_stream(read_fd, write_fd) + return _trio_gateway.staple_fd_stream(read_fd, write_fd) def serve_popen_trio(id: str, execmodel: str = "thread") -> None: diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index 593bc6fe..b1d3ed46 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -5,21 +5,30 @@ from __future__ import annotations -import inspect -import linecache -import textwrap import types from collections.abc import Callable from typing import TYPE_CHECKING from typing import Any from . import gateway_base +from ._exec_source import _find_non_builtin_globals +from ._exec_source import _source_of_function +from ._exec_source import normalize_exec_source from .gateway_base import IO from .gateway_base import Channel from .gateway_base import Message from .multi import Group from .xspec import XSpec +__all__ = [ + "Gateway", + "RInfo", + "RemoteStatus", + "_find_non_builtin_globals", + "_source_of_function", + "rinfo_source", +] + class Gateway(gateway_base.BaseGateway): """Gateway to a local or remote Python Interpreter.""" @@ -127,22 +136,7 @@ def remote_exec( will be available in the global namespace of the remotely executing code. """ - call_name = None - file_name = None - if isinstance(source, types.ModuleType): - file_name = inspect.getsourcefile(source) - linecache.updatecache(file_name) # type: ignore[arg-type] - source = inspect.getsource(source) - elif isinstance(source, types.FunctionType): - call_name = source.__name__ - file_name = inspect.getsourcefile(source) - source = _source_of_function(source) - else: - source = textwrap.dedent(str(source)) - - if not call_name and kwargs: - raise TypeError("can't pass kwargs to non-function remote_exec") - + source, file_name, call_name = normalize_exec_source(source, kwargs) channel = self.newchannel() self._send( Message.CHANNEL_EXEC, @@ -185,52 +179,3 @@ def rinfo_source(channel) -> None: pid=os.getpid(), ) ) - - -def _find_non_builtin_globals(source: str, codeobj: types.CodeType) -> list[str]: - import ast - import builtins - - vars = dict.fromkeys(codeobj.co_varnames) - return [ - node.id - for node in ast.walk(ast.parse(source)) - if isinstance(node, ast.Name) - and node.id not in vars - and node.id not in builtins.__dict__ - ] - - -def _source_of_function(function: types.FunctionType | Callable[..., object]) -> str: - if function.__name__ == "": - raise ValueError("can't evaluate lambda functions'") - # XXX: we dont check before remote instantiation - # if arguments are used properly - try: - sig = inspect.getfullargspec(function) - except AttributeError: - args = inspect.getargspec(function)[0] - else: - args = sig.args - if not args or args[0] != "channel": - raise ValueError("expected first function argument to be `channel`") - - closure = function.__closure__ - codeobj = function.__code__ - - if closure is not None: - raise ValueError("functions with closures can't be passed") - - try: - source = inspect.getsource(function) - except OSError as e: - raise ValueError("can't find source file for %s" % function) from e - - source = textwrap.dedent(source) # just for inner functions - - used_globals = _find_non_builtin_globals(source, codeobj) - if used_globals: - raise ValueError("the use of non-builtin globals isn't supported", used_globals) - - leading_ws = "\n" * (codeobj.co_firstlineno - 1) - return leading_ws + source diff --git a/testing/test_gateway.py b/testing/test_gateway.py index c3c87e4a..b7098773 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -531,9 +531,9 @@ def test_no_tracing_by_default(self): ], ) def test_popen_args(spec: str, expected_args: list[str]) -> None: - from execnet import _trio_host + from execnet import _trio_gateway - args = _trio_host.popen_module_args(execnet.XSpec(spec + "//id=gw0")) + args = _trio_gateway.popen_module_args(execnet.XSpec(spec + "//id=gw0")) assert args[: len(expected_args)] == expected_args assert args[len(expected_args) :][:3] == ["-u", "-m", "execnet._trio_worker"] diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index d95622af..4825beec 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -17,6 +17,7 @@ from execnet import gateway_base from execnet._trio_gateway import AsyncGateway +from execnet._trio_gateway import open_popen_gateway from execnet.gateway_base import Message from execnet.gateway_base import RemoteError from execnet.gateway_base import dumps_internal @@ -272,6 +273,73 @@ async def main() -> None: trio.run(main) +def _remote_add(channel, a, b) -> None: # type: ignore[no-untyped-def] + channel.send(a + b) + + +class TestPopenAsyncGateway: + """Integration: an AsyncGateway serving a real popen worker inside + the user's own trio run (no host thread).""" + + def test_remote_exec_roundtrip(self) -> None: + async def main() -> None: + async with open_popen_gateway() as gateway: + channel = await gateway.remote_exec( + "channel.send(channel.receive() + 1)" + ) + await channel.send(41) + assert await channel.receive() == 42 + await channel.wait_closed() + + trio.run(main) + + def test_remote_exec_function_with_kwargs(self) -> None: + async def main() -> None: + async with open_popen_gateway() as gateway: + channel = await gateway.remote_exec(_remote_add, a=40, b=2) + assert await channel.receive() == 42 + + trio.run(main) + + def test_remote_error_propagates(self) -> None: + async def main() -> None: + async with open_popen_gateway() as gateway: + channel = await gateway.remote_exec("raise ValueError('kaboom')") + with pytest.raises(RemoteError, match="kaboom"): + await channel.receive() + + trio.run(main) + + def test_exec_finish_closes_channel_ending_iteration(self) -> None: + async def main() -> None: + async with open_popen_gateway() as gateway: + channel = await gateway.remote_exec( + "for i in range(3): channel.send(i)" + ) + assert [item async for item in channel] == [0, 1, 2] + + trio.run(main) + + def test_concurrent_remote_execs(self) -> None: + async def main() -> None: + async with open_popen_gateway() as gateway: + results = [] + + async def run_one(value: int) -> None: + channel = await gateway.remote_exec( + "channel.send(channel.receive() * 10)" + ) + await channel.send(value) + results.append(await channel.receive()) + + async with trio.open_nursery() as nursery: + for value in range(5): + nursery.start_soon(run_one, value) + assert sorted(results) == [0, 10, 20, 30, 40] + + trio.run(main) + + def test_channel_reconfigure_string_coercion() -> None: async def main() -> None: async with gateway_pair() as (left, right): diff --git a/testing/test_xspec.py b/testing/test_xspec.py index f9681f55..661e50af 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -76,10 +76,10 @@ def test_vagrant_options(self) -> None: assert args[:-1] == ["vagrant", "ssh", "default", "--", "-C"] def test_popen_with_sudo_python(self) -> None: - from execnet import _trio_host + from execnet import _trio_gateway spec = XSpec("popen//python=sudo python3//id=gw0") - args = _trio_host.popen_module_args(spec) + args = _trio_gateway.popen_module_args(spec) assert args[:5] == ["sudo", "python3", "-u", "-m", "execnet._trio_worker"] def test_env(self) -> None: From b3b1ab871c496db9a49730516512fdfe254bd0ca Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 15:51:33 +0200 Subject: [PATCH 23/91] feat: add AsyncGroup owning gateways as nursery child scopes AsyncGroup is the trio-native group: an async context manager whose nursery serves every makegateway() as a child task. Leaving the block terminates all gateways concurrently with the safe_terminate contract -- GATEWAY_TERMINATE plus a timeout grace, then kill, bounded at roughly twice the timeout even when a kill sticks (issues #43/#221) -- with the cleanup shielded so external cancellation cannot leak workers. open_popen_gateway becomes a thin single-gateway AsyncGroup wrapper, so the popen integration tests now exercise the group termination path too. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 142 ++++++++++++++++++++++++++++------- testing/test_trio_gateway.py | 44 +++++++++++ 2 files changed, 159 insertions(+), 27 deletions(-) diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index f51eec9d..37be39c8 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -28,11 +28,15 @@ from collections.abc import Callable from contextlib import asynccontextmanager from contextlib import suppress +from typing import TYPE_CHECKING from typing import Any from typing import Protocol import trio +if TYPE_CHECKING: + from typing_extensions import Self + from ._exec_source import normalize_exec_source from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate @@ -613,21 +617,8 @@ async def serve_gateway( await gateway.aclose() -@asynccontextmanager -async def open_popen_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGateway]: - """Spawn a popen worker and serve an AsyncGateway over its stdio. - - Runs inside the caller's own trio run -- no host thread involved. On - exit the worker is asked to terminate (GATEWAY_TERMINATE), and killed - if it has not exited within a bounded grace period. - """ - from .xspec import XSpec - - if not isinstance(spec, XSpec): - spec = XSpec(spec) - if spec.execmodel is None: - # the sync Group normally stamps this before spawning - spec.execmodel = "thread" +async def _connect_popen_worker(spec: Any) -> tuple[ByteStream, trio.Process]: + """Spawn a popen worker for ``spec`` and complete the ready handshake.""" process = await open_popen_process(popen_worker_argv(spec)) try: stream = staple_process_stream(process) @@ -637,16 +628,113 @@ async def open_popen_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGa process.kill() await process.wait() raise - try: - async with serve_gateway(stream, id=spec.id or "popen-async") as gateway: - try: - yield gateway - finally: - await gateway.terminate() - finally: - with trio.CancelScope(shield=True): - with trio.move_on_after(5): - await process.wait() - if process.returncode is None: - process.kill() + return stream, process + + +class AsyncGroup: + """Trio-native group: an async context manager owning the gateway nursery. + + Gateways created with :meth:`makegateway` are served as child tasks of + the group's nursery. Leaving the ``async with`` block terminates every + gateway with the safe_terminate contract: GATEWAY_TERMINATE plus a + ``timeout`` grace, then kill -- bounded at roughly twice the timeout + even when a kill gets stuck (see issues #43 / #221). + """ + + def __init__(self, termination_timeout: float = 10.0) -> None: + self._termination_timeout = termination_timeout + self._nursery: trio.Nursery | None = None + self._gateways: list[AsyncGateway] = [] + self._processes: dict[AsyncGateway, trio.Process] = {} + + def __repr__(self) -> str: + ids = [gateway.id for gateway in self._gateways] + return f"" + + async def __aenter__(self) -> Self: + self._nursery_manager = trio.open_nursery() + self._nursery = await self._nursery_manager.__aenter__() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: types.TracebackType | None, + ) -> bool | None: + # The serve tasks only end once their gateways shut down, so + # terminate before letting the nursery join its children. + # Shielded: cleanup stays bounded even under cancellation. + terminate_error: BaseException | None = None + try: + with trio.CancelScope(shield=True): + await self.terminate(self._termination_timeout) + except BaseException as error: + terminate_error = error + self._nursery = None + suppress_body_exc = await self._nursery_manager.__aexit__( + exc_type, exc_value, traceback + ) + if terminate_error is not None: + raise terminate_error + return suppress_body_exc + + async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: + """Create a gateway for ``spec`` served on the group's nursery. + + Only popen-style specs (including uv-provisioned ``python=``) are + supported on the async path for now. + """ + from .xspec import XSpec + + if self._nursery is None: + raise RuntimeError(f"{self!r} is not entered") + if not isinstance(spec, XSpec): + spec = XSpec(spec) + if not (spec.popen or spec.python): + raise ValueError(f"unsupported spec for AsyncGroup: {spec!r}") + if spec.execmodel is None: + # the sync Group normally stamps this before spawning + spec.execmodel = "thread" + if spec.id is None: + spec.id = "gw%d" % len(self._gateways) + stream, process = await _connect_popen_worker(spec) + gateway = AsyncGateway(stream, id=spec.id, _startcount=1) + await self._nursery.start(gateway._serve) + self._gateways.append(gateway) + self._processes[gateway] = process + return gateway + + async def terminate(self, timeout: float | None = None) -> None: + """Terminate all gateways; never hangs (kill after ``timeout``).""" + gateways = list(self._gateways) + self._gateways.clear() + async with trio.open_nursery() as nursery: + for gateway in gateways: + nursery.start_soon(self._terminate_one, gateway, timeout) + + async def _terminate_one( + self, gateway: AsyncGateway, timeout: float | None + ) -> None: + grace = math.inf if timeout is None else timeout + await gateway.terminate() + process = self._processes.pop(gateway, None) + if process is None: + return + with trio.move_on_after(grace): + await process.wait() + if process.returncode is None: + process.kill() + with trio.move_on_after(grace): await process.wait() + + +@asynccontextmanager +async def open_popen_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGateway]: + """Spawn one popen worker and serve an AsyncGateway over its stdio. + + Runs inside the caller's own trio run -- no host thread involved. + Convenience for a single-gateway :class:`AsyncGroup`. + """ + async with AsyncGroup() as group: + yield await group.makegateway(spec) diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index 4825beec..88e52983 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -17,6 +17,7 @@ from execnet import gateway_base from execnet._trio_gateway import AsyncGateway +from execnet._trio_gateway import AsyncGroup from execnet._trio_gateway import open_popen_gateway from execnet.gateway_base import Message from execnet.gateway_base import RemoteError @@ -340,6 +341,49 @@ async def run_one(value: int) -> None: trio.run(main) +class TestAsyncGroup: + def test_multiple_gateways_with_auto_ids(self) -> None: + async def main() -> None: + async with AsyncGroup() as group: + first = await group.makegateway() + second = await group.makegateway() + assert {first.id, second.id} == {"gw0", "gw1"} + for gateway in (first, second): + channel = await gateway.remote_exec("channel.send(42)") + assert await channel.receive() == 42 + + trio.run(main) + + def test_group_exit_terminates_and_reaps_workers(self) -> None: + async def main() -> None: + async with AsyncGroup() as group: + await group.makegateway() + await group.makegateway() + processes = list(group._processes.values()) + # workers exited on GATEWAY_TERMINATE, nobody had to kill them + assert [process.returncode for process in processes] == [0, 0] + + trio.run(main) + + def test_terminate_kills_hung_worker_within_bound(self) -> None: + async def main() -> None: + async with AsyncGroup(termination_timeout=1.0) as group: + gateway = await group.makegateway() + await gateway.remote_exec("import time\nwhile True: time.sleep(1)") + processes = list(group._processes.values()) + assert all(process.returncode is not None for process in processes) + + trio.run(main) + + def test_unsupported_spec_is_rejected(self) -> None: + async def main() -> None: + async with AsyncGroup() as group: + with pytest.raises(ValueError, match="unsupported spec"): + await group.makegateway("ssh=nowhere.example.invalid") + + trio.run(main) + + def test_channel_reconfigure_string_coercion() -> None: async def main() -> None: async with gateway_pair() as (left, right): From 8286b778bef0fe068b61cdbd20bd1b91ac5173bc Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 16:44:50 +0200 Subject: [PATCH 24/91] feat: make the via tunnel frame-native over raw channel routing The via transport no longer double-frames. ChannelFactory grows a raw-receiver registry (CHANNEL_DATA payloads for registered ids route verbatim, no serialization) plus allocate_id, giving the sync gateways the low-level half of the two-level channel model. The master relay forwards the sub-worker's ready byte alone and then runs its stdout through a FrameDecoder, sending exactly one whole sub-protocol frame per CHANNEL_DATA -- and writes coordinator payloads (one frame each, by writer construction) straight to the sub's stdin. Coordinator ends of the tunnel match: RawTunnelStream (sync master, replacing the interim ChannelByteStream hack) and RawChannelStream (async master, wrapping a RawChannel as a ByteStream). AsyncGroup accepts via= specs relayed through a group member, and terminates tunneled gateways before their masters. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 92 ++++++++++++++++++++++++++--- src/execnet/_trio_host.py | 111 +++++++++++++++++++++++------------ src/execnet/gateway_base.py | 45 ++++++++++++++ testing/test_trio_gateway.py | 21 +++++++ 4 files changed, 223 insertions(+), 46 deletions(-) diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 37be39c8..94f33838 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -246,6 +246,46 @@ def _close_from_remote(self, error: RemoteError | None, *, sendonly: bool) -> No self._payload_send.close() +class RawChannelStream: + """``ByteStream`` over a :class:`RawChannel` -- the frame-native via tunnel. + + The gateway writer performs one ``send_all`` per frame, so every raw + payload carries exactly one whole sub-protocol frame (the master relay + keeps that invariant in the other direction). ``receive_some`` buffers + payloads and honours ``max_bytes`` for the handshake read. + """ + + def __init__(self, raw: RawChannel) -> None: + self._raw = raw + self._buf = bytearray() + self._eof = False + + async def send_all(self, data: bytes) -> None: + await self._raw.send_bytes(data) + + async def receive_some(self, max_bytes: int | None = None) -> bytes: + if not self._buf and not self._eof: + try: + self._buf += await self._raw.receive_bytes() + except EOFError: + self._eof = True + except RemoteError as exc: + self._eof = True + raise EOFError(f"via tunnel closed: {exc}") from None + if max_bytes is None: + max_bytes = len(self._buf) + out = bytes(self._buf[:max_bytes]) + del self._buf[:max_bytes] + return out + + async def send_eof(self) -> None: + with suppress(OSError): + await self._raw.send_eof() + + async def aclose(self) -> None: + await self._raw.aclose() + + class AsyncChannel: """Serialized object API over a :class:`RawChannel`. @@ -517,7 +557,7 @@ async def _writer(self) -> None: try: async for frame in self._outbound: await self._stream.send_all(frame) - except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + except (trio.BrokenResourceError, trio.ClosedResourceError, OSError) as exc: self._trace("writer failed", exc) if self._error is None: self._error = exc @@ -682,8 +722,9 @@ async def __aexit__( async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: """Create a gateway for ``spec`` served on the group's nursery. - Only popen-style specs (including uv-provisioned ``python=``) are - supported on the async path for now. + Popen-style specs (including uv-provisioned ``python=``) and + ``via=`` sub-gateways relayed through a group member are supported + on the async path for now. """ from .xspec import XSpec @@ -698,20 +739,53 @@ async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: spec.execmodel = "thread" if spec.id is None: spec.id = "gw%d" % len(self._gateways) - stream, process = await _connect_popen_worker(spec) + process: trio.Process | None = None + if spec.via: + stream: ByteStream = await self._open_via_stream(spec) + else: + stream, process = await _connect_popen_worker(spec) gateway = AsyncGateway(stream, id=spec.id, _startcount=1) await self._nursery.start(gateway._serve) self._gateways.append(gateway) - self._processes[gateway] = process + if process is not None: + self._processes[gateway] = process return gateway + async def _open_via_stream(self, spec: Any) -> ByteStream: + """Ask the ``spec.via`` master to spawn a sub-worker; tunnel over a + raw channel (each payload one whole sub-protocol frame).""" + from . import _provision + + master = self._gateway_by_id(spec.via) + raw = master.open_raw_channel() + request = _provision.spawn_request(spec) + await master._send(Message.GATEWAY_START_SUB, raw.id, dumps_internal(request)) + stream = RawChannelStream(raw) + await read_handshake_ack(stream, "via") + return stream + + def _gateway_by_id(self, id: str) -> AsyncGateway: + for gateway in self._gateways: + if gateway.id == id: + return gateway + raise KeyError(f"no gateway {id!r} in {self!r}") + async def terminate(self, timeout: float | None = None) -> None: - """Terminate all gateways; never hangs (kill after ``timeout``).""" + """Terminate all gateways; never hangs (kill after ``timeout``). + + Tunneled (``via``) gateways go first so their termination frames + still travel through a live master. + """ gateways = list(self._gateways) self._gateways.clear() - async with trio.open_nursery() as nursery: - for gateway in gateways: - nursery.start_soon(self._terminate_one, gateway, timeout) + tunneled = [gw for gw in gateways if gw not in self._processes] + spawned = [gw for gw in gateways if gw in self._processes] + for batch in (tunneled, spawned): + if not batch: + continue + async with trio.open_nursery() as nursery: + for gateway in batch: + nursery.start_soon(self._terminate_one, gateway, timeout) async def _terminate_one( self, gateway: AsyncGateway, timeout: float | None diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 78be2c71..1bccc30e 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -49,36 +49,46 @@ _CHANNEL_EOF = object() -class ChannelByteStream: - """``ByteStream`` tunnelled over a sync execnet ``Channel``. +class RawTunnelStream: + """``ByteStream`` over a raw channel id on a sync master ``Gateway``. - INTERIM HACK: the ``via`` transport currently runs the sub-gateway protocol - as raw bytes over a channel to the master, which double-frames it (sub frame - -> CHANNEL_DATA -> master frame). This bridges the sync ``Channel`` to the - async protocol; it dissolves once low-level raw channels exist (Phase B.4). - - The channel callback (on the host loop) feeds an unbounded memory channel - that ``receive_some`` drains; writes ``channel.send`` raw bytes. + The frame-native via tunnel: the master relays whole sub-protocol + frames as verbatim CHANNEL_DATA payloads (no serialization, no + double-framing), so this stream only buffers payloads; the + sub-session's FrameDecoder sees exact frame boundaries. """ - def __init__(self, channel: Any) -> None: - self._channel = channel + def __init__(self, gateway: BaseGateway, channelid: int) -> None: + self._gateway = gateway + self.channelid = channelid self._send, self._recv = trio.open_memory_channel[Any](math.inf) self._buf = bytearray() self._eof = False - channel.setcallback(self._send.send_nowait, endmarker=_CHANNEL_EOF) + self._closed = False + gateway._channelfactory.register_raw_receiver( + channelid, self._on_data, self._on_close + ) + + def _on_data(self, data: bytes) -> None: + self._send.send_nowait(data) + + def _on_close(self, error: Any) -> None: + self._send.send_nowait(_CHANNEL_EOF if error is None else error) async def send_all(self, data: bytes) -> None: - self._channel.send(data) + self._gateway._send(Message.CHANNEL_DATA, self.channelid, data) async def receive_some(self, max_bytes: int | None = None) -> bytes: if not self._buf and not self._eof: - data = await self._recv.receive() - if data is _CHANNEL_EOF: + item = await self._recv.receive() + if item is _CHANNEL_EOF: + self._eof = True + elif isinstance(item, Exception): self._eof = True + raise EOFError(f"via tunnel closed: {item}") from None else: - assert isinstance(data, bytes) - self._buf += data + assert isinstance(item, bytes) + self._buf += item if max_bytes is None: max_bytes = len(self._buf) out = bytes(self._buf[:max_bytes]) @@ -86,10 +96,18 @@ async def receive_some(self, max_bytes: int | None = None) -> bytes: return out async def send_eof(self) -> None: - self._channel.close() + self._close_tunnel() async def aclose(self) -> None: - self._channel.close() + self._close_tunnel() + + def _close_tunnel(self) -> None: + if self._closed: + return + self._closed = True + self._gateway._channelfactory.unregister_raw_receiver(self.channelid) + with suppress(OSError): + self._gateway._send(Message.CHANNEL_CLOSE, self.channelid) async def adopt_socket(socket_fd: int) -> trio.SocketStream: @@ -689,24 +707,35 @@ def start_socketserver_via( async def _start_sub_and_relay( gateway: BaseGateway, channelid: int, request: dict[str, Any] ) -> None: - """Spawn a requested sub-worker and relay its Message protocol over the channel. - - Runs on the master's Trio host: bytes from the channel go to the sub's - stdin, and the sub's stdout goes back on the channel (the ``via`` transport). + """Spawn a requested sub-worker and relay its Message protocol frames. + + Runs on the master's Trio host (the ``via`` transport). The tunnel is + frame-native both ways: coordinator payloads arrive verbatim through the + raw-receiver registry and go to the sub's stdin unchanged (each payload + one whole frame), while the sub's stdout runs through a FrameDecoder so + every CHANNEL_DATA sent back carries exactly one frame -- except the + initial ready byte, which is forwarded on its own for the handshake. A stdin preamble (shipped wheel for a dev-version ssh sub) is streamed before the relayed protocol bytes. """ from . import _provision - channel = gateway._channelfactory.new(channelid) + def send_close_error(text: str) -> None: + with suppress(OSError): + gateway._send(Message.CHANNEL_CLOSE_ERROR, channelid, dumps_internal(text)) + try: args, preamble = _provision.sub_spawn_argv(request) process = await open_popen_process(args) except Exception as exc: - channel.close(f"could not spawn via sub-gateway: {exc}") + send_close_error(f"could not spawn via sub-gateway: {exc}") return send_ch, recv_ch = trio.open_memory_channel[Any](math.inf) - channel.setcallback(send_ch.send_nowait, endmarker=_CHANNEL_EOF) + gateway._channelfactory.register_raw_receiver( + channelid, + send_ch.send_nowait, + lambda error: send_ch.send_nowait(_CHANNEL_EOF), + ) async def coordinator_to_sub() -> None: assert process.stdin is not None @@ -721,12 +750,18 @@ async def coordinator_to_sub() -> None: async def sub_to_coordinator() -> None: assert process.stdout is not None - while True: - data = await process.stdout.receive_some(65536) - if not data: - break - channel.send(data) - channel.close() + ack = bytes(await process.stdout.receive_some(1)) + if ack: + gateway._send(Message.CHANNEL_DATA, channelid, ack) + decoder = FrameDecoder() + while True: + data = bytes(await process.stdout.receive_some(RECEIVE_CHUNK)) + if not data: + break + for message in decoder.feed(data): + gateway._send(Message.CHANNEL_DATA, channelid, message.pack()) + with suppress(OSError): + gateway._send(Message.CHANNEL_CLOSE, channelid) try: async with trio.open_nursery() as nursery: @@ -736,9 +771,9 @@ async def sub_to_coordinator() -> None: # Do not let a relay failure crash the host nursery; surface it on # the channel so the coordinator does not hang on the handshake. gateway._trace("via sub relay failed:", exc) - with suppress(Exception): - channel.close(f"via sub-gateway relay failed: {exc}") + send_close_error(f"via sub-gateway relay failed: {exc}") finally: + gateway._channelfactory.unregister_raw_receiver(channelid) with trio.move_on_after(5): await process.wait() @@ -759,14 +794,16 @@ def makegateway_via_trio(group: Any, spec: Any) -> Gateway: master = group[spec.via] host: TrioHost = group._ensure_trio_host() - channel = master.newchannel() + channelid = master._channelfactory.allocate_id() request = _provision.spawn_request(spec) - master._send(Message.GATEWAY_START_SUB, channel.id, dumps_internal(request)) remote = spec.ssh or spec.vagrant_ssh remoteaddress = f"{remote}[via {spec.via}]" if remote else None async def _create_and_attach() -> Gateway: - io = ChannelByteStream(channel) + # Register the raw receiver before the request goes out so no + # relayed frame can arrive unrouted. + io = RawTunnelStream(master, channelid) + master._send(Message.GATEWAY_START_SUB, channelid, dumps_internal(request)) await read_handshake_ack(io, "via") gw = execnet.Gateway(_TempIO(group.execmodel), spec) session = await host.start_session(gw, io) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 9d39cce0..22afa46e 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -890,6 +890,12 @@ def __init__(self, gateway: BaseGateway, startcount: int = 1) -> None: self._callbacks: dict[ int, tuple[Callable[[Any], Any], object, tuple[bool, bool]] ] = {} + # Channel ID => (feed, on_close) receiving CHANNEL_DATA verbatim + # (no serialization) -- the low-level raw channel routing used by + # byte-shaped consumers such as the via tunnel. + self._raw_receivers: dict[ + int, tuple[Callable[[bytes], None], Callable[[RemoteError | None], None]] + ] = {} self._writelock = gateway.execmodel.Lock() self.gateway = gateway self.count = startcount @@ -910,6 +916,31 @@ def new(self, id: int | None = None) -> Channel: channel = self._channels[id] = Channel(self.gateway, id) return channel + def allocate_id(self) -> int: + """Reserve a fresh channel id without creating a Channel object.""" + with self._writelock: + if self.finished: + raise OSError(f"connection already closed: {self.gateway}") + id = self.count + self.count += 2 + return id + + def register_raw_receiver( + self, + id: int, + feed: Callable[[bytes], None], + on_close: Callable[[RemoteError | None], None], + ) -> None: + """Route CHANNEL_DATA payloads for ``id`` verbatim to ``feed``. + + ``on_close`` fires once when the channel closes (with the peer's + RemoteError, if any) or when receiving finishes. + """ + self._raw_receivers[id] = (feed, on_close) + + def unregister_raw_receiver(self, id: int) -> None: + self._raw_receivers.pop(id, None) + def channels(self) -> list[Channel]: return self._list(self._channels.values()) @@ -925,6 +956,11 @@ def _no_longer_opened(self, id: int) -> None: callback(endmarker) def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> None: + raw = self._raw_receivers.pop(id, None) + if raw is not None: + _feed, on_close = raw + on_close(remoteerror) + return channel = self._channels.get(id) if channel is None: # channel already in "deleted" state @@ -945,6 +981,11 @@ def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> Non def _local_receive(self, id: int, data) -> None: # executes in receiver thread + raw = self._raw_receivers.get(id) + if raw is not None: + feed, _on_close = raw + feed(data) + return channel = self._channels.get(id) try: callback, _endmarker, strconfig = self._callbacks[id] @@ -974,6 +1015,10 @@ def _finished_receiving(self) -> None: self._local_close(id, sendonly=True) for id in self._list(self._callbacks): self._no_longer_opened(id) + for id in self._list(self._raw_receivers): + item = self._raw_receivers.pop(id, None) + if item is not None: + item[1](None) class ChannelFile: diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index 88e52983..630d3e7d 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -375,6 +375,27 @@ async def main() -> None: trio.run(main) + def test_via_gateway_relays_through_master(self) -> None: + async def main() -> None: + async with AsyncGroup() as group: + master = await group.makegateway("popen//id=master") + sub = await group.makegateway("popen//via=master") + master_channel = await master.remote_exec( + "import os; channel.send(os.getpid())" + ) + sub_channel = await sub.remote_exec( + "import os; channel.send(os.getpid())" + ) + master_pid = await master_channel.receive() + sub_pid = await sub_channel.receive() + # a real second process, reached through the master's relay + assert sub_pid != master_pid + echo = await sub.remote_exec("channel.send(channel.receive() * 2)") + await echo.send(21) + assert await echo.receive() == 42 + + trio.run(main) + def test_unsupported_spec_is_rejected(self) -> None: async def main() -> None: async with AsyncGroup() as group: From 57f899f78cc9b20e733c77cbb138deb5ce593f36 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 16:51:16 +0200 Subject: [PATCH 25/91] docs: record B.4 completion in the phase handoff Co-Authored-By: Claude Fable 5 --- handoff-phase-b-async-core.md | 145 ++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 handoff-phase-b-async-core.md diff --git a/handoff-phase-b-async-core.md b/handoff-phase-b-async-core.md new file mode 100644 index 00000000..8ba250ec --- /dev/null +++ b/handoff-phase-b-async-core.md @@ -0,0 +1,145 @@ +# Handoff: Phase B — invert execnet onto an async-native Trio core + +For a fresh session on branch `feat/trio-host-thread-io`. B.1–B.4 are done; +work continues at **B.5**. Plan context lives in session memory +(`trio-port-plan`), but everything needed is restated here. + +Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` +(never grep-filter pre-commit output). ssh paths have a real local harness +in `testing/test_ssh_local.py` (asyncssh server; system ssh client needed). +Known flake: `test_socket_installvia` EOFs rarely under load. + +## Where the repo stands (2026-07-24, after commit `8286b77`) + +Trio is the only IO path; no source is shipped over the wire (workers run +`python -m execnet._trio_worker `, foreign/remote interpreters +are uv-provisioned via `_provision.py`, dev coordinators ship a wheel). All +transports work: popen, `python=`, ssh, vagrant_ssh, socket, and `via` +sub-gateways (`GATEWAY_START_SUB` spawn-request + byte relay on the master). +The architecture is still sync-first: Trio hides behind the sync +`Channel`/`Gateway`/`Group`. Phase B inverts this. + +File map (src/execnet/): + +| file | role | +|---|---| +| `gateway_base.py` | `Message` wire protocol + **`FrameDecoder`** (sans-IO), serializer (incl. duck-typed `save_AsyncChannel` → CHANNEL opcode), sync `Channel`/`ChannelFactory` (now with **raw-receiver registry**: `register_raw_receiver`/`allocate_id`, verbatim CHANNEL_DATA routing), `BaseGateway`/`WorkerGateway`, `ExecModel`, `WorkerPool`, `HostNotFound` | +| `_trio_gateway.py` | **the async-native core (B.4)**: `ByteStream` Protocol, `RawChannel` (id-routed verbatim byte payloads, sync-Channel close semantics: OSError on closed sends, EOFError/RemoteError on receive, `send_eof` = LAST_MESSAGE/sendonly), `AsyncChannel` (dumps/loads per item, strconfig/RECONFIGURE, timeout via `fail_after`, `wait_closed`, channel-passing), `AsyncGateway` (single serve task: reader dispatches inline — no locks; writer drains unbounded queue, one `send_all` per frame), `AsyncGroup` (async CM owning the nursery; `makegateway` popen/`python=`/`via=`; terminate = tunneled first, then GATEWAY_TERMINATE + timeout + kill, ~2×timeout bound), `RawChannelStream` (RawChannel→ByteStream adapter), `open_popen_gateway`, transport helpers (`staple_*`, `read_handshake_ack`, `popen_worker_argv`) | +| `portal.py` | **`LoopPortal`** (trio-token holder) and **`SyncReceiver`** (loop→plain-thread queue, KI-interruptible `get()`) | +| `_trio_host.py` | sync-facade host: `TrioHost` (loop thread), `ProtocolSession`, `RawTunnelStream` (via tunnel over sync master, raw registry based), gateway factories, `GATEWAY_START_*` handlers (`_start_sub_and_relay` is frame-native: ready byte alone, then one whole frame per CHANNEL_DATA) | +| `_trio_worker.py` | worker entry, `TrioWorkerExec` (FIFO `_pump` admission), `_prepare_protocol_fds` | +| `gateway.py` | sync coordinator `Gateway` (remote_exec, exit, rinfo) | +| `_exec_source.py` | remote_exec source normalization shared by sync + async coordinators | +| `multi.py` | sync `Group`, `MultiChannel`, `safe_terminate` (WorkerPool-based) | +| `_provision.py` | uv provisioning, wheel build/ship/materialize, argv builders | + +Async-core tests live in `testing/test_trio_gateway.py` (memory_stream_pair +protocol tests + popen/via integration inside `trio.run`). + +Semantics that MUST survive (xdist depends on them): + +- Sends from non-loop threads block until the frame hit the OS write + (120s → OSError) so abrupt `os._exit` cannot drop "sent" data; sends from + the loop thread only enqueue. After close: `OSError("cannot send (already + closed?)")`; `trio.RunFinishedError` maps to the same. +- exec admission order = message arrival order (see `TrioWorkerExec._pump`; + `test_main_thread_only_concurrent_remote_exec_deadlock` guards this — + trio shuffles its run batch, so never rely on task-spawn order). +- Channel callbacks run on the receiver (loop) thread — keep for now. + +## Phase B goal + +Three public namespaces, all pure Trio (**no anyio/asyncio** — an +`execnet.anyio` backend is deferred Phase E; keep idioms portable: neutral +stream protocol, sans-IO protocol logic, no gratuitous trio-only constructs): + +``` +execnet.sync – today's blocking API, rebuilt as a facade (top-level + execnet.Group etc. stay as compatibility aliases) +execnet.trio – trio-native AsyncGroup/AsyncGateway/AsyncChannel, awaited + directly inside the user's own trio.run +execnet.portal – the communicating API (exists as portal.py; public + exposure happens in B.6) +``` + +## Remaining work + +### B.4 Async-native core — DONE (commits `0eb6cc5`..`8286b77`) + +Everything lives in `_trio_gateway.py` (see file map). Landed: +RawChannel + AsyncChannel two-level model, AsyncGateway single-task +dispatch (no `_receivelock` on the async path), AsyncGroup with bounded +nursery-scoped termination, async popen/`python=`/`via=` gateways with +`remote_exec` inside the user's own `trio.run`, and the frame-native via +tunnel (raw-receiver registry in the sync `ChannelFactory`; +`ChannelByteStream` is gone). + +Leftovers deliberately deferred: + +- **ChannelFile / makefile over RawChannel** — do together with the B.5 + facade (the current `ChannelFileRead` is str-based; decide there whether + a text wrapper stays for backward compat). +- rsync data plane / wheel shipping over raw channels — later. +- async ssh/socket gateways — the async `makegateway` only does + popen-style and `via=`; other transports stay on the sync host path + until B.5/B.6 need them. +- worker-side CHANNEL_EXEC on an AsyncGateway is rejected with a + RemoteError ("unsupported message") — async exec is Phase C + (`exec=task`). + +### B.5 Rebuild the sync API as a facade + +Public surface stays byte-for-byte: `Group`, `makegateway` spec strings, +`Gateway.remote_exec/exit/reconfigure/remote_status/hasreceiver`, `Channel` +send/receive/setcallback/makefile/waitclose/reconfigure, `MultiChannel`, +`RSync`, `execnet.dumps/loads/dump/load`, `HostNotFound`, `TimeoutError`, +`RemoteError`, `DataFormatError`. Existing tests keep running against the +facade unchanged — that is the acceptance bar. + +Known tricky spots: + +- `Channel.__del__` sends CHANNEL_CLOSE/LAST_MESSAGE during GC — must go + through the portal as a non-waiting post, and tolerate interpreter + shutdown (today: `suppress(OSError, ValueError)` + `Message is not None` + guard). +- KeyboardInterrupt lands on the main thread while blocked inside a portal + call — decide propagation (today the sync side blocks in + `threading.Event.wait`, which KI can interrupt). +- `group.execmodel` / `set_execmodel` / `spec.execmodel` stay as deprecated + shims. `main_thread_only` must keep working throughout (pytest-xdist runs + GUI-bound code on the worker main thread via + `TrioWorkerExec.integrate_as_primary_thread`). +- Retire `ExecModel` internals and `WorkerPool` at the END of the phase + (WorkerPool is still the worker exec duck-type target for STATUS and is + used by `multi.safe_terminate` + `testing/test_threadpool.py` + + `testing/conftest.py`'s `pool` fixture). + +### B.6 Namespace split + +Introduce `execnet.sync` / `execnet.trio` / `execnet.portal`; top-level +`execnet.*` aliases into `execnet.sync`. Existing suite pinned to the +facade; add trio-native tests (memory_stream_pair + FrameDecoder for +protocol-level, real popen gateways inside `trio.run` for integration). + +## Invariants (do not regress) + +- No source shipping, ever. Workers import installed execnet+trio; rough + major/minor version check (`_trio_worker._check_version`) warns. +- Wheel-on-demand for `GATEWAY_START_SUB` is a recorded TODO in + `_provision.spawn_request` (currently ships eagerly whenever the sub-spec + has `python=`/`ssh=`). +- Worker teardown: `_trio_worker._run_worker` ends with `os._exit(0)` + because trio's `to_thread` cache uses non-daemon threads. +- `Group.terminate(timeout)` must never hang (bounded even when kill is + stuck). + +## After Phase B (context, don't build now) + +- **C**: worker config axes `loop=thread|main` × `exec=thread|main|task` + (compat: `execmodel=thread` → loop=main+exec=thread; `main_thread_only` → + loop=thread+exec=main); `exec=task` later enables async remote_exec. +- **D**: docs + trio-surface tests, pytest-xdist verification. +- **E** (deferred): `execnet.anyio` — coordinator core on anyio, + `backend="asyncio"` knob for the facade. `FrameDecoder` and the B.4 + channel layers are IO-free by construction, so they port as-is; anyio's + `StapledByteStream` satisfies the `ByteStream` protocol structurally. From f932084db8e0dadd362078a36b74b3c66649096e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 23:34:23 +0200 Subject: [PATCH 26/91] feat: rebuild the sync API as a facade over the async gateway engine The sync coordinator and worker now run their protocol IO on the B.4 async core instead of the parallel ProtocolSession implementation: - AsyncGateway grows per-frame write acknowledgements (outbound queue items carry an optional on_written callback; the writer fails pending frames on shutdown instead of stranding their senders), plus a _finalize hook for subclass shutdown work. - AsyncGroup learns every transport (ssh=, vagrant_ssh=, socket= with installvia=, alongside popen/python=/via=), an overridable gateway factory, per-process reaper tasks, and remoteaddress stamping. - SyncBridgeGateway subclasses AsyncGateway to dispatch messages into the classic sync Message handlers under the receive lock; it keeps the xdist invariant that non-loop-thread sends block until the OS write (120s -> OSError) while loop-thread sends only enqueue, all in one portal-posted FIFO. ProtocolSession is gone; the worker serves on the same bridge. - multi.Group owns a FacadeAsyncGroup task on its TrioHost: makegateway delegates to AsyncGroup.makegateway, and terminate keeps the exit()/join() member contract while the bounded GATEWAY_TERMINATE + grace + kill shutdown runs through AsyncGroup.terminate. - Channel.__del__ posts its close message through the portal without waiting, so GC never blocks on a dying loop. - RawTunnelStream.aclose now feeds EOF to its own reader; previously a terminated via bridge could wait forever on a reader that no longer had a registered raw receiver. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 235 +++++++++++++-- src/execnet/_trio_host.py | 539 +++++++++++++---------------------- src/execnet/_trio_worker.py | 2 +- src/execnet/gateway_base.py | 19 +- src/execnet/multi.py | 61 ++-- testing/test_trio_gateway.py | 2 +- 6 files changed, 460 insertions(+), 398 deletions(-) diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 94f33838..fa4f2ba9 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -40,6 +40,7 @@ from ._exec_source import normalize_exec_source from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate +from .gateway_base import HostNotFound from .gateway_base import Message from .gateway_base import RemoteError from .gateway_base import TimeoutError @@ -411,6 +412,8 @@ class AsyncGateway: """ _error: BaseException | None = None + #: where the peer lives, when the transport knows (ssh host, socket addr) + remoteaddress: str | None = None def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None: self._stream = stream @@ -420,7 +423,9 @@ def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None self._channelfactory = _AsyncChannelFactory(self) self._count = _startcount self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) - self._outbound_send, self._outbound = trio.open_memory_channel[bytes](math.inf) + self._outbound_send, self._outbound = trio.open_memory_channel[ + tuple[bytes, Callable[[BaseException | None], None] | None] + ](math.inf) self._closed = False self._serve_started = False self._writer_done = trio.Event() @@ -526,10 +531,20 @@ async def _serve( finally: self._closed = True self._outbound_send.close() - self._finish_channels() - with trio.CancelScope(shield=True), suppress(Exception): - await self._stream.aclose() - self._done.set() + try: + await self._finalize() + finally: + with trio.CancelScope(shield=True), suppress(Exception): + await self._stream.aclose() + self._done.set() + + async def _finalize(self) -> None: + """Serve-shutdown hook: release the channel layer. + + The sync facade overrides this to close its sync channels and shut + down execution instead. + """ + self._finish_channels() async def _reader(self) -> None: decoder = FrameDecoder() @@ -554,21 +569,43 @@ async def _reader(self) -> None: self._error = exc async def _writer(self) -> None: + error: BaseException | None = None try: - async for frame in self._outbound: - await self._stream.send_all(frame) + async for frame, on_written in self._outbound: + try: + await self._stream.send_all(frame) + except BaseException as exc: + error = exc + if on_written is not None: + on_written(exc) + raise + if on_written is not None: + on_written(None) except (trio.BrokenResourceError, trio.ClosedResourceError, OSError) as exc: self._trace("writer failed", exc) if self._error is None: self._error = exc - self._outbound_send.close() # fail future sends fast else: # Queue closed and drained: signal write-EOF to the peer. with suppress(Exception): await self._stream.send_eof() finally: + # No more writes will happen: fail queued frames instead of + # leaving their senders waiting on acknowledgements. + self._outbound_send.close() + self._fail_pending_writes(error) self._writer_done.set() + def _fail_pending_writes(self, error: BaseException | None) -> None: + exc = error if error is not None else OSError("cannot send (already closed?)") + while True: + try: + _frame, on_written = self._outbound.receive_nowait() + except (trio.WouldBlock, trio.EndOfChannel, trio.ClosedResourceError): + return + if on_written is not None: + on_written(exc) + def _dispatch(self, message: Message) -> None: """Route one message; runs inline on the serve task.""" code = message.msgcode @@ -627,14 +664,32 @@ async def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> No # The queue is unbounded, so this never waits on the peer -- but it # is a real checkpoint and raises once the gateway is closed. try: - await self._outbound_send.send(Message(msgcode, channelid, data).pack()) + await self._outbound_send.send( + (Message(msgcode, channelid, data).pack(), None) + ) + except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + raise OSError("cannot send (already closed?)") from exc + + def enqueue_frame( + self, + frame: bytes, + on_written: Callable[[BaseException | None], None] | None = None, + ) -> None: + """Queue one wire frame (sync, loop thread only). + + ``on_written`` fires exactly once: after the frame reached the OS + write, or with the failure when it never will. Raises OSError when + the outbound side is already closed. + """ + try: + self._outbound_send.send_nowait((frame, on_written)) except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: raise OSError("cannot send (already closed?)") from exc def _send_nowait(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: """Enqueue a frame from a dispatch handler (sync, inline on the loop).""" - with suppress(trio.BrokenResourceError, trio.ClosedResourceError): - self._outbound_send.send_nowait(Message(msgcode, channelid, data).pack()) + with suppress(OSError): + self.enqueue_frame(Message(msgcode, channelid, data).pack()) def _finish_channels(self) -> None: for channel in list(self._channels.values()): @@ -657,18 +712,99 @@ async def serve_gateway( await gateway.aclose() -async def _connect_popen_worker(spec: Any) -> tuple[ByteStream, trio.Process]: - """Spawn a popen worker for ``spec`` and complete the ready handshake.""" - process = await open_popen_process(popen_worker_argv(spec)) +def ssh_transport_args(spec: Any) -> tuple[list[str], bytes]: + """``(ssh argv, stdin preamble)`` for an ssh worker. + + The remote runs the uv-provisioned worker; for a dev coordinator the + preamble carries the wheel bytes that the remote command receives. + """ + from . import _provision + + remote_command, preamble = _provision.ssh_remote_command(spec) + assert spec.ssh is not None + return _provision.ssh_argv(spec.ssh, spec.ssh_config, remote_command), preamble + + +def vagrant_transport_args(spec: Any) -> tuple[list[str], bytes]: + """``(vagrant ssh argv, stdin preamble)`` for a vagrant_ssh worker.""" + from . import _provision + + remote_command, preamble = _provision.ssh_remote_command(spec) + assert spec.vagrant_ssh is not None + argv = _provision.vagrant_ssh_argv( + spec.vagrant_ssh, spec.ssh_config, remote_command + ) + return argv, preamble + + +async def connect_command_worker( + args: list[str], + *, + preamble: bytes = b"", + remoteaddress: str | None = None, +) -> tuple[ByteStream, trio.Process]: + """Spawn ``args`` and complete the worker ready handshake. + + ``preamble`` is streamed to the worker's stdin before the handshake + (a shipped wheel that the remote receives with ``head -c``). With a + ``remoteaddress``, a handshake EOF plus exit code 255 (ssh could not + reach or authenticate the host) becomes :class:`HostNotFound`. + """ + process = await open_popen_process(args) try: stream = staple_process_stream(process) - await read_handshake_ack(stream, "popen") + if preamble: + await stream.send_all(preamble) + await read_handshake_ack(stream, "bootstrap") + except BaseException as exc: + host_not_found = False + with trio.CancelScope(shield=True): + if isinstance(exc, EOFError) and remoteaddress is not None: + with trio.move_on_after(5): + host_not_found = await process.wait() == 255 + with trio.move_on_after(5): + process.kill() + await process.wait() + if host_not_found: + assert remoteaddress is not None + raise HostNotFound(remoteaddress) from None + raise + return stream, process + + +async def connect_socket_worker( + address: tuple[str, int], remoteaddress: str +) -> ByteStream: + """Connect to a running socketserver and complete the ready handshake.""" + try: + stream = await trio.open_tcp_stream(*address) + except OSError as exc: + raise HostNotFound(remoteaddress) from exc + try: + await read_handshake_ack(stream, "socket") except BaseException: with trio.CancelScope(shield=True), trio.move_on_after(5): - process.kill() - await process.wait() + await stream.aclose() raise - return stream, process + return stream + + +async def start_socketserver_via( + gateway: AsyncGateway, bind_host: str = "localhost" +) -> tuple[str, int]: + """Ask ``gateway`` (protocol message) to start a one-shot socket listener. + + Returns the ``(host, port)`` the coordinator should connect to. + """ + channel = gateway.open_channel() + await gateway._send( + Message.GATEWAY_START_SOCKET, channel.id, dumps_internal(bind_host) + ) + realhost, realport = await channel.receive() + await channel.wait_closed() + if not realhost or realhost in ("0.0.0.0", "::"): + realhost = "localhost" + return realhost, int(realport) class AsyncGroup: @@ -722,9 +858,10 @@ async def __aexit__( async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: """Create a gateway for ``spec`` served on the group's nursery. - Popen-style specs (including uv-provisioned ``python=``) and - ``via=`` sub-gateways relayed through a group member are supported - on the async path for now. + All transport types are supported: popen (including uv-provisioned + ``python=``), ``ssh=``, ``vagrant_ssh=``, ``socket=`` (with + ``installvia=``), and ``via=`` sub-gateways relayed through a group + member. """ from .xspec import XSpec @@ -732,25 +869,73 @@ async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: raise RuntimeError(f"{self!r} is not entered") if not isinstance(spec, XSpec): spec = XSpec(spec) - if not (spec.popen or spec.python): - raise ValueError(f"unsupported spec for AsyncGroup: {spec!r}") if spec.execmodel is None: # the sync Group normally stamps this before spawning spec.execmodel = "thread" if spec.id is None: spec.id = "gw%d" % len(self._gateways) process: trio.Process | None = None + remoteaddress: str | None = None if spec.via: stream: ByteStream = await self._open_via_stream(spec) + remote = spec.ssh or spec.vagrant_ssh + if remote: + remoteaddress = f"{remote}[via {spec.via}]" + elif spec.socket: + address, remoteaddress = await self._resolve_socket_address(spec) + stream = await connect_socket_worker(address, remoteaddress) + elif spec.ssh: + args, preamble = ssh_transport_args(spec) + remoteaddress = spec.ssh + stream, process = await connect_command_worker( + args, preamble=preamble, remoteaddress=remoteaddress + ) + elif spec.vagrant_ssh: + args, preamble = vagrant_transport_args(spec) + remoteaddress = spec.vagrant_ssh + stream, process = await connect_command_worker( + args, preamble=preamble, remoteaddress=remoteaddress + ) + elif spec.popen or spec.python: + stream, process = await connect_command_worker(popen_worker_argv(spec)) else: - stream, process = await _connect_popen_worker(spec) - gateway = AsyncGateway(stream, id=spec.id, _startcount=1) + raise ValueError(f"unsupported spec for AsyncGroup: {spec!r}") + gateway = self._make_gateway(stream, spec) + gateway.remoteaddress = remoteaddress await self._nursery.start(gateway._serve) self._gateways.append(gateway) if process is not None: self._processes[gateway] = process + self._nursery.start_soon(self._reap_process, process) return gateway + def _make_gateway(self, stream: ByteStream, spec: Any) -> AsyncGateway: + """Construct the gateway object for a freshly connected stream. + + Overridden by the sync facade to build bridge gateways instead. + """ + return AsyncGateway(stream, id=spec.id, _startcount=1) + + async def _reap_process(self, process: trio.Process) -> None: + # Prompt reaping for workers that exit on their own (no zombies). + with suppress(Exception): + await process.wait() + + async def _resolve_socket_address(self, spec: Any) -> tuple[tuple[str, int], str]: + """``((host, port), remoteaddress)`` for a ``socket=`` spec. + + ``installvia=`` asks that group member to start a one-shot + socketserver first. The sync facade overrides this to talk to its + sync master gateway. + """ + if getattr(spec, "installvia", None): + master = self._gateway_by_id(spec.installvia) + realhost, realport = await start_socketserver_via(master) + return (realhost, realport), "%s:%d" % (realhost, realport) + assert spec.socket is not None + host_str, _, port_str = spec.socket.rpartition(":") + return (host_str, int(port_str)), spec.socket + async def _open_via_stream(self, spec: Any) -> ByteStream: """Ask the ``spec.via`` master to spawn a sub-worker; tunnel over a raw channel (each payload one whole sub-protocol frame).""" diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 1bccc30e..bf0e3a7e 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -23,11 +23,12 @@ import trio from ._trio_gateway import RECEIVE_CHUNK +from ._trio_gateway import AsyncGateway +from ._trio_gateway import AsyncGroup from ._trio_gateway import ByteStream from ._trio_gateway import open_popen_process -from ._trio_gateway import popen_worker_argv from ._trio_gateway import read_handshake_ack -from ._trio_gateway import staple_process_stream +from ._trio_gateway import ssh_transport_args from .gateway_base import ExecModel from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate @@ -40,10 +41,13 @@ if TYPE_CHECKING: from .gateway import Gateway from .gateway_base import BaseGateway + from .multi import Group T = TypeVar("T") -_CLOSE_WRITE = object() + +# Kept name: the ssh argv/preamble builder moved to the async core. +ssh_trio_args = ssh_transport_args _CHANNEL_EOF = object() @@ -106,6 +110,8 @@ def _close_tunnel(self) -> None: return self._closed = True self._gateway._channelfactory.unregister_raw_receiver(self.channelid) + # Unblock our own reader: no more payloads will be routed here. + self._send.send_nowait(_CHANNEL_EOF) with suppress(OSError): self._gateway._send(Message.CHANNEL_CLOSE, self.channelid) @@ -125,19 +131,21 @@ async def adopt_socket(socket_fd: int) -> trio.SocketStream: class SyncIOHandle: - """Sync IO facade for Group.terminate wait/kill/close_write.""" + """Sync IO facade for Gateway.exit close_write and terminate wait/kill.""" remoteaddress: str def __init__( self, execmodel: ExecModel, - session: ProtocolSession, + session: SyncBridgeGateway, *, + process: trio.Process | None = None, remoteaddress: str | None = None, ) -> None: self.execmodel = execmodel self._session = session + self._process = process if remoteaddress is not None: self.remoteaddress = remoteaddress @@ -148,68 +156,134 @@ def write(self, data: bytes) -> None: raise RuntimeError("sync write not supported on Trio IO handle") def close_read(self) -> None: - self._session.request_close_read() + return def close_write(self) -> None: self._session.request_close_write() def wait(self) -> int | None: - return self._session.wait_process() + process = self._process + if process is None: + return None + + async def _wait() -> int | None: + # Always await wait() so the child is reaped (no zombies). + code: int | None = await process.wait() + return code + + try: + return self._session.host.call(_wait) + except Exception: + return process.returncode def kill(self) -> None: - self._session.kill_process() + process = self._process + if process is None: + return + + async def _kill() -> None: + with trio.move_on_after(5): + process.kill() + await process.wait() + + try: + self._session.host.call(_kill) + except Exception as exc: + trace("ERROR killing trio process:", exc) + + +class SyncBridgeGateway(AsyncGateway): + """Async engine serving a sync ``BaseGateway``. + The reader/writer/framing machinery is inherited from + :class:`AsyncGateway`; dispatch is overridden to run the classic sync + ``Message`` handlers (channel queues, callbacks, exec scheduling) under + the gateway's receive lock instead of routing to raw channels. -class ProtocolSession: - """Reader/writer tasks for one gateway connection.""" + Foreign threads send through :meth:`enqueue_message`: every enqueue goes + through the portal so loop callbacks and threads land in one global FIFO, + and non-loop threads wait until the frame hit the OS write (120s -> + OSError) so an abrupt ``os._exit`` cannot drop already-"sent" data. + """ def __init__( self, - gateway: BaseGateway, - io: ByteStream, + stream: ByteStream, *, - process: trio.Process | None = None, + id: str, + sync_gateway: BaseGateway, host: TrioHost, ) -> None: - self.gateway = gateway - self.io = io - self.process = process + super().__init__(stream, id=id) + self.sync_gateway = sync_gateway self.host = host - self._outbound_send: trio.MemorySendChannel[object] - self._outbound_recv: trio.MemoryReceiveChannel[object] - self._outbound_send, self._outbound_recv = trio.open_memory_channel(math.inf) - self._done = threading.Event() - self._process_exitcode: int | None = None - self._process_done = threading.Event() + self._done_sync = threading.Event() self._send_closed = False - self._lock = threading.Lock() + self._send_lock = threading.Lock() - def _post_outbound(self, item: object) -> None: - """Schedule ``item`` onto the outbound channel. + # -- engine hooks (run on the host loop) -- - Goes through the portal even from the host thread so every send — - loop callbacks and foreign threads alike — lands in one global FIFO - order. Raises ``trio.RunFinishedError`` after loop shutdown. - """ - self.host.portal.post(self._outbound_send.send_nowait, item) + def _dispatch(self, message: Message) -> None: + gateway = self.sync_gateway + try: + with gateway._receivelock: + message.received(gateway) + except (GatewayReceivedTerminate, EOFError): + raise + except Exception as exc: + gateway._trace("dispatch failed:", gateway._geterrortext(exc)) + raise EOFError("error dispatching message") from exc + + async def _finalize(self) -> None: + gateway = self.sync_gateway + with self._send_lock: + self._send_closed = True + if gateway._error is None: + gateway._error = self._error + gateway._trace("[trio-bridge] finishing channels") + gateway._channelfactory._finished_receiving() + # Unblock the worker's join() before heavy exec-pool shutdown + # so the primary thread is not waiting on _done while terminate waits + # on the primary thread draining work. + self._done_sync.set() + gateway._trace("[trio-bridge] terminating execution") + # May sleep/SIGINT; keep it off the Trio scheduling thread. + await trio.to_thread.run_sync( + gateway._terminate_execution, abandon_on_cancel=True + ) + + # -- sync session interface (any thread) -- def enqueue_message(self, message: Message) -> None: """Enqueue a frame; wait until written when safe to block. - Non-host threads wait for the OS write so abrupt ``os._exit`` (xdist - crash tests) cannot drop already-"sent" data still in the queue. - - The Trio host thread (receiver callbacks) must not wait — that would - deadlock the writer task on the same event loop. + The Trio host thread (receiver callbacks) must not wait — that + would deadlock the writer task on the same event loop. """ + frame = message.pack() wait = not self.host.portal.is_loop_thread() done = threading.Event() if wait else None errors: list[BaseException] = [] - with self._lock: - if self._send_closed or self._done.is_set(): + + def on_written(error: BaseException | None) -> None: + if error is not None: + errors.append(error) + if done is not None: + done.set() + + def post() -> None: + try: + self.enqueue_frame(frame, on_written) + except OSError as exc: + on_written(exc) + + with self._send_lock: + if self._send_closed: raise OSError("cannot send (already closed?)") try: - self._post_outbound((message.pack(), done, errors)) + # Through the portal even from the host thread so every + # send lands in one global FIFO order. + self.host.portal.post(post) except trio.RunFinishedError: raise OSError("cannot send (already closed?)") from None if done is None: @@ -219,155 +293,33 @@ def enqueue_message(self, message: Message) -> None: if errors: raise OSError("cannot send (already closed?)") from errors[0] + def post_message(self, message: Message) -> None: + """Best-effort non-waiting send (Channel.__del__ during GC).""" + frame = message.pack() + + def post() -> None: + with suppress(OSError): + self.enqueue_frame(frame) + + try: + self.host.portal.post(post) + except trio.RunFinishedError: + raise OSError("cannot send (already closed?)") from None + def request_close_write(self) -> None: - with self._lock: + with self._send_lock: if self._send_closed: return self._send_closed = True with suppress(trio.RunFinishedError): - self._post_outbound(_CLOSE_WRITE) - - def request_close_read(self) -> None: - # Reader observes EOF / cancel; nothing required from callers. - return + # The writer drains queued frames, then signals write-EOF. + self.host.portal.post(self._outbound_send.close) def wait_done(self, timeout: float | None = None) -> bool: - return self._done.wait(timeout) - - def wait_process(self) -> int | None: - if self.process is None: - return None - - async def _wait() -> int | None: - assert self.process is not None - # Always await wait() so the child is reaped (no zombies). - code: int | None = await self.process.wait() - self._process_exitcode = code - self._process_done.set() - return code - - try: - return self.host.call(_wait) - except Exception: - self._process_done.wait() - return self._process_exitcode - - def kill_process(self) -> None: - if self.process is None: - return - - async def _kill() -> None: - assert self.process is not None - with trio.move_on_after(5): - self.process.kill() - self._process_exitcode = await self.process.wait() - self._process_done.set() - - try: - self.host.call(_kill) - except Exception as exc: - trace("ERROR killing trio process:", exc) + return self._done_sync.wait(timeout) def is_alive(self) -> bool: - return not self._done.is_set() - - async def task( - self, task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED - ) -> None: - try: - async with trio.open_nursery() as nursery: - nursery.start_soon(self._writer) - if self.process is not None: - nursery.start_soon(self._supervisor) - task_status.started() - await self._reader() - nursery.cancel_scope.cancel() - finally: - await self._finish() - - async def _reader(self) -> None: - gateway = self.gateway - - def log(*msg: object) -> None: - gateway._trace("[trio-receiver]", *msg) - - log("RECEIVER: starting") - decoder = FrameDecoder() - try: - while True: - data = await self.io.receive_some(RECEIVE_CHUNK) - if not data: - decoder.close() # raises EOFError on a mid-frame EOF - raise EOFError("clean EOF") - for msg in decoder.feed(data): - log("received", msg) - with gateway._receivelock: - msg.received(gateway) - except GatewayReceivedTerminate: - log("GATEWAY_TERMINATE") - except EOFError as exc: - log("EOF without prior gateway termination message") - gateway._error = exc - except Exception as exc: - log(gateway._geterrortext(exc)) - log("finishing receiver") - - async def _writer(self) -> None: - async for item in self._outbound_recv: - if not await self._writer_handle_item(item): - return - - async def _writer_handle_item(self, item: object) -> bool: - """Handle one outbound queue item. Return False when writer should stop.""" - if item is _CLOSE_WRITE: - try: - await self.io.send_eof() - except Exception as exc: - self.gateway._trace("send_eof failed", exc) - return False - assert isinstance(item, tuple) - blob, done, errors = item - assert isinstance(blob, bytes) - try: - await self.io.send_all(blob) - except Exception as exc: - self.gateway._trace("write failed", exc) - errors.append(exc) - with self._lock: - self._send_closed = True - finally: - if done is not None: - done.set() - return True - - async def _supervisor(self) -> None: - assert self.process is not None - try: - self._process_exitcode = await self.process.wait() - finally: - self._process_done.set() - - async def _finish(self) -> None: - gateway = self.gateway - with self._lock: - self._send_closed = True - with suppress(trio.RunFinishedError): - self._post_outbound(_CLOSE_WRITE) - gateway._trace("[trio-receiver] finishing channels") - gateway._channelfactory._finished_receiving() - # Unblock the worker's join() before heavy exec-pool shutdown - # so the primary thread is not waiting on _done while terminate waits - # on the primary thread draining work. - self._done.set() - gateway._trace("[trio-receiver] terminating execution") - # May sleep/SIGINT; keep it off the Trio scheduling thread. - await trio.to_thread.run_sync( - gateway._terminate_execution, abandon_on_cancel=True - ) - try: - await self.io.aclose() - except Exception: - pass + return not self._done_sync.is_set() class TrioHost: @@ -430,16 +382,15 @@ def start_soon(self, async_fn: Callable[..., Any], *args: Any) -> None: self._nursery.start_soon(async_fn, *args) async def start_session( - self, - gateway: BaseGateway, - io: ByteStream, - *, - process: trio.Process | None = None, - ) -> ProtocolSession: + self, gateway: BaseGateway, io: ByteStream + ) -> SyncBridgeGateway: + """Serve ``gateway`` over ``io`` as a task on the root nursery.""" if self._nursery is None: raise RuntimeError("TrioHost nursery is not available") - session = ProtocolSession(gateway, io, process=process, host=self) - await self._nursery.start(session.task) + session = SyncBridgeGateway( + io, id=str(gateway.id), sync_gateway=gateway, host=self + ) + await self._nursery.start(session._serve) return session def stop(self, timeout: float | None = 5.0) -> None: @@ -484,145 +435,79 @@ def kill(self) -> None: return -def _open_trio_gateway( - group: Any, - spec: Any, - args: list[str], - *, - remoteaddress: str | None = None, - preamble: bytes = b"", -) -> Gateway: - """Spawn ``args``, do the worker handshake, and attach a Trio session. - - Shared by the popen and ssh factories; ``args`` already encodes how the - worker is launched (direct module, uv-provisioned, or wrapped in ssh). - ``preamble`` is streamed to the worker's stdin before the handshake (used to - ship a wheel to a remote that receives it with ``head -c``). - """ - import execnet - - from .gateway_base import HostNotFound - - host: TrioHost = group._ensure_trio_host() - - async def _create_and_attach() -> Gateway: - process = await open_popen_process(args) - try: - async_io = staple_process_stream(process) - if preamble: - await async_io.send_all(preamble) - await read_handshake_ack(async_io, "bootstrap") - except EOFError: - with trio.move_on_after(5): - code = await process.wait() - # ssh exits 255 when it cannot reach/authenticate the host. - if remoteaddress is not None and code == 255: - raise HostNotFound(remoteaddress) from None - with trio.move_on_after(5): - process.kill() - await process.wait() - raise - except BaseException: - with trio.move_on_after(5): - process.kill() - await process.wait() - raise - - gw = execnet.Gateway(_TempIO(group.execmodel), spec) - session = await host.start_session(gw, async_io, process=process) - gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) - return gw - - return host.call(_create_and_attach) - - -def makegateway_popen_trio(group: Any, spec: Any) -> Gateway: - """Create a popen Gateway on the Trio IO path. +class FacadeAsyncGroup(AsyncGroup): + """AsyncGroup owning the async side of a sync ``Group``. - Same-interpreter popen launches ``python -m execnet._trio_worker`` directly; - a foreign interpreter (``python=``) is provisioned via ``uv``. Either way the - worker imports execnet + trio; nothing is sent over the wire to bootstrap it. + Runs on the group's :class:`TrioHost`. Gateways come out as + :class:`SyncBridgeGateway` objects bound to freshly built sync + ``Gateway`` facades, and the via / installvia flows go through the sync + master gateway (its dispatch is sync, so async channels cannot be used + on it). """ - return _open_trio_gateway(group, spec, popen_worker_argv(spec)) - - -def ssh_trio_args(spec: Any) -> tuple[list[str], bytes]: - """``(ssh argv, stdin preamble)`` for the Trio ssh path. - - The remote runs the uv-provisioned worker; for a dev coordinator the - preamble carries the wheel bytes that the remote command receives. - """ - from . import _provision - - remote_command, preamble = _provision.ssh_remote_command(spec) - assert spec.ssh is not None - return _provision.ssh_argv(spec.ssh, spec.ssh_config, remote_command), preamble - - -def makegateway_ssh_trio(group: Any, spec: Any) -> Gateway: - """Create an ssh Gateway on the Trio IO path (uv-provisioned worker).""" - args, preamble = ssh_trio_args(spec) - return _open_trio_gateway( - group, spec, args, remoteaddress=spec.ssh, preamble=preamble - ) - - -def makegateway_vagrant_trio(group: Any, spec: Any) -> Gateway: - """Create a ``vagrant ssh``-wrapped Gateway on the Trio IO path.""" - from . import _provision - - remote_command, preamble = _provision.ssh_remote_command(spec) - assert spec.vagrant_ssh is not None - args = _provision.vagrant_ssh_argv( - spec.vagrant_ssh, spec.ssh_config, remote_command - ) - return _open_trio_gateway( - group, spec, args, remoteaddress=spec.vagrant_ssh, preamble=preamble - ) + def __init__(self, group: Group, host: TrioHost) -> None: + super().__init__() + self.group = group + self.host = host + self.shutdown = trio.Event() -def makegateway_socket_trio(group: Any, spec: Any) -> Gateway: - """Connect a Trio TCP stream to a running ``execnet-socketserver``. + def _make_gateway(self, stream: ByteStream, spec: Any) -> AsyncGateway: + import execnet - The server spawns the worker and synthesises its config, so the coordinator - just connects, waits for the worker's ``b"1"`` handshake, and attaches a Trio - session (no local process). - """ - import execnet + sync_gw = execnet.Gateway(_TempIO(self.group.execmodel), spec) + bridge = SyncBridgeGateway( + stream, id=spec.id, sync_gateway=sync_gw, host=self.host + ) + sync_gw._attach_trio_session(bridge) + return bridge - from .gateway_base import HostNotFound + async def _open_via_stream(self, spec: Any) -> ByteStream: + from . import _provision - host: TrioHost = group._ensure_trio_host() - if getattr(spec, "installvia", None): - realhost, realport = start_socketserver_via(group[spec.installvia]) - address = (realhost, realport) - remoteaddress = "%s:%d" % (realhost, realport) - else: + master = self.group[spec.via] + channelid = master._channelfactory.allocate_id() + request = _provision.spawn_request(spec) + # Register the raw receiver before the request goes out so no + # relayed frame can arrive unrouted. + io = RawTunnelStream(master, channelid) + master._send(Message.GATEWAY_START_SUB, channelid, dumps_internal(request)) + await read_handshake_ack(io, "via") + return io + + async def _resolve_socket_address(self, spec: Any) -> tuple[tuple[str, int], str]: + if getattr(spec, "installvia", None): + master = self.group[spec.installvia] + # Blocking sync channel receive on the master: run in a thread + # while this loop keeps dispatching the master's messages. + realhost, realport = await trio.to_thread.run_sync( + start_socketserver_via, master, abandon_on_cancel=True + ) + return (realhost, realport), "%s:%d" % (realhost, realport) assert spec.socket is not None host_str, _, port_str = spec.socket.rpartition(":") - address = (host_str, int(port_str)) - remoteaddress = spec.socket + return (host_str, int(port_str)), spec.socket - async def _create_and_attach() -> Gateway: - try: - stream = await trio.open_tcp_stream(*address) - except OSError as exc: - raise HostNotFound(remoteaddress) from exc - try: - await read_handshake_ack(stream, "socket") - except BaseException: - with trio.move_on_after(5): - await stream.aclose() - raise + async def run(self, task_status: trio.TaskStatus[FacadeAsyncGroup]) -> None: + """Own the group nursery as a host task until :attr:`shutdown`.""" + async with self: + task_status.started(self) + await self.shutdown.wait() - gw = execnet.Gateway(_TempIO(group.execmodel), spec) - session = await host.start_session(gw, stream) - gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) - return gw - return host.call(_create_and_attach) +def makegateway_trio(group: Group, spec: Any) -> Gateway: + """Create a sync-facade Gateway for ``spec`` on the group's Trio host.""" + host: TrioHost = group._ensure_trio_host() + async_group: FacadeAsyncGroup = group._ensure_async_group() + bridge = host.call(async_group.makegateway, spec) + assert isinstance(bridge, SyncBridgeGateway) + gw: Gateway = bridge.sync_gateway # type: ignore[assignment] + gw._io = SyncIOHandle( + group.execmodel, + bridge, + process=async_group._processes.get(bridge), + remoteaddress=bridge.remoteaddress, + ) + return gw _socket_worker_counter = itertools.count() @@ -784,31 +669,3 @@ def handle_start_sub(gateway: BaseGateway, channelid: int, data: bytes) -> None: assert isinstance(request, dict) host: TrioHost = gateway._trio_exec.host # type: ignore[attr-defined] host.start_soon(_start_sub_and_relay, gateway, channelid, request) - - -def makegateway_via_trio(group: Any, spec: Any) -> Gateway: - """Create a ``via`` gateway: a sub-worker spawned by and relayed through the master.""" - import execnet - - from . import _provision - - master = group[spec.via] - host: TrioHost = group._ensure_trio_host() - channelid = master._channelfactory.allocate_id() - request = _provision.spawn_request(spec) - remote = spec.ssh or spec.vagrant_ssh - remoteaddress = f"{remote}[via {spec.via}]" if remote else None - - async def _create_and_attach() -> Gateway: - # Register the raw receiver before the request goes out so no - # relayed frame can arrive unrouted. - io = RawTunnelStream(master, channelid) - master._send(Message.GATEWAY_START_SUB, channelid, dumps_internal(request)) - await read_handshake_ack(io, "via") - gw = execnet.Gateway(_TempIO(group.execmodel), spec) - session = await host.start_session(gw, io) - gw._attach_trio_session(session) - gw._io = SyncIOHandle(group.execmodel, session, remoteaddress=remoteaddress) - return gw - - return host.call(_create_and_attach) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 01b0186c..6a2d04f1 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -224,7 +224,7 @@ def _run_worker(host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel) - """Attach ``io`` as the gateway session and serve until shutdown.""" gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model) - async def _start() -> _trio_host.ProtocolSession: + async def _start() -> _trio_host.SyncBridgeGateway: return await host.start_session(gateway, io) session = host.call(_start) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 22afa46e..2923eaa3 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -706,7 +706,11 @@ def __del__(self) -> None: else: msgcode = Message.CHANNEL_CLOSE with suppress(OSError, ValueError): # ignore problems with sending - self.gateway._send(msgcode, self.id) + # Never wait during GC: post the close best-effort. + send = getattr( + self.gateway, "_send_nonblocking", self.gateway._send + ) + send(msgcode, self.id) def _getremoteerror(self): try: @@ -1130,6 +1134,19 @@ def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: # ValueError might be because the IO is already closed raise OSError("cannot send (already closed?)") from e + def _send_nonblocking(self, msgcode: int, channelid: int = 0) -> None: + """Best-effort send that never waits (used during GC). + + Safe to call from any thread, including while the interpreter or + the IO loop is shutting down. + """ + message = Message(msgcode, channelid) + session = self._trio_session + if session is not None: + session.post_message(message) + return + message.to_io(self._io) + def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: channel.close("execution disallowed") diff --git a/src/execnet/multi.py b/src/execnet/multi.py index c4892995..ca3b69dd 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -12,7 +12,7 @@ from collections.abc import Iterable from collections.abc import Iterator from collections.abc import Sequence -from functools import partial +from contextlib import suppress from threading import Lock from typing import TYPE_CHECKING from typing import Any @@ -51,6 +51,7 @@ def __init__( self._autoidlock = Lock() self._gateways_to_join: list[Gateway] = [] self._trio_host: Any = None + self._async_group: Any = None # we use the same execmodel for all of the Gateway objects # we spawn on our side. Probably we should not allow different # execmodels between different groups but not clear. @@ -69,6 +70,20 @@ def _ensure_trio_host(self) -> Any: self._trio_host.start() return self._trio_host + def _ensure_async_group(self) -> Any: + """The FacadeAsyncGroup owning the async side, running on the host.""" + if self._async_group is None: + from . import _trio_host + + host = self._ensure_trio_host() + + async def _start() -> Any: + async_group = _trio_host.FacadeAsyncGroup(self, host) + return await host._nursery.start(async_group.run) + + self._async_group = host.call(_start) + return self._async_group + @property def execmodel(self) -> ExecModel: return self._execmodel @@ -152,18 +167,9 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: spec.execmodel = self.remote_execmodel.backend from . import _trio_host - if spec.socket: - gw = _trio_host.makegateway_socket_trio(self, spec) - elif spec.via: - gw = _trio_host.makegateway_via_trio(self, spec) - elif spec.ssh: - gw = _trio_host.makegateway_ssh_trio(self, spec) - elif spec.vagrant_ssh: - gw = _trio_host.makegateway_vagrant_trio(self, spec) - elif spec.popen: - gw = _trio_host.makegateway_popen_trio(self, spec) - else: + if not (spec.socket or spec.via or spec.ssh or spec.vagrant_ssh or spec.popen): raise ValueError(f"no gateway type found for {spec._spec!r}") + gw = _trio_host.makegateway_trio(self, spec) gw.spec = spec self._register(gw) if spec.chdir or spec.nice or spec.env: @@ -211,6 +217,10 @@ def _unregister(self, gateway: Gateway) -> None: def _cleanup_atexit(self) -> None: trace(f"=== atexit cleanup {self!r} ===") self.terminate(timeout=1.0) + if self._async_group is not None: + with suppress(Exception): + self._trio_host.call_sync(self._async_group.shutdown.set) + self._async_group = None if self._trio_host is not None: self._trio_host.stop(timeout=1.0) self._trio_host = None @@ -225,7 +235,7 @@ def terminate(self, timeout: float | None = None) -> None: Timeout defaults to None meaning open-ended waiting and no kill attempts. """ - while self: + while self or self._gateways_to_join: vias: set[str] = set() for gw in self: if gw.spec.via: @@ -233,23 +243,16 @@ def terminate(self, timeout: float | None = None) -> None: for gw in self: if gw.id not in vias: gw.exit() - - def join_wait(gw: Gateway) -> None: + if self._async_group is not None: + # Tunneled (via) gateways terminate before their masters, + # each with a GATEWAY_TERMINATE + timeout grace, then kill; + # bounded at roughly twice the timeout (issues #43 / #221). + try: + self._trio_host.call(self._async_group.terminate, timeout) + except Exception as exc: + trace("group terminate error:", exc) + for gw in self._gateways_to_join: gw.join() - gw._io.wait() - - def kill(gw: Gateway) -> None: - trace("Gateways did not come down after timeout: %r" % gw) - gw._io.kill() - - safe_terminate( - self.execmodel, - timeout, - [ - (partial(join_wait, gw), partial(kill, gw)) - for gw in self._gateways_to_join - ], - ) self._gateways_to_join[:] = [] def remote_exec( diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index 630d3e7d..a6f27981 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -400,7 +400,7 @@ def test_unsupported_spec_is_rejected(self) -> None: async def main() -> None: async with AsyncGroup() as group: with pytest.raises(ValueError, match="unsupported spec"): - await group.makegateway("ssh=nowhere.example.invalid") + await group.makegateway("id=notype") trio.run(main) From 80e237ada130a0379dc35681bb661fd67ff9c322 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 24 Jul 2026 23:51:32 +0200 Subject: [PATCH 27/91] docs: record B.5 completion in the phase handoff Co-Authored-By: Claude Fable 5 --- handoff-phase-b-async-core.md | 88 +++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 36 deletions(-) diff --git a/handoff-phase-b-async-core.md b/handoff-phase-b-async-core.md index 8ba250ec..9379ca6d 100644 --- a/handoff-phase-b-async-core.md +++ b/handoff-phase-b-async-core.md @@ -1,7 +1,7 @@ # Handoff: Phase B — invert execnet onto an async-native Trio core -For a fresh session on branch `feat/trio-host-thread-io`. B.1–B.4 are done; -work continues at **B.5**. Plan context lives in session memory +For a fresh session on branch `feat/trio-host-thread-io`. B.1–B.5 are done; +work continues at **B.6**. Plan context lives in session memory (`trio-port-plan`), but everything needed is restated here. Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` @@ -9,28 +9,38 @@ Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` in `testing/test_ssh_local.py` (asyncssh server; system ssh client needed). Known flake: `test_socket_installvia` EOFs rarely under load. -## Where the repo stands (2026-07-24, after commit `8286b77`) +## Where the repo stands (2026-07-24, after B.5) Trio is the only IO path; no source is shipped over the wire (workers run `python -m execnet._trio_worker `, foreign/remote interpreters are uv-provisioned via `_provision.py`, dev coordinators ship a wheel). All transports work: popen, `python=`, ssh, vagrant_ssh, socket, and `via` sub-gateways (`GATEWAY_START_SUB` spawn-request + byte relay on the master). -The architecture is still sync-first: Trio hides behind the sync -`Channel`/`Gateway`/`Group`. Phase B inverts this. + +**The inversion landed (B.5)**: there is one protocol engine — +`AsyncGateway` — and the sync API is a facade over it. `ProtocolSession` +is gone. Coordinator and worker sessions are `SyncBridgeGateway` +(an `AsyncGateway` subclass) whose `_dispatch` runs the classic sync +`Message.received` handlers under `_receivelock`; the sync +`Channel`/`ChannelFactory` state machine in `gateway_base` is unchanged +and remains the semantic layer for blocking users (and the worker's +exec'd code). `multi.Group` owns a `FacadeAsyncGroup` task on its +`TrioHost`; `makegateway` and the bounded process shutdown delegate to +`AsyncGroup`, while `terminate` still calls each member's +`exit()`/`join()` (pinned by `test_basic_group`). File map (src/execnet/): | file | role | |---|---| -| `gateway_base.py` | `Message` wire protocol + **`FrameDecoder`** (sans-IO), serializer (incl. duck-typed `save_AsyncChannel` → CHANNEL opcode), sync `Channel`/`ChannelFactory` (now with **raw-receiver registry**: `register_raw_receiver`/`allocate_id`, verbatim CHANNEL_DATA routing), `BaseGateway`/`WorkerGateway`, `ExecModel`, `WorkerPool`, `HostNotFound` | -| `_trio_gateway.py` | **the async-native core (B.4)**: `ByteStream` Protocol, `RawChannel` (id-routed verbatim byte payloads, sync-Channel close semantics: OSError on closed sends, EOFError/RemoteError on receive, `send_eof` = LAST_MESSAGE/sendonly), `AsyncChannel` (dumps/loads per item, strconfig/RECONFIGURE, timeout via `fail_after`, `wait_closed`, channel-passing), `AsyncGateway` (single serve task: reader dispatches inline — no locks; writer drains unbounded queue, one `send_all` per frame), `AsyncGroup` (async CM owning the nursery; `makegateway` popen/`python=`/`via=`; terminate = tunneled first, then GATEWAY_TERMINATE + timeout + kill, ~2×timeout bound), `RawChannelStream` (RawChannel→ByteStream adapter), `open_popen_gateway`, transport helpers (`staple_*`, `read_handshake_ack`, `popen_worker_argv`) | +| `gateway_base.py` | `Message` wire protocol + **`FrameDecoder`** (sans-IO), serializer (incl. duck-typed `save_AsyncChannel` → CHANNEL opcode), sync `Channel`/`ChannelFactory` (raw-receiver registry: `register_raw_receiver`/`allocate_id`, verbatim CHANNEL_DATA routing), `BaseGateway`/`WorkerGateway` (`_send` via bridge session, `_send_nonblocking` for GC), `ExecModel`, `WorkerPool`, `HostNotFound` | +| `_trio_gateway.py` | **the async-native core**: `ByteStream` Protocol, `RawChannel`, `AsyncChannel`, `AsyncGateway` (single serve task; outbound queue items are `(frame, on_written)` so sync senders can wait for the OS write; writer fails pending frames on shutdown; `_finalize` hook for subclass shutdown), `AsyncGroup` (async CM owning the nursery; **all transports**: popen/`python=`/ssh/vagrant_ssh/socket(+installvia)/`via=`; overridable `_make_gateway`/`_open_via_stream`/`_resolve_socket_address`; per-process reaper tasks; terminate = tunneled first, GATEWAY_TERMINATE + timeout + kill, ~2×timeout bound), `RawChannelStream`, `open_popen_gateway`, transport helpers (`connect_command_worker` incl. ssh-255→HostNotFound, `connect_socket_worker`, `start_socketserver_via` (async), `ssh_transport_args`, `popen_worker_argv`) | | `portal.py` | **`LoopPortal`** (trio-token holder) and **`SyncReceiver`** (loop→plain-thread queue, KI-interruptible `get()`) | -| `_trio_host.py` | sync-facade host: `TrioHost` (loop thread), `ProtocolSession`, `RawTunnelStream` (via tunnel over sync master, raw registry based), gateway factories, `GATEWAY_START_*` handlers (`_start_sub_and_relay` is frame-native: ready byte alone, then one whole frame per CHANNEL_DATA) | -| `_trio_worker.py` | worker entry, `TrioWorkerExec` (FIFO `_pump` admission), `_prepare_protocol_fds` | +| `_trio_host.py` | sync-facade host: `TrioHost` (loop thread; `start_session` returns a bridge), **`SyncBridgeGateway`** (sync dispatch + portal-FIFO `enqueue_message`/`post_message`/`request_close_write`, threading-`Event` `wait_done`), **`FacadeAsyncGroup`** (bridge-building AsyncGroup; via/installvia through the sync master), `makegateway_trio`, `SyncIOHandle` (remoteaddress/wait/kill/close_write), `RawTunnelStream` (via tunnel; `aclose` feeds own reader EOF), `GATEWAY_START_*` handlers (`_start_sub_and_relay` frame-native) | +| `_trio_worker.py` | worker entry, `TrioWorkerExec` (FIFO `_pump` admission), `_prepare_protocol_fds`; serves on a `SyncBridgeGateway` | | `gateway.py` | sync coordinator `Gateway` (remote_exec, exit, rinfo) | | `_exec_source.py` | remote_exec source normalization shared by sync + async coordinators | -| `multi.py` | sync `Group`, `MultiChannel`, `safe_terminate` (WorkerPool-based) | +| `multi.py` | sync `Group` facade (`_ensure_async_group`, terminate via `AsyncGroup`), `MultiChannel`, `safe_terminate` (WorkerPool-based, kept for tests) | | `_provision.py` | uv provisioning, wheel build/ship/materialize, argv builders | Async-core tests live in `testing/test_trio_gateway.py` (memory_stream_pair @@ -87,32 +97,38 @@ Leftovers deliberately deferred: RemoteError ("unsupported message") — async exec is Phase C (`exec=task`). -### B.5 Rebuild the sync API as a facade - -Public surface stays byte-for-byte: `Group`, `makegateway` spec strings, -`Gateway.remote_exec/exit/reconfigure/remote_status/hasreceiver`, `Channel` -send/receive/setcallback/makefile/waitclose/reconfigure, `MultiChannel`, -`RSync`, `execnet.dumps/loads/dump/load`, `HostNotFound`, `TimeoutError`, -`RemoteError`, `DataFormatError`. Existing tests keep running against the -facade unchanged — that is the acceptance bar. - -Known tricky spots: - -- `Channel.__del__` sends CHANNEL_CLOSE/LAST_MESSAGE during GC — must go - through the portal as a non-waiting post, and tolerate interpreter - shutdown (today: `suppress(OSError, ValueError)` + `Message is not None` - guard). -- KeyboardInterrupt lands on the main thread while blocked inside a portal - call — decide propagation (today the sync side blocks in - `threading.Event.wait`, which KI can interrupt). -- `group.execmodel` / `set_execmodel` / `spec.execmodel` stay as deprecated - shims. `main_thread_only` must keep working throughout (pytest-xdist runs - GUI-bound code on the worker main thread via - `TrioWorkerExec.integrate_as_primary_thread`). -- Retire `ExecModel` internals and `WorkerPool` at the END of the phase - (WorkerPool is still the worker exec duck-type target for STATUS and is - used by `multi.safe_terminate` + `testing/test_threadpool.py` + - `testing/conftest.py`'s `pool` fixture). +### B.5 Rebuild the sync API as a facade — DONE (commit `f932084`) + +Public surface stayed byte-for-byte; the whole sync suite passes +unchanged (508 passed). What landed, and the decisions taken: + +- One engine: `SyncBridgeGateway(AsyncGateway)` serves both coordinator + and worker; `ProtocolSession` deleted. Sync dispatch still runs the + `gateway_base` Message handlers, so the sync `Channel`/`ChannelFactory` + semantics (queues, callbacks, ENDMARKER, weakref drop) are untouched — + and remain what the worker's exec'd code sees. +- Send invariant kept: `enqueue_message` posts every frame through the + portal (one global FIFO); non-loop threads wait on the frame's + `on_written` ack (120s → OSError), the loop thread only enqueues. +- `Channel.__del__` now goes through `_send_nonblocking` → + `SyncBridgeGateway.post_message` (portal post, never waits; falls back + to `_send` for duck-typed test gateways). +- KI decision: blocking data paths (send-ack, receive, waitclose, join) + wait on `threading.Event`/`queue.Queue` — KI-interruptible as before. + Management ops (makegateway, terminate) run inside `portal.run` + (`trio.from_thread.run`) where KI is deferred; accepted for now. +- `Group.terminate` still calls member `exit()`/`join()` (contract pinned + by `test_basic_group`), with process wait/kill delegated to + `AsyncGroup.terminate` (tunneled-first, ~2×timeout bound). +- Gotcha fixed on the way: a stream-closing `aclose` on the via tunnel + (`RawTunnelStream`) must feed EOF to its own reader, else the bridge's + serve task never finishes and terminate hangs. +- `ChannelFile`/`makefile`: kept the existing str-based `gateway_base` + classes as-is for backward compat (they only use channel send/receive). +- NOT yet retired: `ExecModel` internals and `WorkerPool` (still the + worker STATUS duck-type shape and used by `multi.safe_terminate`, + `testing/test_threadpool.py`, conftest's `pool` fixture). Do this at + the end of the phase (B.6 or later) if the tests pinning them move. ### B.6 Namespace split From 4f84335d5a465c8971376663ef47e0cc3f6ba648 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 16:53:36 +0200 Subject: [PATCH 28/91] feat: split the public surface into execnet.sync / execnet.trio / execnet.portal execnet.sync re-exports the blocking facade; the top-level execnet.* names now alias into it. execnet.trio exposes the async-native core (AsyncGroup/AsyncGateway/AsyncChannel, raw channels, stream helpers, shared serialization + errors) for use inside the caller's own trio.run. execnet.portal (LoopPortal/SyncReceiver) is the communicating layer both build on. The trio and portal modules load lazily via module __getattr__ so plain `import execnet` still does not import the trio event loop. Co-Authored-By: Claude Fable 5 --- src/execnet/__init__.py | 59 +++++++++++++++++--------- src/execnet/portal.py | 2 + src/execnet/sync.py | 49 ++++++++++++++++++++++ src/execnet/trio.py | 61 +++++++++++++++++++++++++++ testing/test_namespaces.py | 85 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 237 insertions(+), 19 deletions(-) create mode 100644 src/execnet/sync.py create mode 100644 src/execnet/trio.py create mode 100644 testing/test_namespaces.py diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index cb91cb66..2108f4b2 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -4,29 +4,40 @@ pure python lib for connecting to local and remote Python Interpreters. +Three public namespaces: + +* :mod:`execnet.sync` — the blocking API; the top-level ``execnet.*`` + names below are aliases into it. +* :mod:`execnet.trio` — the trio-native API, awaited inside your own + ``trio.run``. +* :mod:`execnet.portal` — cross-thread / cross-loop communication + primitives shared by both. + (c) 2012, Holger Krekel and others """ +from typing import Any + from ._version import version as __version__ -from .gateway import Gateway -from .gateway_base import Channel -from .gateway_base import DataFormatError -from .gateway_base import DumpError -from .gateway_base import HostNotFound -from .gateway_base import LoadError -from .gateway_base import RemoteError -from .gateway_base import TimeoutError -from .gateway_base import dump -from .gateway_base import dumps -from .gateway_base import load -from .gateway_base import loads -from .multi import Group -from .multi import MultiChannel -from .multi import default_group -from .multi import makegateway -from .multi import set_execmodel -from .rsync import RSync -from .xspec import XSpec +from .sync import Channel +from .sync import DataFormatError +from .sync import DumpError +from .sync import Gateway +from .sync import Group +from .sync import HostNotFound +from .sync import LoadError +from .sync import MultiChannel +from .sync import RemoteError +from .sync import RSync +from .sync import TimeoutError +from .sync import XSpec +from .sync import default_group +from .sync import dump +from .sync import dumps +from .sync import load +from .sync import loads +from .sync import makegateway +from .sync import set_execmodel __all__ = [ "Channel", @@ -50,3 +61,13 @@ "makegateway", "set_execmodel", ] + + +def __getattr__(name: str) -> Any: + # Lazy namespace modules: keep ``import execnet`` from loading the + # trio event loop machinery until it is actually used. + if name in ("trio", "portal"): + import importlib + + return importlib.import_module(f".{name}", __name__) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/execnet/portal.py b/src/execnet/portal.py index 9804ca45..a204cc5a 100644 --- a/src/execnet/portal.py +++ b/src/execnet/portal.py @@ -24,6 +24,8 @@ import trio +__all__ = ["LoopPortal", "SyncReceiver"] + T = TypeVar("T") diff --git a/src/execnet/sync.py b/src/execnet/sync.py new file mode 100644 index 00000000..0d3c835f --- /dev/null +++ b/src/execnet/sync.py @@ -0,0 +1,49 @@ +"""The blocking execnet API. + +A facade over the trio-native core in :mod:`execnet.trio`: gateways run +their protocol IO on a dedicated Trio host thread while this surface +blocks the calling thread. The top-level ``execnet.*`` names are aliases +into this module. +""" + +from .gateway import Gateway +from .gateway_base import Channel +from .gateway_base import DataFormatError +from .gateway_base import DumpError +from .gateway_base import HostNotFound +from .gateway_base import LoadError +from .gateway_base import RemoteError +from .gateway_base import TimeoutError +from .gateway_base import dump +from .gateway_base import dumps +from .gateway_base import load +from .gateway_base import loads +from .multi import Group +from .multi import MultiChannel +from .multi import default_group +from .multi import makegateway +from .multi import set_execmodel +from .rsync import RSync +from .xspec import XSpec + +__all__ = [ + "Channel", + "DataFormatError", + "DumpError", + "Gateway", + "Group", + "HostNotFound", + "LoadError", + "MultiChannel", + "RSync", + "RemoteError", + "TimeoutError", + "XSpec", + "default_group", + "dump", + "dumps", + "load", + "loads", + "makegateway", + "set_execmodel", +] diff --git a/src/execnet/trio.py b/src/execnet/trio.py new file mode 100644 index 00000000..df0ab493 --- /dev/null +++ b/src/execnet/trio.py @@ -0,0 +1,61 @@ +"""The trio-native execnet API. + +Everything here is awaited directly inside your own ``trio.run`` — no +host thread, no blocking calls:: + + import trio + import execnet.trio + + async def main(): + async with execnet.trio.AsyncGroup() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(6 * 7)") + print(await channel.receive()) + + trio.run(main) + +The serialization helpers and error types are shared with the blocking +API in :mod:`execnet.sync`. +""" + +from ._trio_gateway import AsyncChannel +from ._trio_gateway import AsyncGateway +from ._trio_gateway import AsyncGroup +from ._trio_gateway import ByteStream +from ._trio_gateway import RawChannel +from ._trio_gateway import RawChannelStream +from ._trio_gateway import open_popen_gateway +from ._trio_gateway import serve_gateway +from .gateway_base import DataFormatError +from .gateway_base import DumpError +from .gateway_base import HostNotFound +from .gateway_base import LoadError +from .gateway_base import RemoteError +from .gateway_base import TimeoutError +from .gateway_base import dump +from .gateway_base import dumps +from .gateway_base import load +from .gateway_base import loads +from .xspec import XSpec + +__all__ = [ + "AsyncChannel", + "AsyncGateway", + "AsyncGroup", + "ByteStream", + "DataFormatError", + "DumpError", + "HostNotFound", + "LoadError", + "RawChannel", + "RawChannelStream", + "RemoteError", + "TimeoutError", + "XSpec", + "dump", + "dumps", + "load", + "loads", + "open_popen_gateway", + "serve_gateway", +] diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py new file mode 100644 index 00000000..dd9b3777 --- /dev/null +++ b/testing/test_namespaces.py @@ -0,0 +1,85 @@ +"""The three public namespaces: execnet.sync / execnet.trio / execnet.portal. + +Top-level ``execnet.*`` is an alias surface over ``execnet.sync``; the +trio namespace exposes the async-native core; the portal namespace the +cross-thread primitives. +""" + +from __future__ import annotations + +import subprocess +import sys + +import execnet +import execnet.portal +import execnet.sync +import execnet.trio + + +def test_top_level_names_alias_execnet_sync() -> None: + for name in execnet.sync.__all__: + assert getattr(execnet, name) is getattr(execnet.sync, name), name + + +def test_top_level_all_matches_sync_surface() -> None: + assert set(execnet.__all__) == set(execnet.sync.__all__) | {"__version__"} + + +def test_trio_namespace_exposes_async_core() -> None: + from execnet import _trio_gateway + + assert execnet.trio.AsyncGroup is _trio_gateway.AsyncGroup + assert execnet.trio.AsyncGateway is _trio_gateway.AsyncGateway + assert execnet.trio.AsyncChannel is _trio_gateway.AsyncChannel + assert execnet.trio.open_popen_gateway is _trio_gateway.open_popen_gateway + # serialization + errors are shared with the sync surface + assert execnet.trio.RemoteError is execnet.RemoteError + assert execnet.trio.dumps is execnet.dumps + + +def test_portal_namespace() -> None: + assert execnet.portal.__all__ == ["LoopPortal", "SyncReceiver"] + assert execnet.portal.LoopPortal is not None + + +def test_lazy_submodule_attribute_access() -> None: + # After ``import execnet`` alone, execnet.trio / execnet.portal are + # reachable as attributes (PEP 562) without having been imported. + out = subprocess.run( + [ + sys.executable, + "-c", + "import execnet; print(execnet.trio.AsyncGroup.__name__);" + " print(execnet.portal.LoopPortal.__name__)", + ], + capture_output=True, + text=True, + check=True, + ) + assert out.stdout.split() == ["AsyncGroup", "LoopPortal"] + + +def test_import_execnet_does_not_import_trio() -> None: + # The blocking surface must stay importable without loading the trio + # event loop machinery (it loads lazily on first gateway / namespace + # use). + out = subprocess.run( + [ + sys.executable, + "-c", + "import sys, execnet; print('trio' in sys.modules)", + ], + capture_output=True, + text=True, + check=True, + ) + assert out.stdout.strip() == "False" + + +def test_unknown_attribute_raises() -> None: + try: + execnet.does_not_exist + except AttributeError as exc: + assert "does_not_exist" in str(exc) + else: + raise AssertionError("expected AttributeError") From 33decae92ad469da1e6568aa9b20c2c9f002590c Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 16:55:44 +0200 Subject: [PATCH 29/91] docs: record phase B completion in the handoff Co-Authored-By: Claude Fable 5 --- handoff-phase-b-async-core.md | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/handoff-phase-b-async-core.md b/handoff-phase-b-async-core.md index 9379ca6d..ab078740 100644 --- a/handoff-phase-b-async-core.md +++ b/handoff-phase-b-async-core.md @@ -1,8 +1,8 @@ # Handoff: Phase B — invert execnet onto an async-native Trio core -For a fresh session on branch `feat/trio-host-thread-io`. B.1–B.5 are done; -work continues at **B.6**. Plan context lives in session memory -(`trio-port-plan`), but everything needed is restated here. +For a fresh session on branch `feat/trio-host-thread-io`. **Phase B is +complete (B.1–B.6)**; work continues at **Phase C**. Plan context lives in +session memory (`trio-port-plan`), but everything needed is restated here. Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` (never grep-filter pre-commit output). ssh paths have a real local harness @@ -130,12 +130,28 @@ unchanged (508 passed). What landed, and the decisions taken: `testing/test_threadpool.py`, conftest's `pool` fixture). Do this at the end of the phase (B.6 or later) if the tests pinning them move. -### B.6 Namespace split - -Introduce `execnet.sync` / `execnet.trio` / `execnet.portal`; top-level -`execnet.*` aliases into `execnet.sync`. Existing suite pinned to the -facade; add trio-native tests (memory_stream_pair + FrameDecoder for -protocol-level, real popen gateways inside `trio.run` for integration). +### B.6 Namespace split — DONE (commit `4f84335`) + +`execnet.sync` (blocking facade re-exports; top-level `execnet.*` aliases +into it), `execnet.trio` (AsyncGroup/AsyncGateway/AsyncChannel, raw +channels, stream helpers, shared serialization + errors), and +`execnet.portal` (LoopPortal/SyncReceiver). `trio`/`portal` load lazily +via module `__getattr__`, so `import execnet` still does not import the +trio event loop (pinned by `testing/test_namespaces.py`). Trio-native +protocol/integration tests already live in `testing/test_trio_gateway.py` +(memory_stream_pair + real popen gateways inside `trio.run`). + +### End-of-phase leftovers (deferred into C/D) + +- Retire `ExecModel` internals and `WorkerPool`: still pinned by + `multi.safe_terminate`, `testing/test_threadpool.py`, and conftest's + `pool` fixture, and `get_execmodel`/backend names feed Phase C's + `execmodel=` compat mapping — retire once C lands the new worker + config axes. +- ChannelFile / makefile over RawChannel (async makefile) — untouched; + the sync str-based wrappers stay for backward compat. +- rsync data plane / wheel shipping over raw channels — later. +- Async remote_exec on the worker (`exec=task`) — Phase C. ## Invariants (do not regress) From 51b9053a7663e088532e7b791017a24a7ec92666 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 20:38:41 +0200 Subject: [PATCH 30/91] fix: attach the bridge session before it can serve messages The worker created its SyncBridgeGateway inside host.call and only attached it to the sync gateway afterwards on the main thread. A coordinator message arriving in that window (STATUS, CHANNEL_EXEC as the first action on a fresh gateway) dispatched into a gateway whose _trio_session was still None, so the reply fell through to the sync IO stub and killed the session -- the whole test suite under pytest -n 12 showed this as freshly created gateways being dead on arrival. The race predates the B.5 facade (ProtocolSession had the same window). SyncBridgeGateway now attaches itself in __init__, before its serve task can dispatch anything, and the redundant attach calls at the two construction sites are gone. Also drop the apipkg-era unknown-attribute test from the namespace tests -- unknown attributes are plain Python module behaviour now. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_host.py | 8 +++++--- src/execnet/_trio_worker.py | 5 +++-- testing/test_namespaces.py | 9 --------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index bf0e3a7e..4d05690d 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -220,6 +220,10 @@ def __init__( self._done_sync = threading.Event() self._send_closed = False self._send_lock = threading.Lock() + # Attach before any serving can happen: the first inbound message + # may need to reply through gateway._send, which must already + # route to this session (not the sync IO stub). + sync_gateway._attach_trio_session(self) # -- engine hooks (run on the host loop) -- @@ -455,11 +459,9 @@ def _make_gateway(self, stream: ByteStream, spec: Any) -> AsyncGateway: import execnet sync_gw = execnet.Gateway(_TempIO(self.group.execmodel), spec) - bridge = SyncBridgeGateway( + return SyncBridgeGateway( stream, id=spec.id, sync_gateway=sync_gw, host=self.host ) - sync_gw._attach_trio_session(bridge) - return bridge async def _open_via_stream(self, spec: Any) -> ByteStream: from . import _provision diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 6a2d04f1..bfaeea43 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -225,10 +225,11 @@ def _run_worker(host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel) - gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model) async def _start() -> _trio_host.SyncBridgeGateway: + # The bridge attaches itself to the gateway before serving starts, + # so inbound messages can reply through gateway._send right away. return await host.start_session(gateway, io) - session = host.call(_start) - gateway._attach_trio_session(session) + host.call(_start) try: if main_thread_only: diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index dd9b3777..5f44f8a9 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -74,12 +74,3 @@ def test_import_execnet_does_not_import_trio() -> None: check=True, ) assert out.stdout.strip() == "False" - - -def test_unknown_attribute_raises() -> None: - try: - execnet.does_not_exist - except AttributeError as exc: - assert "does_not_exist" in str(exc) - else: - raise AssertionError("expected AttributeError") From a69b844d9e5f0d80546147218f18da56d59bb7cf Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 20:40:39 +0200 Subject: [PATCH 31/91] chore: add pytest-xdist to the dev group, ignore claude local settings Co-Authored-By: Claude Fable 5 --- .gitignore | 1 + pyproject.toml | 3 +++ uv.lock | 19 ++++++++++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d734b32f..9f676ac3 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ include/ .vagrant.d/ .config/ .local/ +.claude/settings.local.json diff --git a/pyproject.toml b/pyproject.toml index f419c31d..4e94c938 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,9 @@ testing = [ ] [dependency-groups] +dev = [ + "pytest-xdist>=3.8.0", +] testing = [ "execnet[testing]", ] diff --git a/uv.lock b/uv.lock index e4657f7b..15185f2a 100644 --- a/uv.lock +++ b/uv.lock @@ -372,7 +372,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -398,6 +398,9 @@ testing = [ ] [package.dev-dependencies] +dev = [ + { name = "pytest-xdist" }, +] testing = [ { name = "execnet", extra = ["testing"] }, ] @@ -416,6 +419,7 @@ requires-dist = [ provides-extras = ["testing"] [package.metadata.requires-dev] +dev = [{ name = "pytest-xdist", specifier = ">=3.8.0" }] testing = [{ name = "execnet", extras = ["testing"] }] [[package]] @@ -819,6 +823,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-discovery" version = "1.5.0" From 72b0eefd392f7f97f4693751091c574bf10b918e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 20:46:41 +0200 Subject: [PATCH 32/91] =?UTF-8?q?docs:=20hand=20off=20phase=20C/D=20?= =?UTF-8?q?=E2=80=94=20worker=20axes=20gap=20analysis,=20supersede=20the?= =?UTF-8?q?=20B=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- handoff-phase-b-async-core.md | 11 ++- handoff-phase-c-worker-axes.md | 139 +++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 handoff-phase-c-worker-axes.md diff --git a/handoff-phase-b-async-core.md b/handoff-phase-b-async-core.md index ab078740..d3a69ff6 100644 --- a/handoff-phase-b-async-core.md +++ b/handoff-phase-b-async-core.md @@ -1,8 +1,8 @@ # Handoff: Phase B — invert execnet onto an async-native Trio core -For a fresh session on branch `feat/trio-host-thread-io`. **Phase B is -complete (B.1–B.6)**; work continues at **Phase C**. Plan context lives in -session memory (`trio-port-plan`), but everything needed is restated here. +**SUPERSEDED by `handoff-phase-c-worker-axes.md`** — kept as the Phase B +record. **Phase B is complete (B.1–B.6)** on branch +`feat/trio-host-thread-io`; work continues at **Phase C**. Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` (never grep-filter pre-commit output). ssh paths have a real local harness @@ -123,6 +123,11 @@ unchanged (508 passed). What landed, and the decisions taken: - Gotcha fixed on the way: a stream-closing `aclose` on the via tunnel (`RawTunnelStream`) must feed EOF to its own reader, else the bridge's serve task never finishes and terminate hangs. +- Post-B fix (`51b9053`): the session-attach startup race — the bridge + must attach itself to the sync gateway in `__init__`, before serving + starts, or a first-message reply goes through the IO stub and kills + the fresh gateway (predated B.5; surfaced by `pytest -n 12`). The + suite now runs xdist-clean. - `ChannelFile`/`makefile`: kept the existing str-based `gateway_base` classes as-is for backward compat (they only use channel send/receive). - NOT yet retired: `ExecModel` internals and `WorkerPool` (still the diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md new file mode 100644 index 00000000..ced2d895 --- /dev/null +++ b/handoff-phase-c-worker-axes.md @@ -0,0 +1,139 @@ +# Handoff: Phase C — worker config axes (`loop=` × `exec=`) + Phase D + +For a fresh session on branch `feat/trio-host-thread-io`. Phase B (the +inversion onto the async-native Trio core) is **complete**; this doc +supersedes `handoff-phase-b-async-core.md` (kept for the B record). +Plan context lives in session memory (`trio-port-plan`), but everything +needed is restated here. + +Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` +(never grep-filter pre-commit output). The suite is xdist-clean: `uv run +pytest testing/ -n 12` passes (~7s) since the session-attach race fix +(`51b9053`) — keep it that way. ssh paths have a real local harness in +`testing/test_ssh_local.py` (asyncssh server; system ssh client needed). +Known flake: `test_socket_installvia` EOFs rarely under load. + +## Where the repo stands (2026-07-25, after `a69b844`) + +One protocol engine: `AsyncGateway` (`_trio_gateway.py`). The sync API +is a facade over it — coordinator and worker sessions are +`SyncBridgeGateway` (an `AsyncGateway` subclass in `_trio_host.py`) whose +`_dispatch` runs the classic sync `Message.received` handlers under +`_receivelock`; `SyncBridgeGateway.__init__` attaches itself to the sync +gateway *before* serving starts (the fix for the startup race where a +STATUS/CHANNEL_EXEC arriving first replied through the IO stub and killed +the session). `multi.Group` owns a `FacadeAsyncGroup` task on its +`TrioHost`; makegateway and bounded process shutdown delegate to +`AsyncGroup`, while `terminate` keeps the member `exit()`/`join()` +contract (pinned by `test_basic_group`). `AsyncGroup` supports every +transport (popen, `python=`, ssh, vagrant_ssh, socket with installvia, +`via=`). Public namespaces: `execnet.sync` (top-level `execnet.*` +aliases into it), `execnet.trio`, `execnet.portal` (trio/portal lazy via +module `__getattr__`; `import execnet` must not import trio — pinned by +`testing/test_namespaces.py`). + +No source is shipped over the wire: workers run +`python -m execnet._trio_worker `; foreign/remote +interpreters are uv-provisioned via `_provision.py`; dev coordinators +ship a wheel. + +File map (src/execnet/): + +| file | role | +|---|---| +| `gateway_base.py` | `Message` wire protocol + sans-IO `FrameDecoder`, serializer (CHANNEL opcode incl. duck-typed `save_AsyncChannel`), sync `Channel`/`ChannelFactory` (raw-receiver registry), `BaseGateway`/`WorkerGateway` (`_send` via bridge, `_send_nonblocking` for GC), `ExecModel`, `WorkerPool`, `HostNotFound` | +| `_trio_gateway.py` | async core: `ByteStream` Protocol, `RawChannel`/`AsyncChannel`, `AsyncGateway` (outbound queue of `(frame, on_written)`; `_finalize` hook), `AsyncGroup` (all transports, overridable `_make_gateway`/`_open_via_stream`/`_resolve_socket_address`, reapers, bounded terminate), stream/argv helpers | +| `_trio_host.py` | `TrioHost` (loop thread), `SyncBridgeGateway`, `FacadeAsyncGroup`, `makegateway_trio`, `SyncIOHandle`, `RawTunnelStream` (via tunnel; `aclose` feeds own reader EOF), `GATEWAY_START_*` handlers | +| `_trio_worker.py` | worker entry (`_main`/`serve_popen_trio`/`serve_socket_trio`), `TrioWorkerExec` (FIFO `_pump` admission; `integrate_as_primary_thread`), `_prepare_protocol_fds`, `_check_version` | +| `gateway.py` / `multi.py` | sync `Gateway` / sync `Group` facade, `MultiChannel`, `safe_terminate` (WorkerPool-based, kept for tests) | +| `sync.py` / `trio.py` / `portal.py` | the three public namespaces | +| `_exec_source.py`, `_provision.py`, `xspec.py`, `rsync.py` | source normalization, uv provisioning + argv builders, spec parsing, rsync | + +## Semantics that MUST survive (xdist depends on them) + +- Sends from non-loop threads block until the frame hit the OS write + (120s → OSError) so abrupt `os._exit` cannot drop "sent" data; loop + thread sends only enqueue. All sends go through one portal-posted FIFO. +- After close: `OSError("cannot send (already closed?)")`; + `trio.RunFinishedError` maps to the same. +- exec admission order = message arrival order (`TrioWorkerExec._pump`; + `test_main_thread_only_concurrent_remote_exec_deadlock` guards this — + trio shuffles its run batch, so never rely on task-spawn order). +- Channel callbacks run on the receiver (loop) thread — keep for now + (open decision: revisit in D). +- `Group.terminate(timeout)` never hangs (~2×timeout bound, issues + #43/#221). +- Sync blocking waits (send-ack, receive, waitclose, join) stay on + `threading.Event`/`queue.Queue` so KeyboardInterrupt can interrupt + them; `portal.run` (KI-deferred) is only for management ops. + +## Phase C — what is missing (nothing of C exists yet) + +Target axes: `loop=thread|main` × `exec=thread|main|task`. +Compat mapping: `execmodel=thread` → `loop=main` + `exec=thread`; +`execmodel=main_thread_only` → `loop=thread` + `exec=main`. + +1. **Spec surface**: `xspec.py` only knows `execmodel=`. Add `loop=` / + `exec=` keys, combination validation, and the compat mapping. Same + for the worker CLI config JSON (`_provision.worker_cli_arg` still + ships `{"id", "execmodel", "coordinator_version"}`). +2. **`loop=main` does not exist**: the worker always starts `TrioHost` + on a dedicated thread (`serve_popen_trio`), main thread parked in + `gateway.join()` or integrated as exec primary. The compat mapping + means plain `execmodel=thread` workers should end up with `trio.run` + on the main thread — a structural change to `_run_worker`, not a + flag. +3. **`exec=main`** is today's `main_thread_only` machinery + (`integrate_as_primary_thread` + `_executetask_complete` deadlock + guard) — keep behavior, re-home under the new naming. +4. **`exec=task` (async remote_exec)**: `CHANNEL_EXEC` on a pure + `AsyncGateway` is still rejected ("unsupported message" in + `_trio_gateway._dispatch`). Needs a task-based exec scheduler + handing exec'd code an `AsyncChannel`, preserving FIFO admission, + plus explicit cancellation semantics. +5. **STATUS / `remote_status()`** reports `execmodel`; must speak the + new axes compatibly. +6. **Retire `ExecModel`/`WorkerPool`** (deferred from B): replace + internals with plain `threading`/`queue`; deprecated shims for + `group.execmodel`/`set_execmodel`/`spec.execmodel`. Pinned today by + `multi.safe_terminate`, `testing/test_threadpool.py`, conftest's + `pool` fixture, `Channel._items` (`execmodel.queue`), and the worker + STATUS duck-type — gated on the compat mapping landing first. +7. **Wheel-on-demand for `GATEWAY_START_SUB`**: recorded TODO in + `_provision.spawn_request` (currently ships eagerly whenever the + sub-spec has `python=`/`ssh=`). + +## Phase D — what is missing + +1. **Docs are entirely stale**: `doc/` still describes source + bootstrapping and execmodels; nothing on uv provisioning, the + no-source-shipping/version-compat policy, the three namespaces, or + `execnet.trio`/`execnet.portal` APIs. `doc/implnotes.rst` references + pre-inversion internals. Changelog entries for the whole branch. +2. **Trio-surface test gaps**: async ssh/socket/vagrant transport tests + (the asyncssh harness only exercises the sync facade; both share + `connect_command_worker`, so a direct `AsyncGroup` ssh test is + cheap), cancellation mid-`remote_exec`/`receive`, `send_eof` and + reconfigure against real workers, B.5 engine paths (write-ack + failure, `enqueue_message` after close). +3. **pytest-xdist verification**: run pytest-xdist's own suite against + this branch plus real `-n` smoke runs (crash/endmarker tests, + `main_thread_only` GUI case). Our own suite running `-n 12` green is + necessary but not sufficient. +4. **Close the open decision**: channel callbacks on the loop thread — + keep or move. + +Recommended order: C.1+C.2 first (spec axes + loop placement) since the +compat mapping unblocks the ExecModel retirement, then `exec=task`; run +the xdist verification (D.3) mid-C as an early canary rather than only +at the end. + +## Invariants (do not regress) + +- No source shipping, ever. Workers import installed execnet+trio; + rough major/minor version check (`_trio_worker._check_version`) warns. +- Worker teardown: `_trio_worker._run_worker` ends with `os._exit(0)` + because trio's `to_thread` cache uses non-daemon threads. +- `import execnet` must not import the trio event loop. +- Keep async-core idioms anyio-portable (neutral `ByteStream`, sans-IO + `FrameDecoder`) — `execnet.anyio` is deferred Phase E. From 2742eb0aee4eb45fe5f39cfd27847146681ab6ed Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 25 Jul 2026 21:05:43 +0200 Subject: [PATCH 33/91] fix: drain asyncssh redirect cleanup in the local ssh harness asyncssh stores its redirect cleanup coroutines (Queue.join et al) un-awaited and only runs them inside process.wait_closed(); the server handler never called it, so every ssh test leaked coroutines that pytest's unraisable hook surfaced as RuntimeWarnings during GC -- visible as stray noise in xdist runs. Await wait_closed() in the handler and drop the filterwarnings mark, which never covered the GC-time warning anyway. Co-Authored-By: Claude Fable 5 --- testing/test_ssh_local.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py index 086ce877..443b5da2 100644 --- a/testing/test_ssh_local.py +++ b/testing/test_ssh_local.py @@ -20,14 +20,9 @@ import execnet -pytestmark = [ - pytest.mark.skipif( - shutil.which("ssh") is None, reason="system ssh client required" - ), - # asyncssh leaves an un-awaited internal Queue.join coroutine on shutdown, - # surfaced by pytest's unraisable-exception hook during GC; harmless here. - pytest.mark.filterwarnings("ignore:coroutine 'Queue.join' was never awaited"), -] +pytestmark = pytest.mark.skipif( + shutil.which("ssh") is None, reason="system ssh client required" +) # Committed, intentionally-insecure test keys (see sshkeys/README.md). SSHKEYS = Path(__file__).parent / "sshkeys" @@ -58,6 +53,10 @@ async def _handle(self, process: asyncssh.SSHServerProcess) -> None: ) await process.redirect(stdin=proc.stdin, stdout=proc.stdout, stderr=proc.stderr) process.exit(await proc.wait()) + # Drain asyncssh's redirect cleanup coroutines (Queue.join et al): + # they are stored un-awaited and only run inside wait_closed(); + # skipping this leaks them and GC prints RuntimeWarnings. + await process.wait_closed() async def _serve(self) -> None: self._server = await asyncssh.listen( From 0d123bdbd2476dcee227eb150c22ae8f30c97ef2 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 08:50:57 +0200 Subject: [PATCH 34/91] docs: record PR #422 and the xfail audit in the phase C handoff Co-Authored-By: Claude Fable 5 --- handoff-phase-c-worker-axes.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index ced2d895..351f6c16 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -6,6 +6,10 @@ supersedes `handoff-phase-b-async-core.md` (kept for the B record). Plan context lives in session memory (`trio-port-plan`), but everything needed is restated here. +The rework is up as **draft PR pytest-dev/execnet#422** (branch pushed +to the `origin` fork). Keep it updated as C/D land; undraft once the +docs overhaul (D.1) at least covers migration notes. + Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` (never grep-filter pre-commit output). The suite is xdist-clean: `uv run pytest testing/ -n 12` passes (~7s) since the session-attach race fix @@ -122,6 +126,16 @@ Compat mapping: `execmodel=thread` → `loop=main` + `exec=thread`; necessary but not sufficient. 4. **Close the open decision**: channel callbacks on the loop thread — keep or move. +5. **xfail markers audit (done 2026-07-26, keep as-is)**: the 11 + consistent XPASSes were investigated — trio's single-loop dispatch + + FIFO admission makes them pass reliably when idle, but under + sustained load `test_gateway_status_busy` (numexecuting race: + `_track_start` runs in a separately scheduled task) and + `test_popen_stderr_tracing` (capfd race) still fail, so the + `flakytest` marks stay. `test_safe_terminate2`'s xpass is CPython + dummy-thread accounting, unrelated to trio. To retire the status + marks for real: retry-poll for `numexecuting == 2` like the tests + already do for `== 0`. Recommended order: C.1+C.2 first (spec axes + loop placement) since the compat mapping unblocks the ExecModel retirement, then `exec=task`; run From 2c7841678694539da379266560313049e4de4522 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 10:02:11 +0200 Subject: [PATCH 35/91] feat: introduce the boundary kit (Wakener/Mailbox/OneShot) in execnet.portal P1 of the host-boundary rethink (handoff-boundary-protocol-rethink.md): loop->consumer egress unifies on a pluggable thread-safe Wakener with two carriers on top -- Mailbox (item streams, generalizing the KI-safe SyncReceiver drain pattern with get_nowait/timeouts) and OneShot (single results). SyncReceiver is replaced by Mailbox; the SyncBridgeGateway write-ack and _done_sync move off ad-hoc threading.Events onto OneShot. Pure refactor: no behavior change, suite stays xdist-clean. Co-Authored-By: Claude Fable 5 --- handoff-boundary-protocol-rethink.md | 129 +++++++++++++++++++++ src/execnet/_trio_host.py | 40 ++++--- src/execnet/_trio_worker.py | 8 +- src/execnet/portal.py | 166 +++++++++++++++++++++++---- testing/test_namespaces.py | 8 +- testing/test_portal.py | 76 ++++++++++++ 6 files changed, 379 insertions(+), 48 deletions(-) create mode 100644 handoff-boundary-protocol-rethink.md create mode 100644 testing/test_portal.py diff --git a/handoff-boundary-protocol-rethink.md b/handoff-boundary-protocol-rethink.md new file mode 100644 index 00000000..52336693 --- /dev/null +++ b/handoff-boundary-protocol-rethink.md @@ -0,0 +1,129 @@ +# Handoff: host boundary protocol rethink (P1–P5, precedes Phase C proper) + +Decided with Ronny 2026-07-26. This supersedes the *ordering* of +`handoff-phase-c-worker-axes.md` (still valid for C content): the boundary +rethink lands first, then C's `loop=`/`exec=` work continues on the +smaller core. The wire protocol (Message framing) is untouched — this is +about how things cross between the trio host loop and its consumers. + +## Rationale + +Consumer→loop is already universal: `LoopPortal.post` rides +`TrioToken.run_sync_soon`, thread-safe from any foreign context (plain +thread, gevent greenlet, asyncio callback, another trio loop). One +ingress, strict FIFO — unchanged. + +Loop→consumer is hardwired to threading primitives (`Channel._items` +execmodel queue, `threading.Event` for waitclose/join/write-acks, +`SyncReceiver`), which is the only reason `ExecModel` still exists. The +fix: the loop never blocks and never knows who listens — it fires a +thread-safe wakeup callable the consumer supplied. Every event loop has +exactly one such primitive: + +| consumer | wakeup primitive | +|---|---| +| plain thread | `threading.Event.set` | +| asyncio | `loop.call_soon_threadsafe` | +| gevent | `hub.loop.async_()` watcher — `send()` is gevent's one documented thread-safe op | +| foreign trio loop | `TrioToken.run_sync_soon` | + +## The boundary kit (grows in `execnet.portal`) + +1. **`Wakener`** — protocol, one thread-safe method `notify()`. The + entire integration surface for a new event loop (eventlet, Qt, …) + is implementing this; no more ExecModel. +2. **`Mailbox[T]`** — unbounded deque + Wakener. `put()` from the loop + (append + notify, never blocks the loop); `get(timeout)` blocking for + sync backends — generalizes the KI-safe `SyncReceiver` drain pattern + (Event.wait + drain, re-check after clear) — and `await`-able for + asyncio/trio wakeners. Replaces `Channel._items`, `SyncReceiver`, + `MultiChannel`'s queue. +3. **`OneShot[T]`** — single-result future on the same wakener. + Replaces the per-send write-ack `threading.Event`; adds + `portal.start(async_fn) -> OneShot` so asyncio callers await + management ops instead of blocking a thread in `portal.run`. + +## Channel unification (decided: full rebase) + +`AsyncGateway._dispatch` becomes the ONLY message router. `RawChannel` +grows a consumer hook: a bound facade diverts payloads into a `Mailbox` +instead of the internal memory channel (callbacks: invoked on the loop +thread, preserving current semantics; moving them later is just posting +via the wakener). + +Sync `Channel` becomes a genuine facade over (raw id, mailbox, portal): + +- `receive()` = `mailbox.get(timeout)` + `loads_internal` at the call + site — deserialization moves off the loop thread; `DataFormatError` + now surfaces at `receive()`. +- `send()` = portal-posted frame + `OneShot` write-ack (non-loop threads + keep block-until-written, 120s → OSError, KI-safe). +- `waitclose()`/`join()` = closed-events on the wakener. +- `__del__` best-effort close keeps `portal.post` (post_message path). +- Unknown inbound ids still auto-create (`_channel_for`). + +Deleted outright: `_receivelock`, `ChannelFactory` dispatch paths +(`_local_receive`/`_local_close`), `SyncBridgeGateway._dispatch`, the +`Message.received` handler-table use, the duplicate close state machine. +`CHANNEL_EXEC`/`GATEWAY_START_*` become per-gateway hooks on the async +core. `TrioWorkerExec` FIFO pump survives unchanged; exec'd code gets +facade channels. + +## Execmodels: retired as machinery, preserved as presets + +execmodel names become presets over three orthogonal axes (spec + worker +CLI config; decided: **both** coordinator and worker side): + +| axis | values | meaning | +|---|---|---| +| `loop` | `main` \| `thread` | where the trio host runs (Phase C) | +| `exec` | `thread` \| `main` \| `task` | where remote_exec code runs (Phase C) | +| `wait` | `thread` \| `gevent` \| `asyncio` | which Wakener blocking facade waits park on (new) | + +Presets: `execmodel=thread` → `loop=main, exec=thread, wait=thread`; +`main_thread_only` → `loop=thread, exec=main, wait=thread`; `gevent` +returns as `loop=thread, exec=thread, wait=gevent` (greenlet channel +waits park the greenlet, not the hub; sends already greenlet-safe via +the portal). eventlet stays dead (third parties can implement Wakener). + +Dies: `ExecModel` ABC, `WorkerPool`, `Reply` (`multi.safe_terminate` +moves to plain threads or delegates to `FacadeAsyncGroup.terminate`). +Stays as deprecated preset-mappers: `get_execmodel`, `set_execmodel`, +`group.execmodel`, `spec.execmodel` (xdist compat). + +## asyncio (decided: land now) + +`execnet.aio` namespace mirroring `execnet.trio`'s surface; every op is +portal ingress + `OneShot`/`Mailbox` on an `AsyncioWakener`. Full +asyncio-native API over the trio host loop, all transports, no anyio +port. Phase E (anyio-native core) becomes optional purity/perf work. + +## Staging + +1. **P1 — boundary kit**: Wakener/Mailbox/OneShot + ThreadWakener; port + SyncReceiver, write-acks, `_done_sync` onto it. Pure refactor. +2. **P2 — channel unification** (big one): sync Channel onto + RawChannel+Mailbox, delete classic dispatch. Straight to the rebase, + no intermediate primitive-swap (suite pins semantics either way). + Canary: `uv run pytest testing/ -n 12` + xdist suite mid-step. +3. **P3 — retirement**: ExecModel/WorkerPool out, preset shims in, + `wait=` joins the C.1 spec axes + worker CLI config. +4. **P4 — `execnet.aio`**. +5. **P5 — gevent wakener** + opt-in CI job (gevent in a dev extra). + +Then Phase C `loop=main` / `exec=task` on the smaller core, and Phase D +docs cover the three namespaces + `execnet.aio` + the axes/presets. + +## Invariants (unchanged, re-mapped) + +- Non-loop-thread sends block until the frame hit the OS write (OneShot + write-ack; 120s → OSError); loop-thread sends only enqueue; all sends + one portal FIFO. +- After close: `OSError("cannot send (already closed?)")`. +- exec admission order = message arrival order (TrioWorkerExec._pump). +- Channel callbacks run on the loop thread (revisit-later stands; the + kit makes moving them cheap). +- `Group.terminate(timeout)` never hangs (~2×timeout). +- Sync blocking waits stay KeyboardInterrupt-interruptible on the main + thread (thread Wakener keeps the Event.wait + drain pattern). +- No source shipping; `import execnet` must not import trio. diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 4d05690d..68b278d7 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -37,6 +37,7 @@ from .gateway_base import loads_internal from .gateway_base import trace from .portal import LoopPortal +from .portal import OneShot if TYPE_CHECKING: from .gateway import Gateway @@ -217,7 +218,7 @@ def __init__( super().__init__(stream, id=id) self.sync_gateway = sync_gateway self.host = host - self._done_sync = threading.Event() + self._done_sync: OneShot[None] = OneShot() self._send_closed = False self._send_lock = threading.Lock() # Attach before any serving can happen: the first inbound message @@ -249,7 +250,7 @@ async def _finalize(self) -> None: # Unblock the worker's join() before heavy exec-pool shutdown # so the primary thread is not waiting on _done while terminate waits # on the primary thread draining work. - self._done_sync.set() + self._done_sync.set(None) gateway._trace("[trio-bridge] terminating execution") # May sleep/SIGINT; keep it off the Trio scheduling thread. await trio.to_thread.run_sync( @@ -266,20 +267,17 @@ def enqueue_message(self, message: Message) -> None: """ frame = message.pack() wait = not self.host.portal.is_loop_thread() - done = threading.Event() if wait else None - errors: list[BaseException] = [] - - def on_written(error: BaseException | None) -> None: - if error is not None: - errors.append(error) - if done is not None: - done.set() + # The ack carries the write failure as a value (never raised into + # the OneShot) so a KeyboardInterrupt in wait() stays distinguishable + # from a stream error. + ack: OneShot[BaseException | None] | None = OneShot() if wait else None def post() -> None: try: - self.enqueue_frame(frame, on_written) + self.enqueue_frame(frame, ack.set if ack is not None else None) except OSError as exc: - on_written(exc) + if ack is not None: + ack.set(exc) with self._send_lock: if self._send_closed: @@ -290,12 +288,14 @@ def post() -> None: self.host.portal.post(post) except trio.RunFinishedError: raise OSError("cannot send (already closed?)") from None - if done is None: + if ack is None: return - if not done.wait(timeout=120.0): - raise OSError("cannot send (write timed out)") - if errors: - raise OSError("cannot send (already closed?)") from errors[0] + try: + error = ack.wait(timeout=120.0) + except TimeoutError: + raise OSError("cannot send (write timed out)") from None + if error is not None: + raise OSError("cannot send (already closed?)") from error def post_message(self, message: Message) -> None: """Best-effort non-waiting send (Channel.__del__ during GC).""" @@ -320,7 +320,11 @@ def request_close_write(self) -> None: self.host.portal.post(self._outbound_send.close) def wait_done(self, timeout: float | None = None) -> bool: - return self._done_sync.wait(timeout) + try: + self._done_sync.wait(timeout) + except TimeoutError: + return False + return True def is_alive(self) -> bool: return not self._done_sync.is_set() diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index bfaeea43..aa3e68d9 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -16,7 +16,7 @@ from .gateway_base import get_execmodel from .gateway_base import loads_internal from .gateway_base import trace -from .portal import SyncReceiver +from .portal import Mailbox if TYPE_CHECKING: from . import _trio_host @@ -48,9 +48,9 @@ def __init__( self._shutting_down = False self._idle = threading.Event() self._idle.set() - self._primary: SyncReceiver[ - tuple[Channel, ExecItem, threading.Event] | None - ] = SyncReceiver() + self._primary: Mailbox[tuple[Channel, ExecItem, threading.Event] | None] = ( + Mailbox() + ) # Exec requests flow through a single pump task so admission happens # strictly in message-arrival order (trio task scheduling order is # deliberately unordered, so per-request tasks would race for the diff --git a/src/execnet/portal.py b/src/execnet/portal.py index a204cc5a..223fdd97 100644 --- a/src/execnet/portal.py +++ b/src/execnet/portal.py @@ -1,30 +1,39 @@ -"""Cross-thread / cross-loop communication primitives. +"""Cross-thread / cross-loop communication primitives — the boundary kit. A :class:`LoopPortal` is a handle to a running trio loop that foreign -threads use to run functions on the loop or push work into it. Because -each direction only needs the *receiving* loop's token, two trio loops in -two threads can communicate by holding each other's portal (the sync -facade's host loop and a user loop; the worker loop and the process main -thread). - -:class:`SyncReceiver` covers the opposite direction: a plain thread -(typically the process main thread) receiving items produced on a loop, -in a way that stays interruptible by KeyboardInterrupt. +threads use to run functions on the loop or push work into it (the +consumer -> loop direction). Because each direction only needs the +*receiving* loop's token, two trio loops in two threads can communicate +by holding each other's portal. + +The loop -> consumer direction never blocks the loop and never knows who +is listening: the loop fires a :class:`Wakener`, a single thread-safe +``notify()`` supplied by the consumer. Implementing that one method is +the entire integration surface for an event loop backend (threads here; +asyncio/gevent wakeners come with their facades). On top of it sit the +two carriers: + +* :class:`Mailbox` -- an item stream (channel payloads, exec requests), +* :class:`OneShot` -- a single result (write acknowledgements, call + results a consumer wants to await instead of block on). """ from __future__ import annotations import queue import threading +import time from collections.abc import Awaitable from collections.abc import Callable from typing import Any from typing import Generic +from typing import Protocol from typing import TypeVar +from typing import cast import trio -__all__ = ["LoopPortal", "SyncReceiver"] +__all__ = ["LoopPortal", "Mailbox", "OneShot", "ThreadWakener", "Wakener"] T = TypeVar("T") @@ -64,35 +73,142 @@ def post(self, sync_fn: Callable[..., object], *args: Any) -> None: self._token.run_sync_soon(sync_fn, *args) -class SyncReceiver(Generic[T]): - """Receive items on a plain thread, KeyboardInterrupt-friendly. +class Wakener(Protocol): + """Thread-safe consumer wakeup fired by the loop. - ``queue.SimpleQueue.get`` parks in a C-level lock acquire that shields - KeyboardInterrupt on the main thread; ``threading.Event.wait`` does not. - So producers put into an unbounded queue and set a wake event, and - :meth:`get` waits on the event and drains the queue. + ``notify()`` must never block and must be safe from any thread; it is + the only thing the loop side ever calls. The blocking mailbox/oneshot + waits additionally need :meth:`wait`/:meth:`clear` executed in the + consumer's own context (a thread here, a greenlet for a gevent + wakener); loop-native consumers such as asyncio wait on their own + side of ``notify`` instead. + """ + + def notify(self) -> None: ... + + def wait(self, timeout: float | None = None) -> bool: ... + + def clear(self) -> None: ... + + +class ThreadWakener: + """Plain-thread wakener on a ``threading.Event``. + + ``threading.Event.wait`` stays interruptible by KeyboardInterrupt on + the main thread (a C-level ``queue.SimpleQueue.get`` does not), which + is why the carriers wait on the wakener and drain a queue instead of + blocking in the queue itself. """ def __init__(self) -> None: + self._event = threading.Event() + + def notify(self) -> None: + self._event.set() + + def wait(self, timeout: float | None = None) -> bool: + return self._event.wait(timeout) + + def clear(self) -> None: + self._event.clear() + + +class Mailbox(Generic[T]): + """Loop -> consumer item stream: an unbounded queue plus a wakener. + + :meth:`put` never blocks and is safe from any thread (including the + loop thread). :meth:`get` blocks in the consumer's context via the + wakener; after draining to empty it clears and re-checks so a ``put`` + racing the clear cannot be lost. Multiple consumers are allowed. + """ + + def __init__(self, wakener: Wakener | None = None) -> None: self._items: queue.SimpleQueue[T] = queue.SimpleQueue() - self._wake = threading.Event() + self._wakener = ThreadWakener() if wakener is None else wakener def put(self, item: T) -> None: """Thread-safe; usable from a loop thread (never blocks).""" self._items.put(item) - self._wake.set() + self._wakener.notify() + + def get_nowait(self) -> T: + """Return the next item or raise ``queue.Empty``.""" + return self._items.get_nowait() - def get(self) -> T: - """Block until an item is available (interruptible on main thread).""" + def get(self, timeout: float | None = None) -> T: + """Block until an item is available; TimeoutError after ``timeout``.""" + deadline = None if timeout is None else time.monotonic() + timeout while True: - self._wake.wait() + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if remaining <= 0 or not self._wakener.wait(remaining): + # Final non-blocking check: a put may have raced the + # timeout (its notify landing after our last wait). + try: + return self._items.get_nowait() + except queue.Empty: + raise TimeoutError( + "no item after %r seconds" % timeout + ) from None try: return self._items.get_nowait() except queue.Empty: - # Re-check after clearing so a put between get_nowait and - # clear cannot be lost. - self._wake.clear() + # Empty: clear, then re-check so a put between get_nowait + # and clear cannot be lost. + self._wakener.clear() try: return self._items.get_nowait() except queue.Empty: continue + + +class OneShot(Generic[T]): + """A single result crossing loop -> consumer exactly once. + + The loop side calls :meth:`set` (or :meth:`set_error`) at most once; + the consumer blocks in :meth:`wait`, which returns the value, + re-raises the stored error, or raises ``TimeoutError``. + """ + + _NOTSET = object() + + def __init__(self, wakener: Wakener | None = None) -> None: + self._wakener = ThreadWakener() if wakener is None else wakener + self._value: Any = self._NOTSET + self._error: BaseException | None = None + self._done = False + + def is_set(self) -> bool: + return self._done + + def set(self, value: T) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + assert not self._done, "OneShot already resolved" + self._value = value + self._done = True + self._wakener.notify() + + def set_error(self, error: BaseException) -> None: + """Resolve with an error that :meth:`wait` will re-raise.""" + assert not self._done, "OneShot already resolved" + self._error = error + self._done = True + self._wakener.notify() + + def wait(self, timeout: float | None = None) -> T: + """Block until resolved; TimeoutError after ``timeout`` seconds.""" + deadline = None if timeout is None else time.monotonic() + timeout + while not self._done: + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if (remaining <= 0 or not self._wakener.wait(remaining)) and ( + not self._done + ): + raise TimeoutError("not resolved after %r seconds" % timeout) + if self._error is not None: + raise self._error + return cast("T", self._value) diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index 5f44f8a9..a8bfef0c 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -38,7 +38,13 @@ def test_trio_namespace_exposes_async_core() -> None: def test_portal_namespace() -> None: - assert execnet.portal.__all__ == ["LoopPortal", "SyncReceiver"] + assert execnet.portal.__all__ == [ + "LoopPortal", + "Mailbox", + "OneShot", + "ThreadWakener", + "Wakener", + ] assert execnet.portal.LoopPortal is not None diff --git a/testing/test_portal.py b/testing/test_portal.py new file mode 100644 index 00000000..1a3759af --- /dev/null +++ b/testing/test_portal.py @@ -0,0 +1,76 @@ +"""The boundary kit: Mailbox and OneShot on the thread wakener.""" + +from __future__ import annotations + +import queue +import threading + +import pytest + +from execnet.portal import Mailbox +from execnet.portal import OneShot + + +class TestMailbox: + def test_put_get_fifo(self) -> None: + box: Mailbox[int] = Mailbox() + for n in range(3): + box.put(n) + assert [box.get(), box.get(), box.get()] == [0, 1, 2] + + def test_get_nowait_empty(self) -> None: + box: Mailbox[int] = Mailbox() + with pytest.raises(queue.Empty): + box.get_nowait() + + def test_get_timeout(self) -> None: + box: Mailbox[int] = Mailbox() + with pytest.raises(TimeoutError): + box.get(timeout=0.01) + + def test_get_blocks_until_put_from_other_thread(self) -> None: + box: Mailbox[str] = Mailbox() + threading.Timer(0.05, box.put, args=["item"]).start() + assert box.get(timeout=5.0) == "item" + + def test_get_after_stale_wakeup(self) -> None: + # A consumed notify leaves the wakener set; the drain pattern must + # still block (and then receive) rather than spin or miss items. + box: Mailbox[int] = Mailbox() + box.put(1) + assert box.get() == 1 + threading.Timer(0.05, box.put, args=[2]).start() + assert box.get(timeout=5.0) == 2 + + +class TestOneShot: + def test_set_then_wait(self) -> None: + shot: OneShot[int] = OneShot() + shot.set(42) + assert shot.is_set() + assert shot.wait() == 42 + # a resolved OneShot stays readable + assert shot.wait(timeout=0.01) == 42 + + def test_wait_timeout(self) -> None: + shot: OneShot[int] = OneShot() + with pytest.raises(TimeoutError): + shot.wait(timeout=0.01) + assert not shot.is_set() + + def test_set_error_reraises(self) -> None: + shot: OneShot[None] = OneShot() + shot.set_error(RuntimeError("boom")) + with pytest.raises(RuntimeError, match="boom"): + shot.wait() + + def test_wait_blocks_until_set_from_other_thread(self) -> None: + shot: OneShot[str] = OneShot() + threading.Timer(0.05, shot.set, args=["done"]).start() + assert shot.wait(timeout=5.0) == "done" + + def test_single_resolution_asserted(self) -> None: + shot: OneShot[int] = OneShot() + shot.set(1) + with pytest.raises(AssertionError): + shot.set(2) From 2a40a559c68170f53751d3eef4116b659c70e583 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:05:02 +0200 Subject: [PATCH 36/91] feat!: rebase the sync Channel onto the async core (single dispatch) P2 of the boundary rethink: AsyncGateway._dispatch is now the only Message router. RawChannel grows a consumer hook (set_consumer) that diverts inbound payloads/closes to a bound facade and replays anything that arrived first; the sync Channel becomes a genuine facade -- a Mailbox fed from the loop (deserialization at the receive() call site), callbacks invoked on the loop thread with the switch-over running there too, sends unchanged through gateway._send. Deleted: the classic Message.received handler table, ChannelFactory's dispatch paths (_local_receive, callback/raw-receiver registries) and BaseGateway._receivelock; RawTunnelStream (the via tunnel now uses RawChannelStream over the session's raw channel on both master and worker relay sides). ChannelFactory remains as registry + id allocator, keeping callback channels strongly alive while registered. An unconsumed remotely-closed RawChannel now stays registered until a consumer claims it: with call-site deserialization a passed channel can bind after its data AND close already arrived, and previously the close forgot the raw channel so the late binder found a fresh empty one and lost the payloads (latent race for async open_channel(id) too). The boundary kit moved to the trio-free execnet._boundary so gateway_base can use Mailbox without `import execnet` importing trio; the standalone-source tests inline it / import the package instead. Co-Authored-By: Claude Fable 5 --- src/execnet/_boundary.py | 161 +++++++++++++++ src/execnet/_trio_gateway.py | 48 ++++- src/execnet/_trio_host.py | 206 +++++++++++-------- src/execnet/_trio_worker.py | 2 +- src/execnet/gateway_base.py | 387 ++++++++++++++++------------------- src/execnet/portal.py | 152 +------------- testing/test_basics.py | 28 ++- testing/test_serializer.py | 9 +- 8 files changed, 542 insertions(+), 451 deletions(-) create mode 100644 src/execnet/_boundary.py diff --git a/src/execnet/_boundary.py b/src/execnet/_boundary.py new file mode 100644 index 00000000..f80f6998 --- /dev/null +++ b/src/execnet/_boundary.py @@ -0,0 +1,161 @@ +"""Trio-free half of the boundary kit: Wakener, Mailbox, OneShot. + +Importable without loading any event loop (``import execnet`` must not +import trio); ``execnet.portal`` re-exports these next to LoopPortal. +""" + +from __future__ import annotations + +import queue +import threading +import time +from typing import Any +from typing import Generic +from typing import Protocol +from typing import TypeVar +from typing import cast + +__all__ = ["Mailbox", "OneShot", "ThreadWakener", "Wakener"] + +T = TypeVar("T") + + +class Wakener(Protocol): + """Thread-safe consumer wakeup fired by the loop. + + ``notify()`` must never block and must be safe from any thread; it is + the only thing the loop side ever calls. The blocking mailbox/oneshot + waits additionally need :meth:`wait`/:meth:`clear` executed in the + consumer's own context (a thread here, a greenlet for a gevent + wakener); loop-native consumers such as asyncio wait on their own + side of ``notify`` instead. + """ + + def notify(self) -> None: ... + + def wait(self, timeout: float | None = None) -> bool: ... + + def clear(self) -> None: ... + + +class ThreadWakener: + """Plain-thread wakener on a ``threading.Event``. + + ``threading.Event.wait`` stays interruptible by KeyboardInterrupt on + the main thread (a C-level ``queue.SimpleQueue.get`` does not), which + is why the carriers wait on the wakener and drain a queue instead of + blocking in the queue itself. + """ + + def __init__(self) -> None: + self._event = threading.Event() + + def notify(self) -> None: + self._event.set() + + def wait(self, timeout: float | None = None) -> bool: + return self._event.wait(timeout) + + def clear(self) -> None: + self._event.clear() + + +class Mailbox(Generic[T]): + """Loop -> consumer item stream: an unbounded queue plus a wakener. + + :meth:`put` never blocks and is safe from any thread (including the + loop thread). :meth:`get` blocks in the consumer's context via the + wakener; after draining to empty it clears and re-checks so a ``put`` + racing the clear cannot be lost. Multiple consumers are allowed. + """ + + def __init__(self, wakener: Wakener | None = None) -> None: + self._items: queue.SimpleQueue[T] = queue.SimpleQueue() + self._wakener = ThreadWakener() if wakener is None else wakener + + def put(self, item: T) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + self._items.put(item) + self._wakener.notify() + + def get_nowait(self) -> T: + """Return the next item or raise ``queue.Empty``.""" + return self._items.get_nowait() + + def get(self, timeout: float | None = None) -> T: + """Block until an item is available; TimeoutError after ``timeout``.""" + deadline = None if timeout is None else time.monotonic() + timeout + while True: + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if remaining <= 0 or not self._wakener.wait(remaining): + # Final non-blocking check: a put may have raced the + # timeout (its notify landing after our last wait). + try: + return self._items.get_nowait() + except queue.Empty: + raise TimeoutError( + "no item after %r seconds" % timeout + ) from None + try: + return self._items.get_nowait() + except queue.Empty: + # Empty: clear, then re-check so a put between get_nowait + # and clear cannot be lost. + self._wakener.clear() + try: + return self._items.get_nowait() + except queue.Empty: + continue + + +class OneShot(Generic[T]): + """A single result crossing loop -> consumer exactly once. + + The loop side calls :meth:`set` (or :meth:`set_error`) at most once; + the consumer blocks in :meth:`wait`, which returns the value, + re-raises the stored error, or raises ``TimeoutError``. + """ + + _NOTSET = object() + + def __init__(self, wakener: Wakener | None = None) -> None: + self._wakener = ThreadWakener() if wakener is None else wakener + self._value: Any = self._NOTSET + self._error: BaseException | None = None + self._done = False + + def is_set(self) -> bool: + return self._done + + def set(self, value: T) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + assert not self._done, "OneShot already resolved" + self._value = value + self._done = True + self._wakener.notify() + + def set_error(self, error: BaseException) -> None: + """Resolve with an error that :meth:`wait` will re-raise.""" + assert not self._done, "OneShot already resolved" + self._error = error + self._done = True + self._wakener.notify() + + def wait(self, timeout: float | None = None) -> T: + """Block until resolved; TimeoutError after ``timeout`` seconds.""" + deadline = None if timeout is None else time.monotonic() + timeout + while not self._done: + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if (remaining <= 0 or not self._wakener.wait(remaining)) and ( + not self._done + ): + raise TimeoutError("not resolved after %r seconds" % timeout) + if self._error is not None: + raise self._error + return cast("T", self._value) diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index fa4f2ba9..fc364772 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -159,6 +159,12 @@ def __init__(self, gateway: AsyncGateway, id: int) -> None: self._receive_closed = trio.Event() # no more payloads will arrive self._remote_error: RemoteError | None = None self._payload_send, self._payloads = trio.open_memory_channel[bytes](math.inf) + # Diversion hooks for a bound facade (sync channel): when set, + # inbound payloads/closes route out of the loop instead of + # buffering for receive_bytes. + self._consumer_payload: Callable[[bytes], None] | None = None + self._consumer_close: Callable[[RemoteError | None, bool], None] | None = None + self._pending_close: tuple[RemoteError | None, bool] | None = None def __repr__(self) -> str: state = "closed" if self._closed else "open" @@ -228,9 +234,40 @@ def _pending_error(self) -> BaseException: or EOFError(f"raw channel {self.id} closed") ) + def set_consumer( + self, + on_payload: Callable[[bytes], None], + on_close: Callable[[RemoteError | None, bool], None], + ) -> None: + """Divert inbound payloads and the close to callbacks (loop thread). + + Already-buffered payloads flush to ``on_payload`` first, and a close + that arrived before binding is replayed to ``on_close`` -- so a + consumer bound late (facade channels bind via a portal post) sees + the exact inbound order. + """ + self._consumer_payload = on_payload + self._consumer_close = on_close + while True: + try: + data = self._payloads.receive_nowait() + except (trio.WouldBlock, trio.EndOfChannel, trio.ClosedResourceError): + break + on_payload(data) + if self._pending_close is not None: + error, sendonly = self._pending_close + self._pending_close = None + on_close(error, sendonly) + if not sendonly: + # the late binder has now claimed the buffered close + self.gateway._forget_channel(self.id) + # dispatch-loop internals (inline on the gateway's serve task) def _feed(self, data: bytes) -> None: + if self._consumer_payload is not None: + self._consumer_payload(data) + return try: self._payload_send.send_nowait(data) except (trio.BrokenResourceError, trio.ClosedResourceError): @@ -243,8 +280,17 @@ def _close_from_remote(self, error: RemoteError | None, *, sendonly: bool) -> No self._receive_closed.set() if not sendonly: self._closed = True - self.gateway._forget_channel(self.id) + if self._consumer_close is not None: + # Without a consumer the closed channel stays registered: + # a passed-channel reference may still bind late and must + # find the buffered payloads and this close, not a fresh + # empty channel under the same id. + self.gateway._forget_channel(self.id) self._payload_send.close() + if self._consumer_close is not None: + self._consumer_close(error, sendonly) + else: + self._pending_close = (error, sendonly) class RawChannelStream: diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 68b278d7..ed4f876e 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -7,12 +7,13 @@ from __future__ import annotations +import functools import itertools import json -import math import subprocess import sys import threading +import weakref from collections.abc import Awaitable from collections.abc import Callable from contextlib import suppress @@ -26,6 +27,7 @@ from ._trio_gateway import AsyncGateway from ._trio_gateway import AsyncGroup from ._trio_gateway import ByteStream +from ._trio_gateway import RawChannelStream from ._trio_gateway import open_popen_process from ._trio_gateway import read_handshake_ack from ._trio_gateway import ssh_transport_args @@ -33,6 +35,7 @@ from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate from .gateway_base import Message +from .gateway_base import RemoteError from .gateway_base import dumps_internal from .gateway_base import loads_internal from .gateway_base import trace @@ -51,72 +54,6 @@ ssh_trio_args = ssh_transport_args -_CHANNEL_EOF = object() - - -class RawTunnelStream: - """``ByteStream`` over a raw channel id on a sync master ``Gateway``. - - The frame-native via tunnel: the master relays whole sub-protocol - frames as verbatim CHANNEL_DATA payloads (no serialization, no - double-framing), so this stream only buffers payloads; the - sub-session's FrameDecoder sees exact frame boundaries. - """ - - def __init__(self, gateway: BaseGateway, channelid: int) -> None: - self._gateway = gateway - self.channelid = channelid - self._send, self._recv = trio.open_memory_channel[Any](math.inf) - self._buf = bytearray() - self._eof = False - self._closed = False - gateway._channelfactory.register_raw_receiver( - channelid, self._on_data, self._on_close - ) - - def _on_data(self, data: bytes) -> None: - self._send.send_nowait(data) - - def _on_close(self, error: Any) -> None: - self._send.send_nowait(_CHANNEL_EOF if error is None else error) - - async def send_all(self, data: bytes) -> None: - self._gateway._send(Message.CHANNEL_DATA, self.channelid, data) - - async def receive_some(self, max_bytes: int | None = None) -> bytes: - if not self._buf and not self._eof: - item = await self._recv.receive() - if item is _CHANNEL_EOF: - self._eof = True - elif isinstance(item, Exception): - self._eof = True - raise EOFError(f"via tunnel closed: {item}") from None - else: - assert isinstance(item, bytes) - self._buf += item - if max_bytes is None: - max_bytes = len(self._buf) - out = bytes(self._buf[:max_bytes]) - del self._buf[:max_bytes] - return out - - async def send_eof(self) -> None: - self._close_tunnel() - - async def aclose(self) -> None: - self._close_tunnel() - - def _close_tunnel(self) -> None: - if self._closed: - return - self._closed = True - self._gateway._channelfactory.unregister_raw_receiver(self.channelid) - # Unblock our own reader: no more payloads will be routed here. - self._send.send_nowait(_CHANNEL_EOF) - with suppress(OSError): - self._gateway._send(Message.CHANNEL_CLOSE, self.channelid) - - async def adopt_socket(socket_fd: int) -> trio.SocketStream: """Worker side: wrap an inherited socket fd and send the handshake. @@ -229,16 +166,52 @@ def __init__( # -- engine hooks (run on the host loop) -- def _dispatch(self, message: Message) -> None: + """Route one message: sync-facade concerns here, the rest to the core. + + CHANNEL_DATA/CLOSE/CLOSE_ERROR/LAST_MESSAGE fall through to the + async core, which routes them to the RawChannel whose consumer is + the bound sync ``Channel``. + """ gateway = self.sync_gateway + code = message.msgcode try: - with gateway._receivelock: - message.received(gateway) + if code == Message.STATUS: + self._answer_status(message) + elif code == Message.CHANNEL_EXEC: + channel = gateway._channelfactory.new(message.channelid) + gateway._local_schedulexec(channel=channel, sourcetask=message.data) + elif code == Message.RECONFIGURE: + data = loads_internal(message.data, gateway) + assert isinstance(data, tuple) + if message.channelid == 0: + gateway._strconfig = data + else: + gateway._channelfactory.new(message.channelid)._strconfig = data + elif code == Message.GATEWAY_START_SOCKET: + handle_start_socket(gateway, message.channelid, message.data) + elif code == Message.GATEWAY_START_SUB: + handle_start_sub(gateway, message.channelid, message.data) + else: + super()._dispatch(message) except (GatewayReceivedTerminate, EOFError): raise except Exception as exc: gateway._trace("dispatch failed:", gateway._geterrortext(exc)) raise EOFError("error dispatching message") from exc + def _answer_status(self, message: Message) -> None: + # we use the channelid to send back information + # but don't instantiate a channel object + gateway = self.sync_gateway + execpool = getattr(gateway, "_execpool", None) + d = { + "numchannels": len(gateway._channelfactory._channels), + "numexecuting": execpool.active_count() if execpool is not None else 0, + "execmodel": gateway.execmodel.backend, + } + gateway._send(Message.CHANNEL_DATA, message.channelid, dumps_internal(d)) + gateway._send(Message.CHANNEL_CLOSE, message.channelid) + async def _finalize(self) -> None: gateway = self.sync_gateway with self._send_lock: @@ -247,6 +220,9 @@ async def _finalize(self) -> None: gateway._error = self._error gateway._trace("[trio-bridge] finishing channels") gateway._channelfactory._finished_receiving() + # EOF the loop-side raw channels that have no sync consumer + # (via-tunnel relays and readers blocked in receive_bytes). + self._finish_channels() # Unblock the worker's join() before heavy exec-pool shutdown # so the primary thread is not waiting on _done while terminate waits # on the primary thread draining work. @@ -259,6 +235,70 @@ async def _finalize(self) -> None: # -- sync session interface (any thread) -- + def bind_sync_channel(self, channel: Any) -> None: + """Route inbound data for ``channel.id`` to the sync channel. + + Callable from any thread: binding happens on the loop via the + portal so it cannot interleave with dispatch, and the raw channel + replays anything (payloads, a close) that arrived first. The loop + side only holds a weakref, preserving the factory's weak-registry + semantics (GC of the last user reference sends the close message); + channels with callbacks are kept alive by the factory instead. + """ + ref = weakref.ref(channel) + channelid = channel.id + + def bind() -> None: + raw = self._channel_for(channelid) + raw.set_consumer( + functools.partial(self._sync_payload, ref), + functools.partial(self._sync_close, ref, channelid), + ) + + with suppress(trio.RunFinishedError): + self.host.portal.post(bind) + + def _sync_payload(self, ref: weakref.ref[Any], data: bytes) -> None: + channel = ref() + if channel is not None: + channel._deliver_payload(data) + # dead ref: data for a deleted channel is dropped, like before + + def _sync_close( + self, + ref: weakref.ref[Any], + channelid: int, + error: RemoteError | None, + sendonly: bool, + ) -> None: + channel = ref() + if channel is None: + # channel already in "deleted" state + if error is not None: + error.warn() + self.sync_gateway._channelfactory._no_longer_opened(channelid) + return + channel._close_from_remote(error, sendonly=sendonly) + + def release_channel(self, channelid: int) -> None: + """Drop the loop-side raw channel for ``channelid`` (best-effort).""" + with suppress(trio.RunFinishedError): + self.host.portal.post(self._forget_channel, channelid) + + def run_on_loop(self, sync_fn: Callable[[], T]) -> T: + """Run ``sync_fn`` on the host loop, excluding dispatch interleaving. + + Falls back to running inline once the loop is gone (no more + deliveries can interleave then anyway). + """ + portal = self.host.portal + if portal.is_loop_thread(): + return sync_fn() + try: + return portal.run_sync(sync_fn) + except trio.RunFinishedError: + return sync_fn() + def enqueue_message(self, message: Message) -> None: """Enqueue a frame; wait until written when safe to block. @@ -471,11 +511,13 @@ async def _open_via_stream(self, spec: Any) -> ByteStream: from . import _provision master = self.group[spec.via] + session = master._trio_session + assert isinstance(session, SyncBridgeGateway) channelid = master._channelfactory.allocate_id() + # Create the raw channel before the request goes out so no relayed + # frame can arrive unrouted (we are on the loop: no dispatch races). + io = RawChannelStream(session._channel_for(channelid)) request = _provision.spawn_request(spec) - # Register the raw receiver before the request goes out so no - # relayed frame can arrive unrouted. - io = RawTunnelStream(master, channelid) master._send(Message.GATEWAY_START_SUB, channelid, dumps_internal(request)) await read_handshake_ack(io, "via") return io @@ -602,7 +644,7 @@ async def _start_sub_and_relay( Runs on the master's Trio host (the ``via`` transport). The tunnel is frame-native both ways: coordinator payloads arrive verbatim through the - raw-receiver registry and go to the sub's stdin unchanged (each payload + session's raw channel and go to the sub's stdin unchanged (each payload one whole frame), while the sub's stdout runs through a FrameDecoder so every CHANNEL_DATA sent back carries exactly one frame -- except the initial ready byte, which is forwarded on its own for the handshake. @@ -621,21 +663,17 @@ def send_close_error(text: str) -> None: except Exception as exc: send_close_error(f"could not spawn via sub-gateway: {exc}") return - send_ch, recv_ch = trio.open_memory_channel[Any](math.inf) - gateway._channelfactory.register_raw_receiver( - channelid, - send_ch.send_nowait, - lambda error: send_ch.send_nowait(_CHANNEL_EOF), - ) + session = gateway._trio_session + assert isinstance(session, SyncBridgeGateway) + raw = session._channel_for(channelid) async def coordinator_to_sub() -> None: assert process.stdin is not None if preamble: await process.stdin.send_all(preamble) - async for data in recv_ch: - if data is _CHANNEL_EOF: - break - await process.stdin.send_all(data) + with suppress(RemoteError): + async for data in raw: + await process.stdin.send_all(data) with trio.move_on_after(5): await process.stdin.aclose() @@ -664,7 +702,7 @@ async def sub_to_coordinator() -> None: gateway._trace("via sub relay failed:", exc) send_close_error(f"via sub-gateway relay failed: {exc}") finally: - gateway._channelfactory.unregister_raw_receiver(channelid) + session._forget_channel(channelid) with trio.move_on_after(5): await process.wait() diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index aa3e68d9..6506199d 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -76,7 +76,7 @@ def _track_finish(self) -> None: self._idle.set() def schedule(self, channel: Channel, sourcetask: bytes) -> None: - """Called from the Trio receiver while holding ``_receivelock``. + """Called from the session dispatch on the Trio host thread. Must not block: deadlock checks and exec run in a nursery task. """ diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 2923eaa3..47bf7d32 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -12,9 +12,12 @@ from __future__ import annotations import abc +import builtins import os +import queue as _queue import struct import sys +import threading import traceback import weakref from _thread import interrupt_main @@ -28,6 +31,8 @@ from typing import cast from typing import overload +from ._boundary import Mailbox + class WriteIO(Protocol): def write(self, data: bytes, /) -> None: ... @@ -390,10 +395,37 @@ def trace(*msg: object) -> None: class Message: - """Encapsulates Messages and their wire protocol.""" + """Encapsulates Messages and their wire protocol. + + Dispatch lives in the async core and the sync bridge session + (``AsyncGateway._dispatch`` / ``SyncBridgeGateway._dispatch``); this + class only carries the framing and the code constants. + """ + + STATUS = 0 + RECONFIGURE = 1 + GATEWAY_TERMINATE = 2 + CHANNEL_EXEC = 3 + CHANNEL_DATA = 4 + CHANNEL_CLOSE = 5 + CHANNEL_CLOSE_ERROR = 6 + CHANNEL_LAST_MESSAGE = 7 + GATEWAY_START_SOCKET = 8 + GATEWAY_START_SUB = 9 - # message code -> name, handler - _types: dict[int, tuple[str, Callable[[Message, BaseGateway], None]]] = {} + # message code -> name + _types: dict[int, str] = { + STATUS: "STATUS", + RECONFIGURE: "RECONFIGURE", + GATEWAY_TERMINATE: "GATEWAY_TERMINATE", + CHANNEL_EXEC: "CHANNEL_EXEC", + CHANNEL_DATA: "CHANNEL_DATA", + CHANNEL_CLOSE: "CHANNEL_CLOSE", + CHANNEL_CLOSE_ERROR: "CHANNEL_CLOSE_ERROR", + CHANNEL_LAST_MESSAGE: "CHANNEL_LAST_MESSAGE", + GATEWAY_START_SOCKET: "GATEWAY_START_SOCKET", + GATEWAY_START_SUB: "GATEWAY_START_SUB", + } def __init__(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: self.msgcode = msgcode @@ -431,103 +463,10 @@ def from_io(io: ReadIO) -> Message: def to_io(self, io: WriteIO) -> None: io.write(self.pack()) - def received(self, gateway: BaseGateway) -> None: - handler = self._types[self.msgcode][1] - handler(self, gateway) - def __repr__(self) -> str: - name = self._types[self.msgcode][0] + name = self._types[self.msgcode] return f"" - def _status(message: Message, gateway: BaseGateway) -> None: - # we use the channelid to send back information - # but don't instantiate a channel object - d = { - "numchannels": len(gateway._channelfactory._channels), - # TODO(typing): Attribute `_execpool` is only on WorkerGateway. - "numexecuting": gateway._execpool.active_count(), # type: ignore[attr-defined] - "execmodel": gateway.execmodel.backend, - } - gateway._send(Message.CHANNEL_DATA, message.channelid, dumps_internal(d)) - gateway._send(Message.CHANNEL_CLOSE, message.channelid) - - STATUS = 0 - _types[STATUS] = ("STATUS", _status) - - def _reconfigure(message: Message, gateway: BaseGateway) -> None: - data = loads_internal(message.data, gateway) - assert isinstance(data, tuple) - strconfig: tuple[bool, bool] = data - if message.channelid == 0: - gateway._strconfig = strconfig - else: - gateway._channelfactory.new(message.channelid)._strconfig = strconfig - - RECONFIGURE = 1 - _types[RECONFIGURE] = ("RECONFIGURE", _reconfigure) - - def _gateway_terminate(message: Message, gateway: BaseGateway) -> None: - raise GatewayReceivedTerminate(gateway) - - GATEWAY_TERMINATE = 2 - _types[GATEWAY_TERMINATE] = ("GATEWAY_TERMINATE", _gateway_terminate) - - def _channel_exec(message: Message, gateway: BaseGateway) -> None: - channel = gateway._channelfactory.new(message.channelid) - gateway._local_schedulexec(channel=channel, sourcetask=message.data) - - CHANNEL_EXEC = 3 - _types[CHANNEL_EXEC] = ("CHANNEL_EXEC", _channel_exec) - - def _channel_data(message: Message, gateway: BaseGateway) -> None: - gateway._channelfactory._local_receive(message.channelid, message.data) - - CHANNEL_DATA = 4 - _types[CHANNEL_DATA] = ("CHANNEL_DATA", _channel_data) - - def _channel_close(message: Message, gateway: BaseGateway) -> None: - gateway._channelfactory._local_close(message.channelid) - - CHANNEL_CLOSE = 5 - _types[CHANNEL_CLOSE] = ("CHANNEL_CLOSE", _channel_close) - - def _channel_close_error(message: Message, gateway: BaseGateway) -> None: - error_message = loads_internal(message.data) - assert isinstance(error_message, str) - remote_error = RemoteError(error_message) - gateway._channelfactory._local_close(message.channelid, remote_error) - - CHANNEL_CLOSE_ERROR = 6 - _types[CHANNEL_CLOSE_ERROR] = ("CHANNEL_CLOSE_ERROR", _channel_close_error) - - def _channel_last_message(message: Message, gateway: BaseGateway) -> None: - gateway._channelfactory._local_close(message.channelid, sendonly=True) - - CHANNEL_LAST_MESSAGE = 7 - _types[CHANNEL_LAST_MESSAGE] = ("CHANNEL_LAST_MESSAGE", _channel_last_message) - - def _gateway_start_socket(message: Message, gateway: BaseGateway) -> None: - # Start a one-shot socketserver on this (Trio) gateway's host and reply - # with the bound (host, port) on the request channel. Handled natively - # instead of shipping source via remote_exec. - from . import _trio_host - - _trio_host.handle_start_socket(gateway, message.channelid, message.data) - - GATEWAY_START_SOCKET = 8 - _types[GATEWAY_START_SOCKET] = ("GATEWAY_START_SOCKET", _gateway_start_socket) - - def _gateway_start_sub(message: Message, gateway: BaseGateway) -> None: - # Spawn a sub-gateway worker (popen, foreign python, or ssh) on this - # (Trio) gateway's host and relay its Message protocol over the request - # channel (the ``via`` transport). - from . import _trio_host - - _trio_host.handle_start_sub(gateway, message.channelid, message.data) - - GATEWAY_START_SUB = 9 - _types[GATEWAY_START_SUB] = ("GATEWAY_START_SUB", _gateway_start_sub) - class FrameDecoder: """Incremental decoder for the 9-byte-header Message framing. @@ -617,7 +556,13 @@ class TimeoutError(IOError): class Channel: - """Communication channel between two Python Interpreter execution points.""" + """Communication channel between two Python Interpreter execution points. + + A facade over the async core: the gateway's Trio session diverts + inbound payloads for this id into a :class:`Mailbox` (or a registered + callback, invoked on the loop thread); ``receive()`` deserializes at + the call site. Sends go through ``gateway._send``. + """ RemoteError = RemoteError TimeoutError = TimeoutError @@ -632,9 +577,12 @@ def __init__(self, gateway: BaseGateway, id: int) -> None: # XXX: defaults copied from Unserializer self._strconfig = getattr(gateway, "_strconfig", (True, False)) self.id = id - self._items = self.gateway.execmodel.queue.Queue() + # serialized payloads (or ENDMARKER); None once a callback is set + self._mailbox: Mailbox[Any] | None = Mailbox() + self._callback: Callable[[Any], Any] | None = None + self._endmarker: object = NO_ENDMARKER_WANTED self._closed = False - self._receiveclosed = self.gateway.execmodel.Event() + self._receiveclosed = threading.Event() self._remoteerrors: list[RemoteError] = [] def _trace(self, *msg: object) -> None: @@ -648,33 +596,36 @@ def setcallback( """Set a callback function for receiving items. All already-queued items will immediately trigger the callback. - Afterwards the callback will execute in the receiver thread + Afterwards the callback will execute in the receiver (loop) thread for each received data item and calls to ``receive()`` will raise an error. If an endmarker is specified the callback will eventually be called with the endmarker when the channel closes. """ - _callbacks = self.gateway._channelfactory._callbacks - with self.gateway._receivelock: - if self._items is None: + + def switch() -> None: + # Runs on the loop thread (inline without a session), so the + # switch-over cannot interleave with payload delivery. + mailbox = self._mailbox + if mailbox is None: raise OSError(f"{self!r} has callback already registered") - items = self._items - self._items = None + self._mailbox = None while 1: try: - olditem = items.get(block=False) - except self.gateway.execmodel.queue.Empty: + olditem = mailbox.get_nowait() + except _queue.Empty: if not (self._closed or self._receiveclosed.is_set()): - _callbacks[self.id] = (callback, endmarker, self._strconfig) + self._callback = callback + self._endmarker = endmarker + self.gateway._channelfactory._register_callback_channel(self) break - else: - if olditem is ENDMARKER: - items.put(olditem) # for other receivers - if endmarker is not NO_ENDMARKER_WANTED: - callback(endmarker) - break - else: - callback(olditem) + if olditem is ENDMARKER: + if endmarker is not NO_ENDMARKER_WANTED: + callback(endmarker) + break + callback(loads_internal(olditem, self)) + + self.gateway._run_on_loop(switch) def __repr__(self) -> str: flag = (self.isclosed() and "closed") or "open" @@ -701,7 +652,7 @@ def __del__(self) -> None: # don't need to try to send a closing or last message # (and often it won't work anymore to send things out) if Message is not None: - if self._items is None: # has_callback + if self._mailbox is None: # has_callback msgcode = Message.CHANNEL_LAST_MESSAGE else: msgcode = Message.CHANNEL_CLOSE @@ -711,6 +662,8 @@ def __del__(self) -> None: self.gateway, "_send_nonblocking", self.gateway._send ) send(msgcode, self.id) + with suppress(Exception): + self.gateway._release_channel(self.id) def _getremoteerror(self): try: @@ -722,6 +675,50 @@ def _getremoteerror(self): pass return None + # + # loop-side delivery (called by the session's raw-channel consumer) + # + def _deliver_payload(self, data: bytes) -> None: + """Route one inbound serialized payload (loop thread).""" + if self._closed: + return # late data for a locally closed channel: drop + callback = self._callback + if callback is None: + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(data) + # no mailbox and no callback: closed for receiving -- drop + else: + try: + callback(loads_internal(data, self)) + except Exception as exc: + self.gateway._trace("exception during callback: %s" % exc) + errortext = self.gateway._geterrortext(exc) + self.gateway._send( + Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(errortext) + ) + self._close_from_remote(RemoteError(errortext)) + + def _close_from_remote(self, remoteerror=None, *, sendonly: bool = False) -> None: + """Close initiated by the peer or session shutdown (loop thread).""" + if remoteerror: + self._remoteerrors.append(remoteerror) + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) + self._fire_endmarker() + self.gateway._channelfactory._no_longer_opened(self.id) + if not sendonly: # otherwise #--> "sendonly" + self._closed = True # --> "closed" + self._receiveclosed.set() + + def _fire_endmarker(self) -> None: + callback = self._callback + if callback is not None: + self._callback = None + if self._endmarker is not NO_ENDMARKER_WANTED: + callback(self._endmarker) + # # public API for channel objects # @@ -788,10 +785,12 @@ def close(self, error=None) -> None: self._remoteerrors.append(error) self._closed = True # --> "closed" self._receiveclosed.set() - queue = self._items - if queue is not None: - queue.put(ENDMARKER) + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) + self._fire_endmarker() self.gateway._channelfactory._no_longer_opened(self.id) + self.gateway._release_channel(self.id) def waitclose(self, timeout: float | None = None) -> None: """Wait until this channel is closed (or the remote side @@ -841,18 +840,18 @@ def receive(self, timeout: float | None = None) -> Any: reraised as channel.RemoteError exceptions containing a textual representation of the remote traceback. """ - itemqueue = self._items - if itemqueue is None: + mailbox = self._mailbox + if mailbox is None: raise OSError("cannot receive(), channel has receiver callback") try: - x = itemqueue.get(timeout=timeout) - except self.gateway.execmodel.queue.Empty: + x = mailbox.get(timeout) + except builtins.TimeoutError: raise self.TimeoutError("no item after %r seconds" % timeout) from None if x is ENDMARKER: - itemqueue.put(x) # for other receivers + mailbox.put(x) # for other receivers raise self._getremoteerror() or EOFError() else: - return x + return loads_internal(x, self) def __iter__(self) -> Iterator[Any]: return self @@ -886,21 +885,22 @@ def reconfigure( class ChannelFactory: + """Registry and id allocator for a gateway's sync channels. + + Message routing lives in the Trio session (the sync channel binds a + consumer on the session's raw channel); the factory only tracks live + channels -- weakly, so dropping the last user reference triggers + ``Channel.__del__``'s close message -- and keeps channels with a + registered callback strongly alive until they close. + """ + def __init__(self, gateway: BaseGateway, startcount: int = 1) -> None: self._channels: weakref.WeakValueDictionary[int, Channel] = ( weakref.WeakValueDictionary() ) - # Channel ID => (callback, end marker, strconfig) - self._callbacks: dict[ - int, tuple[Callable[[Any], Any], object, tuple[bool, bool]] - ] = {} - # Channel ID => (feed, on_close) receiving CHANNEL_DATA verbatim - # (no serialization) -- the low-level raw channel routing used by - # byte-shaped consumers such as the via tunnel. - self._raw_receivers: dict[ - int, tuple[Callable[[bytes], None], Callable[[RemoteError | None], None]] - ] = {} - self._writelock = gateway.execmodel.Lock() + # channels kept strongly alive while their callback is registered + self._callback_channels: dict[int, Channel] = {} + self._writelock = threading.Lock() self.gateway = gateway self.count = startcount self.finished = False @@ -918,6 +918,7 @@ def new(self, id: int | None = None) -> Channel: channel = self._channels[id] except KeyError: channel = self._channels[id] = Channel(self.gateway, id) + self.gateway._bind_channel(channel) return channel def allocate_id(self) -> int: @@ -929,42 +930,21 @@ def allocate_id(self) -> int: self.count += 2 return id - def register_raw_receiver( - self, - id: int, - feed: Callable[[bytes], None], - on_close: Callable[[RemoteError | None], None], - ) -> None: - """Route CHANNEL_DATA payloads for ``id`` verbatim to ``feed``. - - ``on_close`` fires once when the channel closes (with the peer's - RemoteError, if any) or when receiving finishes. - """ - self._raw_receivers[id] = (feed, on_close) - - def unregister_raw_receiver(self, id: int) -> None: - self._raw_receivers.pop(id, None) - def channels(self) -> list[Channel]: return self._list(self._channels.values()) # - # internal methods, called from the receiver thread + # internal methods, called from the loop thread (or local close paths) # + def _register_callback_channel(self, channel: Channel) -> None: + self._callback_channels[channel.id] = channel + def _no_longer_opened(self, id: int) -> None: self._channels.pop(id, None) - item = self._callbacks.pop(id, None) - if item is not None: - callback, endmarker, _strconfig = item - if endmarker is not NO_ENDMARKER_WANTED: - callback(endmarker) + self._callback_channels.pop(id, None) def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> None: - raw = self._raw_receivers.pop(id, None) - if raw is not None: - _feed, on_close = raw - on_close(remoteerror) - return + """Close ``id`` as if the peer had closed it (no message is sent).""" channel = self._channels.get(id) if channel is None: # channel already in "deleted" state @@ -972,57 +952,13 @@ def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> Non remoteerror.warn() self._no_longer_opened(id) else: - # state transition to "closed" state - if remoteerror: - channel._remoteerrors.append(remoteerror) - queue = channel._items - if queue is not None: - queue.put(ENDMARKER) - self._no_longer_opened(id) - if not sendonly: # otherwise #--> "sendonly" - channel._closed = True # --> "closed" - channel._receiveclosed.set() - - def _local_receive(self, id: int, data) -> None: - # executes in receiver thread - raw = self._raw_receivers.get(id) - if raw is not None: - feed, _on_close = raw - feed(data) - return - channel = self._channels.get(id) - try: - callback, _endmarker, strconfig = self._callbacks[id] - except KeyError: - queue = channel._items if channel is not None else None - if queue is None: - pass # drop data - else: - item = loads_internal(data, channel) - queue.put(item) - else: - try: - data = loads_internal(data, channel, strconfig) - callback(data) # even if channel may be already closed - except Exception as exc: - self.gateway._trace("exception during callback: %s" % exc) - errortext = self.gateway._geterrortext(exc) - self.gateway._send( - Message.CHANNEL_CLOSE_ERROR, id, dumps_internal(errortext) - ) - self._local_close(id, errortext) + channel._close_from_remote(remoteerror, sendonly=sendonly) def _finished_receiving(self) -> None: with self._writelock: self.finished = True for id in self._list(self._channels): self._local_close(id, sendonly=True) - for id in self._list(self._callbacks): - self._no_longer_opened(id) - for id in self._list(self._raw_receivers): - item = self._raw_receivers.pop(id, None) - if item is not None: - item[1](None) class ChannelFile: @@ -1099,7 +1035,6 @@ def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.id = id self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) self._channelfactory = ChannelFactory(self, _startcount) - self._receivelock = self.execmodel.RLock() # globals may be NONE at process-termination self.__trace = trace self._geterrortext = geterrortext @@ -1109,8 +1044,36 @@ def _trace(self, *msg: object) -> None: self.__trace(self.id, *msg) def _attach_trio_session(self, session: Any) -> None: - """Attach a Trio ProtocolSession for Message IO (no receiver thread).""" + """Attach the Trio bridge session doing the Message IO.""" self._trio_session = session + # Defensive: channels created before the session existed still + # need their inbound routing diverted to them. + for channel in self._channelfactory.channels(): + session.bind_sync_channel(channel) + + def _bind_channel(self, channel: Channel) -> None: + """Divert the session's inbound routing for ``channel.id`` to it.""" + session = self._trio_session + if session is not None: + session.bind_sync_channel(channel) + + def _release_channel(self, id: int) -> None: + """Drop the session's loop-side state for ``id`` (best-effort).""" + session = self._trio_session + if session is not None: + with suppress(Exception): + session.release_channel(id) + + def _run_on_loop(self, sync_fn: Callable[[], Any]) -> Any: + """Run ``sync_fn`` on the session's loop thread (inline without one). + + Payload dispatch happens on the loop thread, so state switches run + there to exclude interleaving with deliveries. + """ + session = self._trio_session + if session is None: + return sync_fn() + return session.run_on_loop(sync_fn) def _terminate_execution(self) -> None: pass diff --git a/src/execnet/portal.py b/src/execnet/portal.py index 223fdd97..5cc3e30e 100644 --- a/src/execnet/portal.py +++ b/src/execnet/portal.py @@ -20,19 +20,18 @@ from __future__ import annotations -import queue -import threading -import time from collections.abc import Awaitable from collections.abc import Callable from typing import Any -from typing import Generic -from typing import Protocol from typing import TypeVar -from typing import cast import trio +from ._boundary import Mailbox +from ._boundary import OneShot +from ._boundary import ThreadWakener +from ._boundary import Wakener + __all__ = ["LoopPortal", "Mailbox", "OneShot", "ThreadWakener", "Wakener"] T = TypeVar("T") @@ -71,144 +70,3 @@ def post(self, sync_fn: Callable[..., object], *args: Any) -> None: ``trio.RunFinishedError`` once the loop has shut down. """ self._token.run_sync_soon(sync_fn, *args) - - -class Wakener(Protocol): - """Thread-safe consumer wakeup fired by the loop. - - ``notify()`` must never block and must be safe from any thread; it is - the only thing the loop side ever calls. The blocking mailbox/oneshot - waits additionally need :meth:`wait`/:meth:`clear` executed in the - consumer's own context (a thread here, a greenlet for a gevent - wakener); loop-native consumers such as asyncio wait on their own - side of ``notify`` instead. - """ - - def notify(self) -> None: ... - - def wait(self, timeout: float | None = None) -> bool: ... - - def clear(self) -> None: ... - - -class ThreadWakener: - """Plain-thread wakener on a ``threading.Event``. - - ``threading.Event.wait`` stays interruptible by KeyboardInterrupt on - the main thread (a C-level ``queue.SimpleQueue.get`` does not), which - is why the carriers wait on the wakener and drain a queue instead of - blocking in the queue itself. - """ - - def __init__(self) -> None: - self._event = threading.Event() - - def notify(self) -> None: - self._event.set() - - def wait(self, timeout: float | None = None) -> bool: - return self._event.wait(timeout) - - def clear(self) -> None: - self._event.clear() - - -class Mailbox(Generic[T]): - """Loop -> consumer item stream: an unbounded queue plus a wakener. - - :meth:`put` never blocks and is safe from any thread (including the - loop thread). :meth:`get` blocks in the consumer's context via the - wakener; after draining to empty it clears and re-checks so a ``put`` - racing the clear cannot be lost. Multiple consumers are allowed. - """ - - def __init__(self, wakener: Wakener | None = None) -> None: - self._items: queue.SimpleQueue[T] = queue.SimpleQueue() - self._wakener = ThreadWakener() if wakener is None else wakener - - def put(self, item: T) -> None: - """Thread-safe; usable from a loop thread (never blocks).""" - self._items.put(item) - self._wakener.notify() - - def get_nowait(self) -> T: - """Return the next item or raise ``queue.Empty``.""" - return self._items.get_nowait() - - def get(self, timeout: float | None = None) -> T: - """Block until an item is available; TimeoutError after ``timeout``.""" - deadline = None if timeout is None else time.monotonic() + timeout - while True: - if deadline is None: - self._wakener.wait() - else: - remaining = deadline - time.monotonic() - if remaining <= 0 or not self._wakener.wait(remaining): - # Final non-blocking check: a put may have raced the - # timeout (its notify landing after our last wait). - try: - return self._items.get_nowait() - except queue.Empty: - raise TimeoutError( - "no item after %r seconds" % timeout - ) from None - try: - return self._items.get_nowait() - except queue.Empty: - # Empty: clear, then re-check so a put between get_nowait - # and clear cannot be lost. - self._wakener.clear() - try: - return self._items.get_nowait() - except queue.Empty: - continue - - -class OneShot(Generic[T]): - """A single result crossing loop -> consumer exactly once. - - The loop side calls :meth:`set` (or :meth:`set_error`) at most once; - the consumer blocks in :meth:`wait`, which returns the value, - re-raises the stored error, or raises ``TimeoutError``. - """ - - _NOTSET = object() - - def __init__(self, wakener: Wakener | None = None) -> None: - self._wakener = ThreadWakener() if wakener is None else wakener - self._value: Any = self._NOTSET - self._error: BaseException | None = None - self._done = False - - def is_set(self) -> bool: - return self._done - - def set(self, value: T) -> None: - """Thread-safe; usable from a loop thread (never blocks).""" - assert not self._done, "OneShot already resolved" - self._value = value - self._done = True - self._wakener.notify() - - def set_error(self, error: BaseException) -> None: - """Resolve with an error that :meth:`wait` will re-raise.""" - assert not self._done, "OneShot already resolved" - self._error = error - self._done = True - self._wakener.notify() - - def wait(self, timeout: float | None = None) -> T: - """Block until resolved; TimeoutError after ``timeout`` seconds.""" - deadline = None if timeout is None else time.monotonic() + timeout - while not self._done: - if deadline is None: - self._wakener.wait() - else: - remaining = deadline - time.monotonic() - if (remaining <= 0 or not self._wakener.wait(remaining)) and ( - not self._done - ): - raise TimeoutError("not resolved after %r seconds" % timeout) - if self._error is not None: - raise self._error - return cast("T", self._value) diff --git a/testing/test_basics.py b/testing/test_basics.py index 473cd741..29ceef19 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -15,6 +15,7 @@ import pytest import execnet +from execnet import _boundary from execnet import gateway from execnet import gateway_base from execnet.gateway_base import ChannelFactory @@ -66,6 +67,21 @@ def test_errors_on_execnet() -> None: assert hasattr(execnet, "DataFormatError") +def standalone_gateway_base_source() -> str: + """gateway_base's source as a self-contained script. + + The module imports the trio-free boundary kit relatively; for the + run-on-any-python checks the kit source is inlined at the import site + (minus its own future import, which must stay file-leading). + """ + boundary_source = inspect.getsource(_boundary).replace( + "from __future__ import annotations\n", "" + ) + return inspect.getsource(gateway_base).replace( + "from ._boundary import Mailbox\n", boundary_source + ) + + IO_MESSAGE_EXTRA_SOURCE = """ from io import BytesIO @@ -124,7 +140,7 @@ def checker(anypython: str, tmp_path: Path) -> Checker: def test_io_message(checker: Checker) -> None: - out = checker.run_check(inspect.getsource(gateway_base) + IO_MESSAGE_EXTRA_SOURCE) + out = checker.run_check(standalone_gateway_base_source() + IO_MESSAGE_EXTRA_SOURCE) print(out.stdout) assert "all passed" in out.stdout @@ -147,7 +163,7 @@ def send(self, data): def test_geterrortext(checker: Checker) -> None: out = checker.run_check( - inspect.getsource(gateway_base) + standalone_gateway_base_source() + """ class Arg(Exception): pass @@ -324,12 +340,20 @@ class TestPureChannel: @pytest.fixture def fac(self, execmodel: ExecModel) -> ChannelFactory: class FakeGateway: + _trio_session = None + def _trace(self, *args) -> None: pass def _send(self, *k) -> None: pass + def _bind_channel(self, channel) -> None: + pass + + def _release_channel(self, id) -> None: + pass + FakeGateway.execmodel = execmodel # type: ignore[attr-defined] return ChannelFactory(FakeGateway()) # type: ignore[arg-type] diff --git a/testing/test_serializer.py b/testing/test_serializer.py index 06a84cd2..cb2d422c 100644 --- a/testing/test_serializer.py +++ b/testing/test_serializer.py @@ -9,8 +9,9 @@ import execnet -# We use the execnet folder in order to avoid triggering a missing apipkg. -pyimportdir = os.fspath(Path(execnet.__file__).parent) +# The package parent: the serializer is imported as execnet.gateway_base +# (it needs its trio-free sibling execnet._boundary; only stdlib beyond that). +pyimportdir = os.fspath(Path(execnet.__file__).parent.parent) class PythonWrapper: @@ -24,7 +25,7 @@ def dump(self, obj_rep: str) -> bytes: f""" import sys sys.path.insert(0, {pyimportdir!r}) -import gateway_base as serializer +import execnet.gateway_base as serializer sys.stdout = sys.stdout.detach() sys.stdout.write(serializer.dumps_internal({obj_rep})) """ @@ -40,7 +41,7 @@ def load(self, data: bytes) -> list[str]: rf""" import sys sys.path.insert(0, {pyimportdir!r}) -import gateway_base as serializer +import execnet.gateway_base as serializer from io import BytesIO data = {data!r} io = BytesIO(data) From 811fd9956374b2cf2685094d3aca2f30bcec26cd Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:15:41 +0200 Subject: [PATCH 37/91] feat!: retire the ExecModel machinery; add the wait= spec axis P3 of the boundary rethink. The ExecModel ABC, ThreadExecModel, MainThreadOnlyExecModel, WorkerPool and Reply are gone: protocol IO always runs on the Trio host, blocking waits go through the boundary kit, and execution placement is the worker config axes' job. What survives is a single deprecated ExecModel preset carrying the backend name plus the stdlib-delegating members (pytest-xdist builds its remote test queue on execmodel.RLock/Event, so the duck-type stays live); get_execmodel/set_execmodel/spec.execmodel keep working on top of it. safe_terminate now runs its term/kill pairs on daemon threads with the same bounded-wait contract; MultiChannel.make_receive_queue hands out a plain stdlib queue. New wait= axis end to end: the boundary kit gains Flag (idempotent event on a wakener) and a wakener-factory registry ("thread" built in; gevent/asyncio register later); specs accept wait= (validated in makegateway), the worker CLI config ships it, and every blocking carrier (channel mailbox, receive-closed flag, write-acks, join) is created through gateway._new_wakener() on both sides of the wire. Test suite: test_threadpool.py removed with WorkerPool; the pool fixture became a ThreadPoolExecutor; direct execmodel.queue/Event/sleep uses in tests moved to the stdlib equivalents. Co-Authored-By: Claude Fable 5 --- src/execnet/_boundary.py | 70 ++++++++- src/execnet/_provision.py | 1 + src/execnet/_trio_host.py | 6 +- src/execnet/_trio_worker.py | 34 +++-- src/execnet/gateway.py | 2 + src/execnet/gateway_base.py | 274 ++++-------------------------------- src/execnet/multi.py | 63 ++++++--- src/execnet/xspec.py | 1 + testing/conftest.py | 7 +- testing/test_basics.py | 19 ++- testing/test_channel.py | 5 +- testing/test_gateway.py | 2 +- testing/test_multi.py | 7 +- testing/test_termination.py | 19 ++- testing/test_threadpool.py | 222 ----------------------------- testing/test_xspec.py | 8 ++ 16 files changed, 213 insertions(+), 527 deletions(-) delete mode 100644 testing/test_threadpool.py diff --git a/src/execnet/_boundary.py b/src/execnet/_boundary.py index f80f6998..ce211ae1 100644 --- a/src/execnet/_boundary.py +++ b/src/execnet/_boundary.py @@ -9,13 +9,22 @@ import queue import threading import time +from collections.abc import Callable from typing import Any from typing import Generic from typing import Protocol from typing import TypeVar from typing import cast -__all__ = ["Mailbox", "OneShot", "ThreadWakener", "Wakener"] +__all__ = [ + "Flag", + "Mailbox", + "OneShot", + "ThreadWakener", + "Wakener", + "make_wakener", + "register_wakener", +] T = TypeVar("T") @@ -159,3 +168,62 @@ def wait(self, timeout: float | None = None) -> T: if self._error is not None: raise self._error return cast("T", self._value) + + +class Flag: + """An idempotent event on a wakener: may be set any number of times. + + Each Flag owns its wakener exclusively -- sharing one wakener between + carriers would lose wakeups (another carrier's ``clear`` can swallow + this one's ``notify``). + """ + + def __init__(self, wakener: Wakener | None = None) -> None: + self._wakener = ThreadWakener() if wakener is None else wakener + self._flag = False + + def is_set(self) -> bool: + return self._flag + + def set(self) -> None: + """Thread-safe; usable from a loop thread (never blocks).""" + self._flag = True + self._wakener.notify() + + def wait(self, timeout: float | None = None) -> bool: + """Block until set; returns whether the flag is set.""" + deadline = None if timeout is None else time.monotonic() + timeout + while not self._flag: + if deadline is None: + self._wakener.wait() + else: + remaining = deadline - time.monotonic() + if remaining <= 0 or not self._wakener.wait(remaining): + break + return self._flag + + +# wait= axis: named wakener factories; each call returns a fresh instance +# (carriers own their wakener exclusively, see Flag). Backends register +# here ("gevent", "asyncio") next to the built-in "thread". +_WAKENER_FACTORIES: dict[str, Callable[[], Wakener]] = { + "thread": ThreadWakener, +} + + +def register_wakener(name: str, factory: Callable[[], Wakener]) -> None: + """Register a wakener factory for the ``wait=`` spec axis.""" + _WAKENER_FACTORIES[name] = factory + + +def wakener_names() -> list[str]: + return list(_WAKENER_FACTORIES) + + +def make_wakener(name: str) -> Wakener: + """Create a fresh wakener for the named wait backend.""" + try: + factory = _WAKENER_FACTORIES[name] + except KeyError: + raise ValueError(f"unknown wait backend {name!r}") from None + return factory() diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index b1c16335..f5114ad8 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -168,6 +168,7 @@ def worker_cli_arg(spec: Any) -> str: { "id": f"{spec.id}-worker", "execmodel": spec.execmodel, + "wait": spec.wait or "thread", "coordinator_version": execnet.__version__, } ) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index ed4f876e..e29af4d2 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -155,7 +155,7 @@ def __init__( super().__init__(stream, id=id) self.sync_gateway = sync_gateway self.host = host - self._done_sync: OneShot[None] = OneShot() + self._done_sync: OneShot[None] = OneShot(sync_gateway._new_wakener()) self._send_closed = False self._send_lock = threading.Lock() # Attach before any serving can happen: the first inbound message @@ -310,7 +310,9 @@ def enqueue_message(self, message: Message) -> None: # The ack carries the write failure as a value (never raised into # the OneShot) so a KeyboardInterrupt in wait() stays distinguishable # from a stream error. - ack: OneShot[BaseException | None] | None = OneShot() if wait else None + ack: OneShot[BaseException | None] | None = ( + OneShot(self.sync_gateway._new_wakener()) if wait else None + ) def post() -> None: try: diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 6506199d..89aad122 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -201,12 +201,13 @@ def kill(self) -> None: def _build_worker_gateway( - host: _trio_host.TrioHost, id: str, model: ExecModel + host: _trio_host.TrioHost, id: str, model: ExecModel, wait: str = "thread" ) -> tuple[WorkerGateway, TrioWorkerExec, bool]: """Construct the WorkerGateway + Trio exec pool (no IO yet).""" trace(f"creating workergateway on trio id={id!r}") io_stub = _WorkerIOStub(model) gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) + gateway._wait_backend = wait main_thread_only = model.backend == "main_thread_only" trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) @@ -215,14 +216,16 @@ def _build_worker_gateway( gateway._trio_exec = trio_exec gateway._executetask_complete = None if main_thread_only: - gateway._executetask_complete = model.Event() + gateway._executetask_complete = threading.Event() gateway._executetask_complete.set() return gateway, trio_exec, main_thread_only -def _run_worker(host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel) -> None: +def _run_worker( + host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel, wait: str = "thread" +) -> None: """Attach ``io`` as the gateway session and serve until shutdown.""" - gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model) + gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model, wait) async def _start() -> _trio_host.SyncBridgeGateway: # The bridge attaches itself to the gateway before serving starts, @@ -252,7 +255,7 @@ async def _make_fd_io(read_fd: int, write_fd: int) -> Any: return _trio_gateway.staple_fd_stream(read_fd, write_fd) -def serve_popen_trio(id: str, execmodel: str = "thread") -> None: +def serve_popen_trio(id: str, execmodel: str = "thread", wait: str = "thread") -> None: """Serve a WorkerGateway over the stdio pipes (popen / ssh worker).""" from . import _trio_host @@ -266,10 +269,12 @@ def serve_popen_trio(id: str, execmodel: str = "thread") -> None: host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") host.start() io = host.call(_make_fd_io, read_fd, write_fd) - _run_worker(host, io, id, model) + _run_worker(host, io, id, model, wait) -def serve_socket_trio(id: str, execmodel: str, socket_fd: int) -> None: +def serve_socket_trio( + id: str, execmodel: str, socket_fd: int, wait: str = "thread" +) -> None: """Serve a WorkerGateway over an inherited socket fd. Used for the socketserver (an accepted TCP connection) and, in future, a @@ -282,7 +287,7 @@ def serve_socket_trio(id: str, execmodel: str, socket_fd: int) -> None: host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") host.start() io = host.call(_trio_host.adopt_socket, socket_fd) - _run_worker(host, io, id, model) + _run_worker(host, io, id, model, wait) def _rough_version(version: str) -> tuple[int, ...]: @@ -341,9 +346,18 @@ def _main() -> None: config = json.loads(ns.config) _check_version(config["coordinator_version"]) if ns.socket_fd is not None: - serve_socket_trio(config["id"], config["execmodel"], ns.socket_fd) + serve_socket_trio( + config["id"], + config["execmodel"], + ns.socket_fd, + wait=config.get("wait", "thread"), + ) else: - serve_popen_trio(id=config["id"], execmodel=config["execmodel"]) + serve_popen_trio( + id=config["id"], + execmodel=config["execmodel"], + wait=config.get("wait", "thread"), + ) if __name__ == "__main__": diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index b1d3ed46..bec4c778 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -43,6 +43,8 @@ def __init__(self, io: IO, spec: XSpec) -> None: """ super().__init__(io=io, id=spec.id, _startcount=1) self.spec = spec + if spec.wait: + self._wait_backend = spec.wait @property def remoteaddress(self) -> str: diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 47bf7d32..6a286302 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -11,7 +11,6 @@ from __future__ import annotations -import abc import builtins import os import queue as _queue @@ -31,7 +30,10 @@ from typing import cast from typing import overload +from ._boundary import Flag from ._boundary import Mailbox +from ._boundary import Wakener +from ._boundary import make_wakener class WriteIO(Protocol): @@ -58,74 +60,23 @@ def wait(self) -> int | None: ... def kill(self) -> None: ... -class Event(Protocol): - """Protocol for types which look like threading.Event.""" - - def is_set(self) -> bool: ... - - def set(self) -> None: ... - - def clear(self) -> None: ... - - def wait(self, timeout: float | None = None) -> bool: ... +class ExecModel: + """Deprecated preset name for an execution model. + The machinery behind execution models was retired: protocol IO always + runs on the Trio host and blocking waits go through the boundary kit's + wakeners (``execnet._boundary``); the name maps onto the worker config + axes (``loop=`` / ``exec=`` / ``wait=``). The stdlib-delegating + members stay for API compatibility (pytest-xdist builds its test queue + on ``execmodel.RLock``/``Event``) -- every preset is thread-shaped. + """ -class ExecModel(metaclass=abc.ABCMeta): - @property - @abc.abstractmethod - def backend(self) -> str: - raise NotImplementedError() + def __init__(self, backend: str) -> None: + self.backend = backend def __repr__(self) -> str: return "" % self.backend - @property - @abc.abstractmethod - def queue(self): - raise NotImplementedError() - - @property - @abc.abstractmethod - def subprocess(self): - raise NotImplementedError() - - @property - @abc.abstractmethod - def socket(self): - raise NotImplementedError() - - @abc.abstractmethod - def start(self, func, args=()) -> None: - raise NotImplementedError() - - @abc.abstractmethod - def get_ident(self) -> int: - raise NotImplementedError() - - @abc.abstractmethod - def sleep(self, delay: float) -> None: - raise NotImplementedError() - - @abc.abstractmethod - def fdopen(self, fd, mode, bufsize=1, closefd=True): - raise NotImplementedError() - - @abc.abstractmethod - def Lock(self): - raise NotImplementedError() - - @abc.abstractmethod - def RLock(self): - raise NotImplementedError() - - @abc.abstractmethod - def Event(self) -> Event: - raise NotImplementedError() - - -class ThreadExecModel(ExecModel): - backend = "thread" - @property def queue(self): import queue @@ -160,200 +111,24 @@ def start(self, func, args=()) -> None: _thread.start_new_thread(func, args) def fdopen(self, fd, mode, bufsize=1, closefd=True): - import os - return os.fdopen(fd, mode, bufsize, encoding="utf-8", closefd=closefd) def Lock(self): - import threading - return threading.RLock() def RLock(self): - import threading - return threading.RLock() - def Event(self): - import threading - + def Event(self) -> threading.Event: return threading.Event() -class MainThreadOnlyExecModel(ThreadExecModel): - backend = "main_thread_only" - - def get_execmodel(backend: str | ExecModel) -> ExecModel: if isinstance(backend, ExecModel): return backend - if backend == "thread": - return ThreadExecModel() - elif backend == "main_thread_only": - return MainThreadOnlyExecModel() - else: - raise ValueError(f"unknown execmodel {backend!r}") - - -class Reply: - """Provide access to the result of a function execution that got dispatched - through WorkerPool.spawn().""" - - def __init__(self, task, threadmodel: ExecModel) -> None: - self.task = task - self._result_ready = threadmodel.Event() - self.running = True - - def get(self, timeout: float | None = None): - """get the result object from an asynchronous function execution. - if the function execution raised an exception, - then calling get() will reraise that exception - including its traceback. - """ - self.waitfinish(timeout) - try: - return self._result - except AttributeError: - raise self._exc from None - - def waitfinish(self, timeout: float | None = None) -> None: - if not self._result_ready.wait(timeout): - raise OSError(f"timeout waiting for {self.task!r}") - - def run(self) -> None: - func, args, kwargs = self.task - try: - try: - self._result = func(*args, **kwargs) - except BaseException as exc: - self._exc = exc - finally: - self._result_ready.set() - self.running = False - - -class WorkerPool: - """A WorkerPool allows to spawn function executions - to threads, returning a reply object on which you - can ask for the result (and get exceptions reraised). - - This implementation allows the main thread to integrate - itself into performing function execution through - calling integrate_as_primary_thread() which will return - when the pool received a trigger_shutdown(). - - By default allows unlimited number of spawns. - """ - - _primary_thread_task: Reply | None - - def __init__(self, execmodel: ExecModel, hasprimary: bool = False) -> None: - self.execmodel = execmodel - self._running_lock = self.execmodel.Lock() - self._running: set[Reply] = set() - self._shuttingdown = False - self._waitall_events: list[Event] = [] - if hasprimary: - if self.execmodel.backend not in ("thread", "main_thread_only"): - raise ValueError("hasprimary=True requires thread model") - self._primary_thread_task_ready: Event | None = self.execmodel.Event() - else: - self._primary_thread_task_ready = None - - def integrate_as_primary_thread(self) -> None: - """Integrate the thread with which we are called as a primary - thread for executing functions triggered with spawn().""" - assert self.execmodel.backend in ("thread", "main_thread_only"), self.execmodel - primary_thread_task_ready = self._primary_thread_task_ready - assert primary_thread_task_ready is not None - # interacts with code at REF1 - while 1: - primary_thread_task_ready.wait() - reply = self._primary_thread_task - if reply is None: # trigger_shutdown() woke us up - break - self._perform_spawn(reply) - # we are concurrent with trigger_shutdown and spawn - with self._running_lock: - if self._shuttingdown: - break - # Only clear if _try_send_to_primary_thread has not - # yet set the next self._primary_thread_task reply - # after waiting for this one to complete. - if reply is self._primary_thread_task: - primary_thread_task_ready.clear() - - def trigger_shutdown(self) -> None: - with self._running_lock: - self._shuttingdown = True - if self._primary_thread_task_ready is not None: - self._primary_thread_task = None - self._primary_thread_task_ready.set() - - def active_count(self) -> int: - return len(self._running) - - def _perform_spawn(self, reply: Reply) -> None: - reply.run() - with self._running_lock: - self._running.remove(reply) - if not self._running: - while self._waitall_events: - waitall_event = self._waitall_events.pop() - waitall_event.set() - - def _try_send_to_primary_thread(self, reply: Reply) -> bool: - # REF1 in 'thread' model we give priority to running in main thread - # note that we should be called with _running_lock hold - primary_thread_task_ready = self._primary_thread_task_ready - if primary_thread_task_ready is not None: - if not primary_thread_task_ready.is_set(): - self._primary_thread_task = reply - # wake up primary thread - primary_thread_task_ready.set() - return True - elif ( - self.execmodel.backend == "main_thread_only" - and self._primary_thread_task is not None - ): - self._primary_thread_task.waitfinish() - self._primary_thread_task = reply - # wake up primary thread (it's okay if this is already set - # because we waited for the previous task to finish above - # and integrate_as_primary_thread will not clear it when - # it enters self._running_lock if it detects that a new - # task is available) - primary_thread_task_ready.set() - return True - return False - - def spawn(self, func, *args, **kwargs) -> Reply: - """Asynchronously dispatch func(*args, **kwargs) and return a Reply.""" - reply = Reply((func, args, kwargs), self.execmodel) - with self._running_lock: - if self._shuttingdown: - raise ValueError("pool is shutting down") - self._running.add(reply) - if not self._try_send_to_primary_thread(reply): - self.execmodel.start(self._perform_spawn, (reply,)) - return reply - - def terminate(self, timeout: float | None = None) -> bool: - """Trigger shutdown and wait for completion of all executions.""" - self.trigger_shutdown() - return self.waitall(timeout=timeout) - - def waitall(self, timeout: float | None = None) -> bool: - """Wait until all active spawns have finished executing.""" - with self._running_lock: - if not self._running: - return True - # if a Reply still runs, we let run_and_release - # signal us -- note that we are still holding the - # _running_lock to avoid race conditions - my_waitall_event = self.execmodel.Event() - self._waitall_events.append(my_waitall_event) - return my_waitall_event.wait(timeout=timeout) + if backend in ("thread", "main_thread_only"): + return ExecModel(backend) + raise ValueError(f"unknown execmodel {backend!r}") sysex = (KeyboardInterrupt, SystemExit) @@ -578,11 +353,11 @@ def __init__(self, gateway: BaseGateway, id: int) -> None: self._strconfig = getattr(gateway, "_strconfig", (True, False)) self.id = id # serialized payloads (or ENDMARKER); None once a callback is set - self._mailbox: Mailbox[Any] | None = Mailbox() + self._mailbox: Mailbox[Any] | None = Mailbox(gateway._new_wakener()) self._callback: Callable[[Any], Any] | None = None self._endmarker: object = NO_ENDMARKER_WANTED self._closed = False - self._receiveclosed = threading.Event() + self._receiveclosed = Flag(gateway._new_wakener()) self._remoteerrors: list[RemoteError] = [] def _trace(self, *msg: object) -> None: @@ -1028,6 +803,9 @@ class BaseGateway: _trio_session: Any = None # Set by the receiver on EOF without a prior termination message. _error: BaseException | None = None + #: wait= axis: which wakener backend this gateway's blocking waits + #: park on (channels, write-acks, join) + _wait_backend: str = "thread" def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel @@ -1051,6 +829,10 @@ def _attach_trio_session(self, session: Any) -> None: for channel in self._channelfactory.channels(): session.bind_sync_channel(channel) + def _new_wakener(self) -> Wakener: + """A fresh wakener for one blocking-wait carrier (wait= axis).""" + return make_wakener(self._wait_backend) + def _bind_channel(self, channel: Channel) -> None: """Divert the session's inbound routing for ``channel.id`` to it.""" session = self._trio_session @@ -1134,7 +916,7 @@ class WorkerGateway(BaseGateway): _trio_exec: Any = None # The exec pool (a TrioWorkerExec duck-typed as WorkerPool for STATUS). _execpool: Any = None - _executetask_complete: Event | None = None + _executetask_complete: threading.Event | None = None def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: trio_exec = self._trio_exec diff --git a/src/execnet/multi.py b/src/execnet/multi.py index ca3b69dd..ba73bf62 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -7,6 +7,9 @@ from __future__ import annotations import atexit +import queue +import threading +import time import types from collections.abc import Callable from collections.abc import Iterable @@ -20,9 +23,9 @@ from typing import TypeAlias from typing import overload +from ._boundary import wakener_names from .gateway_base import Channel from .gateway_base import ExecModel -from .gateway_base import WorkerPool from .gateway_base import get_execmodel from .gateway_base import trace from .xspec import XSpec @@ -152,6 +155,7 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: id= specifies the gateway id python= specifies which python interpreter to execute execmodel=model 'thread' or 'main_thread_only' execution model + wait=backend wakener for blocking waits ('thread' default) chdir= specifies to which directory to change nice= specifies process priority of new process env:NAME=value specifies a remote environment variable setting. @@ -165,6 +169,10 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: self.allocate_id(spec) if spec.execmodel is None: spec.execmodel = self.remote_execmodel.backend + if spec.wait is not None and spec.wait not in wakener_names(): + raise ValueError( + f"unknown wait backend {spec.wait!r} (known: {wakener_names()})" + ) from . import _trio_host if not (spec.socket or spec.via or spec.ssh or spec.vagrant_ssh or spec.popen): @@ -313,10 +321,10 @@ def make_receive_queue(self, endmarker: object = NO_ENDMARKER_WANTED): try: return self._queue # type: ignore[has-type] except AttributeError: - self._queue = None + self._queue: queue.Queue[tuple[Channel, Any]] | None = None for ch in self._channels: if self._queue is None: - self._queue = ch.gateway.execmodel.queue.Queue() + self._queue = queue.Queue() def putreceived(obj, channel: Channel = ch) -> None: self._queue.put((channel, obj)) # type: ignore[union-attr] @@ -351,32 +359,45 @@ def safe_terminate( """Run terminate/kill pairs in parallel with a hard wait bound. Each termfunc is given ``timeout``. If it does not finish, killfunc runs. - Waiting for the worker pool is also bounded so a stuck kill cannot hang - the caller forever (see issues #43 / #221). + The final wait is also bounded so a stuck kill cannot hang the caller + forever (see issues #43 / #221). ``execmodel`` is accepted for + backward compatibility and unused (daemon threads do the waiting). """ - workerpool = WorkerPool(execmodel) + errors: list[BaseException] = [] def termkill(termfunc: TermKillFunc, killfunc: TermKillFunc) -> None: - termreply = workerpool.spawn(termfunc) - try: - termreply.get(timeout=timeout) - except OSError: + term_done = threading.Event() + term_errors: list[BaseException] = [] + + def run_term() -> None: + try: + termfunc() + except BaseException as exc: + term_errors.append(exc) + finally: + term_done.set() + + threading.Thread(target=run_term, daemon=True).start() + if not term_done.wait(timeout): killfunc() + return + if term_errors: + errors.append(term_errors[0]) - replylist = [ - workerpool.spawn(termkill, termfunc, killfunc) - for termfunc, killfunc in list_of_paired_functions + threads = [ + threading.Thread(target=termkill, args=pair, daemon=True) + for pair in list_of_paired_functions ] + for thread in threads: + thread.start() # Allow term timeout plus a kill attempt; never block indefinitely. wait_timeout = None if timeout is None else timeout * 2 - for reply in replylist: - try: - reply.waitfinish(timeout=wait_timeout) - except OSError: - # termkill still running (typically stuck in killfunc). - continue - reply.get() # propagate worker exceptions, if any - workerpool.waitall(timeout=wait_timeout) + deadline = None if wait_timeout is None else time.monotonic() + wait_timeout + for thread in threads: + remaining = None if deadline is None else max(0, deadline - time.monotonic()) + thread.join(remaining) + if errors: + raise errors[0] default_group = Group() diff --git a/src/execnet/xspec.py b/src/execnet/xspec.py index 0559ed8c..afc58f4c 100644 --- a/src/execnet/xspec.py +++ b/src/execnet/xspec.py @@ -29,6 +29,7 @@ class XSpec: ssh_config: str | None = None vagrant_ssh: str | None = None via: str | None = None + wait: str | None = None def __init__(self, string: str) -> None: self._spec = string diff --git a/testing/conftest.py b/testing/conftest.py index 4d97bf35..7c3ed7a6 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -5,6 +5,7 @@ from collections.abc import Callable from collections.abc import Generator from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor from functools import lru_cache import pytest @@ -12,7 +13,6 @@ import execnet from execnet.gateway import Gateway from execnet.gateway_base import ExecModel -from execnet.gateway_base import WorkerPool from execnet.gateway_base import get_execmodel collect_ignore = ["build", "doc/_build"] @@ -184,5 +184,6 @@ def execmodel(request: pytest.FixtureRequest) -> ExecModel: @pytest.fixture -def pool(execmodel: ExecModel) -> WorkerPool: - return WorkerPool(execmodel=execmodel) +def executor() -> Iterator[ThreadPoolExecutor]: + with ThreadPoolExecutor() as tpe: + yield tpe diff --git a/testing/test_basics.py b/testing/test_basics.py index 29ceef19..c58e4728 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -71,15 +71,23 @@ def standalone_gateway_base_source() -> str: """gateway_base's source as a self-contained script. The module imports the trio-free boundary kit relatively; for the - run-on-any-python checks the kit source is inlined at the import site - (minus its own future import, which must stay file-leading). + run-on-any-python checks the kit source is inlined in place of those + imports (minus its own future import, which must stay file-leading). """ boundary_source = inspect.getsource(_boundary).replace( "from __future__ import annotations\n", "" ) - return inspect.getsource(gateway_base).replace( - "from ._boundary import Mailbox\n", boundary_source - ) + lines = [] + inlined = False + for line in inspect.getsource(gateway_base).splitlines(keepends=True): + if line.startswith("from ._boundary import"): + if not inlined: + inlined = True + lines.append(boundary_source) + else: + lines.append(line) + assert inlined + return "".join(lines) IO_MESSAGE_EXTRA_SOURCE = """ @@ -341,6 +349,7 @@ class TestPureChannel: def fac(self, execmodel: ExecModel) -> ChannelFactory: class FakeGateway: _trio_session = None + _new_wakener = staticmethod(_boundary.ThreadWakener) def _trace(self, *args) -> None: pass diff --git a/testing/test_channel.py b/testing/test_channel.py index ba02b57b..f06b8632 100644 --- a/testing/test_channel.py +++ b/testing/test_channel.py @@ -4,6 +4,7 @@ from __future__ import annotations +import queue import time import pytest @@ -270,14 +271,14 @@ def test_channel_endmarker_callback(self, gw: Gateway) -> None: assert l[3] == 999 def test_channel_endmarker_callback_error(self, gw: Gateway) -> None: - q = gw.execmodel.queue.Queue() + q: queue.Queue[object] = queue.Queue() channel = gw.remote_exec( source=""" raise ValueError() """ ) channel.setcallback(q.put, endmarker=999) - val = q.get(TESTTIMEOUT) + val = q.get(timeout=TESTTIMEOUT) assert val == 999 err = channel._getremoteerror() assert err diff --git a/testing/test_gateway.py b/testing/test_gateway.py index b7098773..da519d4e 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -599,7 +599,7 @@ def test_main_thread_only_concurrent_remote_exec_deadlock( import threading channel.send(threading.current_thread() is threading.main_thread()) # Wait forever, ensuring that the deadlock case triggers. - channel.gateway.execmodel.Event().wait() + threading.Event().wait() """ ) ) diff --git a/testing/test_multi.py b/testing/test_multi.py index e0390298..2f1b634e 100644 --- a/testing/test_multi.py +++ b/testing/test_multi.py @@ -5,6 +5,7 @@ from __future__ import annotations import gc +import threading from collections.abc import Callable from time import sleep @@ -308,12 +309,12 @@ def test_safe_terminate_does_not_hang_when_kill_blocks( out, so a blocking killfunc made Group.terminate() hang indefinitely (seen via pytest-xdist teardown). """ - kill_started = execmodel.Event() - release_kill = execmodel.Event() + kill_started = threading.Event() + release_kill = threading.Event() other_killed: list[int] = [] def term_slow() -> None: - execmodel.sleep(10) + sleep(10) def kill_hang() -> None: kill_started.set() diff --git a/testing/test_termination.py b/testing/test_termination.py index ca119304..01c57427 100644 --- a/testing/test_termination.py +++ b/testing/test_termination.py @@ -1,10 +1,12 @@ import os import pathlib +import queue import shutil import signal import subprocess import sys from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor import pytest from test_gateway import TESTTIMEOUT @@ -12,7 +14,6 @@ import execnet from execnet.gateway import Gateway from execnet.gateway_base import ExecModel -from execnet.gateway_base import WorkerPool execnetdir = pathlib.Path(execnet.__file__).parent.parent @@ -23,7 +24,7 @@ def test_exit_blocked_worker_execution_gateway( - anypython: str, makegateway: Callable[[str], Gateway], pool: WorkerPool + anypython: str, makegateway: Callable[[str], Gateway], executor: ThreadPoolExecutor ) -> None: gateway = makegateway("popen//python=%s" % anypython) gateway.remote_exec( @@ -37,8 +38,7 @@ def doit() -> int: gateway.exit() return 17 - reply = pool.spawn(doit) - x = reply.get(timeout=5.0) + x = executor.submit(doit).result(timeout=5.0) assert x == 17 @@ -48,7 +48,7 @@ def test_endmarker_delivery_on_remote_killterm( if execmodel.backend not in ("thread", "main_thread_only"): pytest.xfail("test and execnet not compatible to greenlets yet") gw = makegateway("popen") - q = execmodel.queue.Queue() + q: queue.Queue[object] = queue.Queue() channel = gw.remote_exec( source=""" import os, time @@ -60,7 +60,7 @@ def test_endmarker_delivery_on_remote_killterm( assert isinstance(pid, int) os.kill(pid, signal.SIGTERM) channel.setcallback(q.put, endmarker=999) - val = q.get(TESTTIMEOUT) + val = q.get(timeout=TESTTIMEOUT) assert val == 999 err = channel._getremoteerror() assert isinstance(err, EOFError) @@ -113,10 +113,8 @@ def test_terminate_implicit_does_trykill( pytester: pytest.Pytester, anypython: str, capfd: pytest.CaptureFixture[str], - pool: WorkerPool, + executor: ThreadPoolExecutor, ) -> None: - if pool.execmodel.backend not in ("thread", "main_thread_only"): - pytest.xfail("only os threading model supported") if sys.version_info >= (3, 12): pytest.xfail( "since python3.12 this test triggers RuntimeError: can't create new thread at interpreter shutdown" @@ -149,8 +147,7 @@ def flush(self): # sync with start-up assert popen.stdout is not None popen.stdout.readline() - reply = pool.spawn(popen.communicate) - reply.get(timeout=50) + executor.submit(popen.communicate).result(timeout=50) _out, err = capfd.readouterr() lines = [x for x in err.splitlines() if "*sys-package" not in x] assert not lines or "Killed" in err diff --git a/testing/test_threadpool.py b/testing/test_threadpool.py deleted file mode 100644 index 510ae020..00000000 --- a/testing/test_threadpool.py +++ /dev/null @@ -1,222 +0,0 @@ -import os -from pathlib import Path - -import pytest - -from execnet.gateway_base import ExecModel -from execnet.gateway_base import WorkerPool - - -def test_execmodel(execmodel: ExecModel, tmp_path: Path) -> None: - assert execmodel.backend - p = tmp_path / "somefile" - p.write_text("content") - fd = os.open(p, os.O_RDONLY) - f = execmodel.fdopen(fd, "r") - assert f.read() == "content" - f.close() - - -def test_execmodel_basic_attrs(execmodel: ExecModel) -> None: - m = execmodel - assert callable(m.start) - assert m.get_ident() - - -def test_simple(pool: WorkerPool) -> None: - reply = pool.spawn(lambda: 42) - assert reply.get() == 42 - - -def test_some(pool: WorkerPool, execmodel: ExecModel) -> None: - q = execmodel.queue.Queue() - num = 4 - - def f(i: int) -> None: - q.put(i) - while q.qsize(): - execmodel.sleep(0.01) - - for i in range(num): - pool.spawn(f, i) - for i in range(num): - q.get() - # assert len(pool._running) == 4 - assert pool.waitall(timeout=1.0) - # execmodel.sleep(1) helps on windows? - assert len(pool._running) == 0 - - -def test_running_semnatics(pool: WorkerPool, execmodel: ExecModel) -> None: - q = execmodel.queue.Queue() - - def first() -> None: - q.get() - - reply = pool.spawn(first) - assert reply.running - assert pool.active_count() == 1 - q.put(1) - assert pool.waitall() - assert pool.active_count() == 0 - assert not reply.running - - -def test_waitfinish_on_reply(pool: WorkerPool) -> None: - l = [] - reply = pool.spawn(lambda: l.append(1)) - reply.waitfinish() - assert l == [1] - reply = pool.spawn(lambda: 0 / 0) - reply.waitfinish() # no exception raised - pytest.raises(ZeroDivisionError, reply.get) - - -@pytest.mark.xfail(reason="WorkerPool does not implement limited size") -def test_limited_size(execmodel: ExecModel) -> None: - pool = WorkerPool(execmodel, size=1) # type: ignore[call-arg] - q = execmodel.queue.Queue() - q2 = execmodel.queue.Queue() - q3 = execmodel.queue.Queue() - - def first() -> None: - q.put(1) - q2.get() - - pool.spawn(first) - assert q.get() == 1 - - def second() -> None: - q3.put(3) - - # we spawn a second pool to spawn the second function - # which should block - pool2 = WorkerPool(execmodel) - pool2.spawn(pool.spawn, second) - assert not pool2.waitall(1.0) - assert q3.qsize() == 0 - q2.put(2) - assert pool2.waitall() - assert pool.waitall() - - -def test_get(pool: WorkerPool) -> None: - def f() -> int: - return 42 - - reply = pool.spawn(f) - result = reply.get() - assert result == 42 - - -def test_get_timeout(execmodel: ExecModel, pool: WorkerPool) -> None: - def f() -> int: - execmodel.sleep(0.2) - return 42 - - reply = pool.spawn(f) - with pytest.raises(IOError): - reply.get(timeout=0.01) - - -def test_get_excinfo(pool: WorkerPool) -> None: - def f() -> None: - raise ValueError("42") - - reply = pool.spawn(f) - with pytest.raises(ValueError): - reply.get(1.0) - with pytest.raises(ValueError): - reply.get(1.0) - - -def test_waitall_timeout(pool: WorkerPool, execmodel: ExecModel) -> None: - q = execmodel.queue.Queue() - - def f() -> None: - q.get() - - reply = pool.spawn(f) - assert not pool.waitall(0.01) - q.put(None) - reply.get(timeout=1.0) - assert pool.waitall(timeout=0.1) - - -@pytest.mark.skipif(not hasattr(os, "dup"), reason="no os.dup") -def test_pool_clean_shutdown( - pool: WorkerPool, capfd: pytest.CaptureFixture[str] -) -> None: - q = pool.execmodel.queue.Queue() - - def f() -> None: - q.get() - - pool.spawn(f) - assert not pool.waitall(timeout=1.0) - pool.trigger_shutdown() - with pytest.raises(ValueError): - pool.spawn(f) - - def wait_then_put() -> None: - pool.execmodel.sleep(0.1) - q.put(1) - - pool.execmodel.start(wait_then_put) - assert pool.waitall() - _out, err = capfd.readouterr() - assert err == "" - - -def test_primary_thread_integration(execmodel: ExecModel) -> None: - if execmodel.backend not in ("thread", "main_thread_only"): - with pytest.raises(ValueError): - WorkerPool(execmodel=execmodel, hasprimary=True) - return - pool = WorkerPool(execmodel=execmodel, hasprimary=True) - queue = execmodel.queue.Queue() - - def do_integrate() -> None: - queue.put(execmodel.get_ident()) - pool.integrate_as_primary_thread() - - execmodel.start(do_integrate) - - def func() -> None: - queue.put(execmodel.get_ident()) - - pool.spawn(func) - ident1 = queue.get() - ident2 = queue.get() - assert ident1 == ident2 - pool.terminate() - - -def test_primary_thread_integration_shutdown(execmodel: ExecModel) -> None: - if execmodel.backend not in ("thread", "main_thread_only"): - pytest.skip("can only run with threading") - pool = WorkerPool(execmodel=execmodel, hasprimary=True) - queue = execmodel.queue.Queue() - - def do_integrate() -> None: - queue.put(execmodel.get_ident()) - pool.integrate_as_primary_thread() - - execmodel.start(do_integrate) - queue.get() - - queue2 = execmodel.queue.Queue() - - def get_two() -> None: - queue.put(execmodel.get_ident()) - queue2.get() - - reply = pool.spawn(get_two) - # make sure get_two is running and blocked on queue2 - queue.get() - # then shut down - pool.trigger_shutdown() - # and let get_two finish - queue2.put(1) - reply.get() - assert pool.waitall(5.0) diff --git a/testing/test_xspec.py b/testing/test_xspec.py index 661e50af..850acbe1 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -119,6 +119,14 @@ class TestMakegateway: def test_no_type(self, makegateway: Callable[[str], Gateway]) -> None: pytest.raises(ValueError, lambda: makegateway("hello")) + def test_wait_axis(self, makegateway: Callable[[str], Gateway]) -> None: + with pytest.raises(ValueError, match="unknown wait backend"): + makegateway("popen//wait=nope") + gw = makegateway("popen//wait=thread") + assert gw._wait_backend == "thread" + channel = gw.remote_exec("channel.send(channel.gateway._wait_backend)") + assert channel.receive() == "thread" + @skip_win_pypy def test_popen_default(self, makegateway: Callable[[str], Gateway]) -> None: gw = makegateway("") From 845510e64a8e9e8542cae7234f6197ec812527f7 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:19:08 +0200 Subject: [PATCH 38/91] feat: add execnet.aio -- the asyncio-native API over the Trio host P4 of the boundary rethink. A fourth public namespace mirroring execnet.trio's surface for asyncio applications: Group/Gateway/Channel are async context managers / awaitables whose operations run as tasks on a dedicated Trio host thread and resolve asyncio futures via loop.call_soon_threadsafe (a _HostBridge per group). All transports work because the protocol engine is the same AsyncGroup; no anyio port and no per-call executor threads. Received channel references arrive wrapped as aio Channels. Known v1 limitation (documented): cancelling a bridged await abandons the operation asyncio-side only; the host-side task runs to completion. `import execnet` stays free of trio/asyncio -- execnet.aio loads lazily like execnet.trio. Co-Authored-By: Claude Fable 5 --- src/execnet/__init__.py | 8 +- src/execnet/aio.py | 321 ++++++++++++++++++++++++++++++++++++++++ testing/test_aio.py | 116 +++++++++++++++ 3 files changed, 442 insertions(+), 3 deletions(-) create mode 100644 src/execnet/aio.py create mode 100644 testing/test_aio.py diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 2108f4b2..66c61821 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -4,14 +4,16 @@ pure python lib for connecting to local and remote Python Interpreters. -Three public namespaces: +Four public namespaces: * :mod:`execnet.sync` — the blocking API; the top-level ``execnet.*`` names below are aliases into it. * :mod:`execnet.trio` — the trio-native API, awaited inside your own ``trio.run``. +* :mod:`execnet.aio` — the asyncio-native API, bridged over a Trio + host thread. * :mod:`execnet.portal` — cross-thread / cross-loop communication - primitives shared by both. + primitives shared by all of them. (c) 2012, Holger Krekel and others """ @@ -66,7 +68,7 @@ def __getattr__(name: str) -> Any: # Lazy namespace modules: keep ``import execnet`` from loading the # trio event loop machinery until it is actually used. - if name in ("trio", "portal"): + if name in ("aio", "portal", "trio"): import importlib return importlib.import_module(f".{name}", __name__) diff --git a/src/execnet/aio.py b/src/execnet/aio.py new file mode 100644 index 00000000..e3004a28 --- /dev/null +++ b/src/execnet/aio.py @@ -0,0 +1,321 @@ +"""The asyncio-native execnet API, bridged over a Trio host thread. + +Everything here is awaited inside your own asyncio event loop:: + + import asyncio + import execnet.aio + + async def main(): + async with execnet.aio.Group() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(6 * 7)") + print(await channel.receive()) + + asyncio.run(main()) + +Protocol IO keeps running on a dedicated Trio host thread (the same +engine as the blocking and trio-native APIs, all transports included); +each awaited operation runs as a task on that host and resolves an +asyncio future via ``loop.call_soon_threadsafe``. No anyio port and no +executor threads per call. + +Note: cancelling a bridged await abandons the operation on the asyncio +side only -- the host-side task runs to completion (a cancelled +``receive`` may still consume the next item). + +The serialization helpers and error types are shared with +:mod:`execnet.sync` and :mod:`execnet.trio`. +""" + +from __future__ import annotations + +import asyncio +import functools +import types +from collections.abc import AsyncIterator +from collections.abc import Awaitable +from collections.abc import Callable +from contextlib import asynccontextmanager +from contextlib import suppress +from typing import TYPE_CHECKING +from typing import Any +from typing import TypeVar + +import trio + +from ._trio_gateway import AsyncChannel +from ._trio_gateway import AsyncGateway +from ._trio_gateway import AsyncGroup +from ._trio_host import TrioHost +from .gateway_base import DataFormatError +from .gateway_base import DumpError +from .gateway_base import HostNotFound +from .gateway_base import LoadError +from .gateway_base import RemoteError +from .gateway_base import TimeoutError +from .gateway_base import dump +from .gateway_base import dumps +from .gateway_base import load +from .gateway_base import loads +from .xspec import XSpec + +if TYPE_CHECKING: + from typing_extensions import Self + +__all__ = [ + "Channel", + "DataFormatError", + "DumpError", + "Gateway", + "Group", + "HostNotFound", + "LoadError", + "RemoteError", + "TimeoutError", + "XSpec", + "dump", + "dumps", + "load", + "loads", + "open_popen_gateway", +] + +T = TypeVar("T") + + +class _HostBridge: + """Await trio-native coroutines on a TrioHost from asyncio.""" + + def __init__(self, host: TrioHost) -> None: + self._host = host + + async def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: + loop = asyncio.get_running_loop() + future: asyncio.Future[T] = loop.create_future() + + def resolve(result: Any, error: BaseException | None) -> None: + if future.cancelled(): + return + if error is not None: + future.set_exception(error) + else: + future.set_result(result) + + def post_result(result: Any, error: BaseException | None) -> None: + # The asyncio loop may already be gone at interpreter/test + # teardown; the result is undeliverable then. + with suppress(RuntimeError): + loop.call_soon_threadsafe(resolve, result, error) + + async def runner() -> None: + try: + result = await async_fn(*args) + except trio.Cancelled: + # host shutdown: the nursery cancel must propagate + post_result(None, RuntimeError("execnet aio host was shut down")) + raise + except BaseException as exc: + post_result(None, exc) + else: + post_result(result, None) + + def spawn() -> None: + self._host.start_soon(runner) + + try: + self._host.portal.post(spawn) + except trio.RunFinishedError: + raise RuntimeError("execnet aio group is not running") from None + return await future + + +class _HostedGroup(AsyncGroup): + """Trio AsyncGroup living as a task on the host nursery.""" + + def __init__(self, termination_timeout: float) -> None: + super().__init__(termination_timeout) + self.shutdown = trio.Event() + self.finished = trio.Event() + + async def run(self, task_status: trio.TaskStatus[_HostedGroup]) -> None: + try: + async with self: + task_status.started(self) + await self.shutdown.wait() + finally: + self.finished.set() + + +class Channel: + """asyncio facade over a trio-native :class:`AsyncChannel`.""" + + RemoteError = RemoteError + TimeoutError = TimeoutError + + def __init__(self, bridge: _HostBridge, channel: AsyncChannel) -> None: + self._bridge = bridge + self._channel = channel + + @property + def id(self) -> int: + return self._channel.id + + def __repr__(self) -> str: + return f"" + + def isclosed(self) -> bool: + """Return True if the channel is closed for sending.""" + return self._channel.isclosed() + + async def send(self, item: object) -> None: + """Serialize ``item`` and send it to the other side.""" + await self._bridge.call(self._channel.send, item) + + async def receive(self, timeout: float | None = None) -> Any: + """Receive the next item sent from the other side. + + EOFError once the peer closed or sent EOF, RemoteError for a peer + close-with-error, TimeoutError after ``timeout`` seconds. A + received channel reference arrives as an :class:`~execnet.aio.Channel`. + """ + result = await self._bridge.call(self._channel.receive, timeout) + if isinstance(result, AsyncChannel): + return Channel(self._bridge, result) + return result + + async def send_eof(self) -> None: + """Signal that no more items follow (peer keeps its send side).""" + await self._bridge.call(self._channel.send_eof) + + async def aclose(self, error: str | None = None) -> None: + """Close the channel; ``error`` reaches the peer as a RemoteError.""" + await self._bridge.call(self._channel.aclose, error) + + async def wait_closed(self) -> None: + """Wait until the peer closed or sent EOF; reraise remote errors.""" + await self._bridge.call(self._channel.wait_closed) + + def __aiter__(self) -> Channel: + return self + + async def __anext__(self) -> Any: + try: + return await self.receive() + except EOFError: + raise StopAsyncIteration from None + + +class Gateway: + """asyncio facade over a trio-native :class:`AsyncGateway`.""" + + def __init__(self, bridge: _HostBridge, gateway: AsyncGateway) -> None: + self._bridge = bridge + self._gateway = gateway + + @property + def id(self) -> str: + return self._gateway.id + + @property + def remoteaddress(self) -> str | None: + return self._gateway.remoteaddress + + def __repr__(self) -> str: + return f"" + + async def remote_exec( + self, + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + **kwargs: object, + ) -> Channel: + """Connect a new channel to remote execution of ``source``. + + Accepts the same source kinds as ``Gateway.remote_exec``: a source + string, a pure function called with ``channel`` and ``**kwargs``, + or a module. + """ + channel = await self._bridge.call( + functools.partial(self._gateway.remote_exec, source, **kwargs) + ) + return Channel(self._bridge, channel) + + async def terminate(self) -> None: + """Send GATEWAY_TERMINATE to the peer, then close this side.""" + await self._bridge.call(self._gateway.terminate) + + +class Group: + """asyncio-native gateway group over a dedicated Trio host thread. + + An async context manager mirroring :class:`execnet.trio.AsyncGroup`; + leaving the ``async with`` block terminates every gateway with the + same bounded contract, then stops the host thread. + """ + + def __init__(self, termination_timeout: float = 10.0) -> None: + self._termination_timeout = termination_timeout + self._host: TrioHost | None = None + self._bridge: _HostBridge | None = None + self._group: _HostedGroup | None = None + + def __repr__(self) -> str: + state = "running" if self._group is not None else "idle" + return f"" + + async def __aenter__(self) -> Self: + assert self._host is None, "group already entered" + loop = asyncio.get_running_loop() + host = TrioHost(name="execnet-aio-group") + # start() blocks until the host loop is ready; keep it off the + # asyncio loop thread. + await loop.run_in_executor(None, host.start) + self._host = host + self._bridge = _HostBridge(host) + + async def start_group() -> _HostedGroup: + # runs on the host loop + group = _HostedGroup(self._termination_timeout) + started: _HostedGroup = await host._nursery.start(group.run) + return started + + self._group = await self._bridge.call(start_group) + return self + + async def __aexit__(self, *exc_info: object) -> None: + group, bridge, host = self._group, self._bridge, self._host + self._group = self._bridge = None + if group is not None and bridge is not None: + + async def stop_group() -> None: + group.shutdown.set() + await group.finished.wait() + + with suppress(RuntimeError): + await bridge.call(stop_group) + if host is not None: + self._host = None + loop = asyncio.get_running_loop() + await loop.run_in_executor(None, functools.partial(host.stop, 5.0)) + + async def makegateway(self, spec: str | XSpec = "popen") -> Gateway: + """Create a gateway for ``spec`` served on the group's host. + + All transports are supported: popen (including uv-provisioned + ``python=``), ``ssh=``, ``vagrant_ssh=``, ``socket=`` (with + ``installvia=``), and ``via=`` sub-gateways. + """ + group, bridge = self._group, self._bridge + if group is None or bridge is None: + raise RuntimeError(f"{self!r} is not entered") + gateway = await bridge.call(group.makegateway, spec) + return Gateway(bridge, gateway) + + +@asynccontextmanager +async def open_popen_gateway(spec: str | XSpec = "popen") -> AsyncIterator[Gateway]: + """Spawn one popen worker and yield an asyncio Gateway to it. + + Convenience for a single-gateway :class:`Group`. + """ + async with Group() as group: + yield await group.makegateway(spec) diff --git a/testing/test_aio.py b/testing/test_aio.py new file mode 100644 index 00000000..6bdb7724 --- /dev/null +++ b/testing/test_aio.py @@ -0,0 +1,116 @@ +"""The asyncio-native API: execnet.aio bridged over the Trio host.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable +from typing import Any +from typing import TypeVar + +import pytest + +import execnet.aio + +T = TypeVar("T") + +TESTTIMEOUT = 30.0 + + +def run(main: Awaitable[T]) -> T: + return asyncio.run(asyncio.wait_for(main, TESTTIMEOUT)) + + +def test_popen_roundtrip() -> None: + async def main() -> None: + async with execnet.aio.Group() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(channel.receive() + 1)") + await channel.send(41) + assert await channel.receive() == 42 + await channel.wait_closed() + + run(main()) + + +def test_open_popen_gateway_iteration() -> None: + async def main() -> list[int]: + async with execnet.aio.open_popen_gateway() as gateway: + channel = await gateway.remote_exec( + "for i in range(4): channel.send(i * 2)" + ) + return [item async for item in channel] + + assert run(main()) == [0, 2, 4, 6] + + +def test_receive_timeout() -> None: + async def main() -> None: + async with execnet.aio.open_popen_gateway() as gateway: + channel = await gateway.remote_exec("channel.receive()") + with pytest.raises(channel.TimeoutError): + await channel.receive(timeout=0.05) + await channel.send(None) + + run(main()) + + +def test_remote_error() -> None: + async def main() -> None: + async with execnet.aio.open_popen_gateway() as gateway: + channel = await gateway.remote_exec("raise ValueError(17)") + with pytest.raises(execnet.aio.RemoteError, match="ValueError"): + await channel.receive() + + run(main()) + + +def test_channel_passing_wraps_aio() -> None: + async def main() -> None: + async with execnet.aio.open_popen_gateway() as gateway: + channel = await gateway.remote_exec( + """ + c = channel.gateway.newchannel() + channel.send(c) + c.send(42) + """ + ) + passed = await channel.receive() + assert isinstance(passed, execnet.aio.Channel) + assert await passed.receive() == 42 + + run(main()) + + +def test_multiple_gateways_and_send_each() -> None: + async def main() -> list[Any]: + async with execnet.aio.Group() as group: + gateways = [await group.makegateway("popen") for _ in range(2)] + channels = [ + await gw.remote_exec("channel.send(channel.receive() * 2)") + for gw in gateways + ] + for i, channel in enumerate(channels): + await channel.send(i + 1) + return [await channel.receive() for channel in channels] + + assert run(main()) == [2, 4] + + +def test_group_not_entered() -> None: + async def main() -> None: + group = execnet.aio.Group() + with pytest.raises(RuntimeError, match="not entered"): + await group.makegateway("popen") + + run(main()) + + +def test_terminate_gateway_explicitly() -> None: + async def main() -> None: + async with execnet.aio.Group() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(1)") + assert await channel.receive() == 1 + await gateway.terminate() + + run(main()) From b01d00937d0428794b736bb06c42b714ec852bd1 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:22:37 +0200 Subject: [PATCH 39/91] feat: gevent wait backend (wait=gevent) with opt-in tests P5, closing out the boundary rethink. execnet._gevent_support provides a GeventWakener whose notify() crosses from the trio host thread via a hub loop.async_ watcher (the one gevent primitive documented as thread-safe) while waits park only the calling greenlet. The hub-bound event/watcher are created lazily in the first waiter's hub so carriers can be constructed on the host loop; a pre-watcher notify is replayed, never lost. The boundary kit resolves wait=gevent by lazily importing the support module, so specs validate without gevent installed. Tests live behind the new `gevent` dependency group (pytest importorskip): wakener unit coverage plus gateway-level proof that channel.receive()/send() park the greenlet, not the hub -- no monkey-patching required. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 3 + src/execnet/_boundary.py | 12 +- src/execnet/_gevent_support.py | 76 ++++++++++++ src/execnet/aio.py | 1 + testing/test_gevent.py | 86 ++++++++++++++ uv.lock | 208 ++++++++++++++++++++++++++++++++- 6 files changed, 383 insertions(+), 3 deletions(-) create mode 100644 src/execnet/_gevent_support.py create mode 100644 testing/test_gevent.py diff --git a/pyproject.toml b/pyproject.toml index 4e94c938..aedd6b59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,9 @@ testing = [ dev = [ "pytest-xdist>=3.8.0", ] +gevent = [ + "gevent>=26.7.0", +] testing = [ "execnet[testing]", ] diff --git a/src/execnet/_boundary.py b/src/execnet/_boundary.py index ce211ae1..9c7c69f4 100644 --- a/src/execnet/_boundary.py +++ b/src/execnet/_boundary.py @@ -205,10 +205,14 @@ def wait(self, timeout: float | None = None) -> bool: # wait= axis: named wakener factories; each call returns a fresh instance # (carriers own their wakener exclusively, see Flag). Backends register -# here ("gevent", "asyncio") next to the built-in "thread". +# here next to the built-in "thread"; entries in the lazy table import +# their module (which registers itself) on first use. _WAKENER_FACTORIES: dict[str, Callable[[], Wakener]] = { "thread": ThreadWakener, } +_LAZY_WAKENER_MODULES: dict[str, str] = { + "gevent": "execnet._gevent_support", +} def register_wakener(name: str, factory: Callable[[], Wakener]) -> None: @@ -217,11 +221,15 @@ def register_wakener(name: str, factory: Callable[[], Wakener]) -> None: def wakener_names() -> list[str]: - return list(_WAKENER_FACTORIES) + return sorted(set(_WAKENER_FACTORIES) | set(_LAZY_WAKENER_MODULES)) def make_wakener(name: str) -> Wakener: """Create a fresh wakener for the named wait backend.""" + if name not in _WAKENER_FACTORIES and name in _LAZY_WAKENER_MODULES: + import importlib + + importlib.import_module(_LAZY_WAKENER_MODULES[name]) try: factory = _WAKENER_FACTORIES[name] except KeyError: diff --git a/src/execnet/_gevent_support.py b/src/execnet/_gevent_support.py new file mode 100644 index 00000000..cd761571 --- /dev/null +++ b/src/execnet/_gevent_support.py @@ -0,0 +1,76 @@ +"""The gevent wait backend (``wait=gevent``). + +Importing this module registers the ``gevent`` wakener factory; the +boundary kit imports it lazily when a spec asks for ``wait=gevent``. + +A blocking wait then parks only the calling greenlet: the carrier's +event lives in the waiting greenlet's hub, and the loop side's +``notify()`` crosses threads through a ``loop.async_`` watcher -- the +one libev/libuv primitive gevent documents as safe to use from other +threads. +""" + +from __future__ import annotations + +import threading +from typing import Any + +import gevent.event +from gevent.hub import get_hub + +from ._boundary import register_wakener + + +class GeventWakener: + """Wakener parking greenlets instead of OS threads. + + ``notify()`` is thread-safe and non-blocking (called from the trio + host thread). The hub-bound pieces (event + async watcher) are + created lazily in the first waiter's hub, so construction is safe + from any thread -- including the host loop, which creates channels + for inbound ids. Waiters must share one hub (one gevent thread per + carrier), which is the ordinary gevent setup. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._notified = False + self._event: gevent.event.Event | None = None + self._watcher: Any = None + + def notify(self) -> None: + with self._lock: + self._notified = True + watcher = self._watcher + if watcher is not None: + watcher.send() + + def _ensure(self) -> gevent.event.Event: + """Create the hub-bound event/watcher in the waiter's context.""" + event = self._event + if event is None: + event = gevent.event.Event() + watcher = get_hub().loop.async_() + watcher.start(event.set) + with self._lock: + self._event = event + self._watcher = watcher + return event + + def wait(self, timeout: float | None = None) -> bool: + event = self._ensure() + with self._lock: + # a notify may have fired before the watcher existed + if self._notified and not event.is_set(): + event.set() + return event.wait(timeout) + + def clear(self) -> None: + with self._lock: + self._notified = False + event = self._event + if event is not None: + event.clear() + + +register_wakener("gevent", GeventWakener) diff --git a/src/execnet/aio.py b/src/execnet/aio.py index e3004a28..bbfe06e3 100644 --- a/src/execnet/aio.py +++ b/src/execnet/aio.py @@ -275,6 +275,7 @@ async def __aenter__(self) -> Self: async def start_group() -> _HostedGroup: # runs on the host loop group = _HostedGroup(self._termination_timeout) + assert host._nursery is not None started: _HostedGroup = await host._nursery.start(group.run) return started diff --git a/testing/test_gevent.py b/testing/test_gevent.py new file mode 100644 index 00000000..e95edfc7 --- /dev/null +++ b/testing/test_gevent.py @@ -0,0 +1,86 @@ +"""The gevent wait backend (wait=gevent): greenlet-parking blocking waits. + +Opt-in: requires the ``gevent`` dependency group (``uv sync --group +gevent``); skipped when gevent is not installed. No monkey-patching is +needed -- the wakener parks the waiting greenlet while the trio host +thread keeps running the protocol. +""" + +from __future__ import annotations + +import threading + +import pytest + +gevent = pytest.importorskip("gevent") + +import execnet # noqa: E402 +from execnet._boundary import Flag # noqa: E402 +from execnet._boundary import Mailbox # noqa: E402 +from execnet._boundary import make_wakener # noqa: E402 + +TESTTIMEOUT = 10.0 + + +@pytest.fixture +def gevent_gw(): + group = execnet.Group() + try: + yield group.makegateway("popen//wait=gevent") + finally: + group.terminate(timeout=5.0) + + +class TestGeventWakener: + def test_mailbox_wakes_greenlet_from_foreign_thread(self) -> None: + box: Mailbox[str] = Mailbox(make_wakener("gevent")) + threading.Timer(0.05, box.put, args=["item"]).start() + result = gevent.spawn(box.get, 5.0) + assert result.get(timeout=TESTTIMEOUT) == "item" + + def test_notify_before_first_wait_is_not_lost(self) -> None: + flag = Flag(make_wakener("gevent")) + flag.set() + assert flag.wait(timeout=1.0) + + def test_wait_parks_greenlet_not_hub(self) -> None: + box: Mailbox[str] = Mailbox(make_wakener("gevent")) + progressed: list[int] = [] + + def other() -> None: + for i in range(5): + progressed.append(i) + gevent.sleep(0.01) + box.put("done") + + waiter = gevent.spawn(box.get, 5.0) + gevent.spawn(other) + # if get() blocked the hub, other() could never run and put() + assert waiter.get(timeout=TESTTIMEOUT) == "done" + assert progressed == [0, 1, 2, 3, 4] + + +class TestGeventGateway: + def test_receive_parks_greenlet_not_hub(self, gevent_gw: execnet.Gateway) -> None: + channel = gevent_gw.remote_exec("channel.send(channel.receive())") + progressed: list[int] = [] + + def other() -> None: + for i in range(5): + progressed.append(i) + gevent.sleep(0.01) + # sending from a greenlet blocks-until-written on the + # gevent wakener, parking only this greenlet + channel.send("hello") + + waiter = gevent.spawn(channel.receive, TESTTIMEOUT) + gevent.spawn(other) + # if receive() blocked the hub, other() could never send and + # the remote echo could never arrive -> this would hang + assert waiter.get(timeout=TESTTIMEOUT) == "hello" + assert progressed == [0, 1, 2, 3, 4] + + def test_waitclose_and_endmarker(self, gevent_gw: execnet.Gateway) -> None: + channel = gevent_gw.remote_exec("channel.send(1)") + assert gevent.spawn(channel.receive, TESTTIMEOUT).get(timeout=TESTTIMEOUT) == 1 + gevent.spawn(channel.waitclose, TESTTIMEOUT).get(timeout=TESTTIMEOUT) diff --git a/uv.lock b/uv.lock index 15185f2a..b885a4b7 100644 --- a/uv.lock +++ b/uv.lock @@ -372,7 +372,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -401,6 +401,9 @@ testing = [ dev = [ { name = "pytest-xdist" }, ] +gevent = [ + { name = "gevent" }, +] testing = [ { name = "execnet", extra = ["testing"] }, ] @@ -420,6 +423,7 @@ provides-extras = ["testing"] [package.metadata.requires-dev] dev = [{ name = "pytest-xdist", specifier = ">=3.8.0" }] +gevent = [{ name = "gevent", specifier = ">=26.7.0" }] testing = [{ name = "execnet", extras = ["testing"] }] [[package]] @@ -431,6 +435,151 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/06/79/b4c714bef36bc4ec2beeae1e0c124f0223888cd8c6feb1cdc56038116920/filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3", size = 97732, upload-time = "2026-07-21T13:17:41.55Z" }, ] +[[package]] +name = "gevent" +version = "26.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation == 'CPython' and sys_platform == 'win32'" }, + { name = "greenlet", marker = "platform_python_implementation == 'CPython'" }, + { name = "zope-event" }, + { name = "zope-interface" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/5c/92002455a57cb3634383e2b822e3bccf409f43cde34528e46428971475cf/gevent-26.7.0.tar.gz", hash = "sha256:5b333a556e38a302b1b8c80525bef16d437e16f1e7767947789406841856a102", size = 6729213, upload-time = "2026-07-22T20:16:04.713Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/60/878d0cdef05d952ac7f17ffe385143fb0f3720afce0f6ff5ddbf7aac0342/gevent-26.7.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:80e98fc808bd9cc5c911d78a443d214bf0c8f96c9fdd296893df7e40364d5f37", size = 2198781, upload-time = "2026-07-22T16:48:28.588Z" }, + { url = "https://files.pythonhosted.org/packages/69/79/6ce781b60049060e9d89b3d0fe60940353adeb39856aaad5ee925fd127e9/gevent-26.7.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bf4b946b47cc6fdbdf9221f891db9a44df92166435c027760ee7dbdfb4039adc", size = 2229318, upload-time = "2026-07-22T17:02:11.713Z" }, + { url = "https://files.pythonhosted.org/packages/6d/38/48898f35c2092d699b755b01918551358db72b554c63f553c8f027d2bf31/gevent-26.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:55ce0b7f87f9befcc788d77eb039b1de89a35f37afc31942e12c7ae090a563b8", size = 1700480, upload-time = "2026-07-22T16:26:16.794Z" }, + { url = "https://files.pythonhosted.org/packages/93/51/53370896942523c333699394ccad379d186648dbfb913f42ce094ddfb4b3/gevent-26.7.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:7f7143823ef99bc657534a2b6e8cbadedc910750cc0b4f4b4438a58d9fe43ab2", size = 1783048, upload-time = "2026-07-22T18:11:25.996Z" }, + { url = "https://files.pythonhosted.org/packages/ba/56/5a2cb36d75d3b626d6ffa116673b34442bf5afd6f7ab4b98e512c1b008d6/gevent-26.7.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:ca4019899830471910129968251c795c8aee59e225fd16326ae01c1f93f3cfa6", size = 1880257, upload-time = "2026-07-22T18:10:40.919Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ed/ee7eb2f03a38a4f33b0f327bab16d3158b3a794588a84dba739cbf4a3e68/gevent-26.7.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:e4042da317a96d12110831cc404855f0c501a5a5aa476a7a18c3b480a5a59233", size = 1819378, upload-time = "2026-07-22T18:29:06.444Z" }, + { url = "https://files.pythonhosted.org/packages/5e/aa/e2a202c03cff4f49bba54cc321c3afc29eee9d6fc452f4aa94d2f00e7d3d/gevent-26.7.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:5c97ca98e1aae427a267eae0fbfe8d0884327e6b1cd51fc2ef6642b8b0b82701", size = 2136837, upload-time = "2026-07-22T16:48:29.939Z" }, + { url = "https://files.pythonhosted.org/packages/ac/64/4892fbca47aa4e06b86aef96d6146bd61175b23b7db431ec7937d73b2f72/gevent-26.7.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5d5d1864bc3db92d1f82d1790395eda99f98b47fd9f7ec02c4e182d7828a8251", size = 1794058, upload-time = "2026-07-22T18:07:10.644Z" }, + { url = "https://files.pythonhosted.org/packages/21/23/90bb7d0c6f59d2973bb8f4bd3164be00e466823dea01568e0cb36f2afb77/gevent-26.7.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:15fd2d88ed5370f8084079758758df91f26d2f68575e1ee76fce604ddba83e5e", size = 2159797, upload-time = "2026-07-22T17:02:13.229Z" }, + { url = "https://files.pythonhosted.org/packages/a3/67/4d1e315ee3052530fa8537e0cdf1f9c3a6606740f7372f420c7d12d5cf82/gevent-26.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:514bda3fff741d7e5ab108ee1d31550a7f4b2fd3dc6e3b6f38dfb8685efdafaa", size = 1682414, upload-time = "2026-07-22T16:26:22.18Z" }, + { url = "https://files.pythonhosted.org/packages/02/b1/d1b1de89677ee39e641ad8501ed72c2b99f1ddba1c14c462675f40b37a66/gevent-26.7.0-cp311-cp311-win_arm64.whl", hash = "sha256:0f26f9a8c32ac0a73f6084c59b63deeacb350e7f1fee5301d95c5e0683a390d4", size = 1562794, upload-time = "2026-07-22T16:27:53.205Z" }, + { url = "https://files.pythonhosted.org/packages/2b/66/104590ad3a9e671b3ef77ad19c1cc50e7f1c8c220b27ddfaf34e5f88bd9c/gevent-26.7.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:92f256285fb43a57f152bd2e51a59cde1cd0b20869ae1e6da583b6beab88ed8a", size = 2953977, upload-time = "2026-07-22T16:23:41.654Z" }, + { url = "https://files.pythonhosted.org/packages/64/07/350d87161378633714184828bdc57c66f9b525eca5249ad1294dc7f8cf58/gevent-26.7.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:0e4fea187c5df7168b9538b4f543fcb0fcbaeb93be3d6cd499c324652c740704", size = 1800960, upload-time = "2026-07-22T18:11:27.242Z" }, + { url = "https://files.pythonhosted.org/packages/c4/6c/ddfe298c2ecb1cfc72a03dd69ba751759f7e7835d3135f171b284eb09ed1/gevent-26.7.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:ce732fe08d0ea65de07eff6e46bade8ac6a6fdb65cc748c713f3d31ae122529e", size = 1900387, upload-time = "2026-07-22T18:10:42.973Z" }, + { url = "https://files.pythonhosted.org/packages/f9/89/2647bbbf1da35a1c271a54452e76065308cbaf684ee32a79f989ca4265f6/gevent-26.7.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:c25b3522072137aecf3389031039230190038f888e257f490b3897d0e0620f74", size = 1848046, upload-time = "2026-07-22T18:29:08.074Z" }, + { url = "https://files.pythonhosted.org/packages/31/52/4f9b4c536b5a0424e328d5a5466640d185020f1f0b08b2de0ca939cc0a0b/gevent-26.7.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:eaaa75c9014df3f8c310c64f53f1152af8c6be32e82734396bed91e1d0e6f35c", size = 2132200, upload-time = "2026-07-22T16:48:31.203Z" }, + { url = "https://files.pythonhosted.org/packages/42/88/1daffa63b257c381df68018e4d3da72b429955cf95a171a46d3220422c8d/gevent-26.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f1956032a9926ac9b4152b2a50bc5a2cc020722ec16928ccaf32e227ee0aae47", size = 1814237, upload-time = "2026-07-22T18:07:12.438Z" }, + { url = "https://files.pythonhosted.org/packages/73/4c/b996cdddca78eac435195cd97dff590c65a9e0052640bcb6f8b6f1e30b1d/gevent-26.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d989a1ad6cc54f5c69bb7304360f98b4fda80da2b773f1047db9fba61ae7379a", size = 2157938, upload-time = "2026-07-22T17:02:14.887Z" }, + { url = "https://files.pythonhosted.org/packages/08/fd/44419d7559a95e238ee45d29df30bad78a91d343cb27b13a6af552b907bb/gevent-26.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:e0c9ce2d80fc0f8894d748a1045ff26ad188e294bad656b29839271800827c85", size = 1685319, upload-time = "2026-07-22T16:26:13.954Z" }, + { url = "https://files.pythonhosted.org/packages/6f/7a/7df1762ccd9a40ce1ca626f50cb7a75758f2c930aabcc73c91cf00cb3448/gevent-26.7.0-cp312-cp312-win_arm64.whl", hash = "sha256:959effe0c56cdee0bf761e5c4e78ab62880be147a2f2aa31112ca2f7e5754e53", size = 1558945, upload-time = "2026-07-22T16:26:56.737Z" }, + { url = "https://files.pythonhosted.org/packages/75/63/0fcfbe3f5696e56424f331ec41e0e447cea79c384848d6019e3f7f340f4b/gevent-26.7.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:b1b89eb5566f75aa8b2bbdb0308e1ac8d9113ca7cff85b45366aea9faad639a1", size = 2976844, upload-time = "2026-07-22T16:24:39.275Z" }, + { url = "https://files.pythonhosted.org/packages/3c/6c/ea2d0afbe760c18df5bd1631dbe5a73d840d9b141cb71e6810157c2ae28a/gevent-26.7.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:449857ce058183442e2d71d83ff0c587a3ddff631e93c6d19a6dffb4814eccad", size = 1802332, upload-time = "2026-07-22T18:11:29.155Z" }, + { url = "https://files.pythonhosted.org/packages/59/90/36f2258f1bfe8601224f6159066103b832c31e1451ce8dc2cd2408b6ecf4/gevent-26.7.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:8260a3f38b05fcf3c283417b18617562dbec74f5784f748e4ba3866789d7f3a4", size = 1901253, upload-time = "2026-07-22T18:10:44.402Z" }, + { url = "https://files.pythonhosted.org/packages/dd/38/86dd67e5c2dfab016a9c935b4338d6cf8f9bfa72dc5a0f3fb879d993127a/gevent-26.7.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:30894398d06747b433c8923a6a77ede61259ce6822a99f6c6e7fa0216ccb73c3", size = 1850489, upload-time = "2026-07-22T18:29:09.694Z" }, + { url = "https://files.pythonhosted.org/packages/f1/33/f5651942a5967483298b6ce6f45572d33120dd0fd01c8991d3fca5b1e8ee/gevent-26.7.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:0b753522498118c9489753de7c612d4baed0edf384d9df2bf9492233ba1c20ff", size = 2129813, upload-time = "2026-07-22T16:48:32.551Z" }, + { url = "https://files.pythonhosted.org/packages/c0/09/abe8217a8fcd3f0e94c9eec024a5499c0f267cb269ccea6c0e2e812319b9/gevent-26.7.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:055a643026dc28daff2be228555a2097937448cc9b58307edebcf81b9d78ff4b", size = 1815121, upload-time = "2026-07-22T18:07:13.988Z" }, + { url = "https://files.pythonhosted.org/packages/40/d6/dbae1cd2d27b62664cefa086035530eb21203d45b12f466272441c048c9c/gevent-26.7.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4e1dc6a2712de67fd210e1f1a408601f6908b042f6420e188106f2f37f94ec71", size = 2155913, upload-time = "2026-07-22T17:02:16.35Z" }, + { url = "https://files.pythonhosted.org/packages/fc/42/90b662f4eb27d7727d4619d5c6be872117f1a9f187b243ec7f6fef988ce7/gevent-26.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:44e5280296129c0915addaefdb37d6e9bc124a77a433b1b1c8ddf1853c53f4e7", size = 1682483, upload-time = "2026-07-22T16:26:30.492Z" }, + { url = "https://files.pythonhosted.org/packages/e1/84/7297c56b9fff463c4ba2f685dbb913a855df903046dc68d14e8655a29ffe/gevent-26.7.0-cp313-cp313-win_arm64.whl", hash = "sha256:9f08b1aa6729f794409ca137e25f671e0d9bbda4451200c5e28a769375365388", size = 1556053, upload-time = "2026-07-22T16:26:14.365Z" }, + { url = "https://files.pythonhosted.org/packages/c1/bb/ab60d496cbdc0293ebbd6c2070b34da0632bd7a2ca20163c17e18d2d2dc9/gevent-26.7.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:0e0e3bf7ae0f82dbc5c6be26b4781e86c97f1e28d516b7a9746ac8b04bcc6948", size = 2992503, upload-time = "2026-07-22T16:24:36.503Z" }, + { url = "https://files.pythonhosted.org/packages/5c/35/75f27c06a82a5b22600aaccbd9567d89bb4091be43e96c02981f10aff23d/gevent-26.7.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:740050b53048207b080a1e183a377c47809ad0b7b7b0cd7eab0dea1045f7e480", size = 1809173, upload-time = "2026-07-22T18:11:30.724Z" }, + { url = "https://files.pythonhosted.org/packages/5b/5f/a6b32b4db3fa76bd8a070f0f46f5306123bf6336e7a0ca0cd2f9b99473df/gevent-26.7.0-cp314-cp314-manylinux_2_28_ppc64le.whl", hash = "sha256:67983607eb6c7bafa362c5c43b69a27145b936c34a3d6441ed42413d62fae0a6", size = 1906630, upload-time = "2026-07-22T18:10:45.836Z" }, + { url = "https://files.pythonhosted.org/packages/e1/87/832495d8fcc05ff7432f038b7c4decbd5632425a2cd5da2ce73cb2d800c4/gevent-26.7.0-cp314-cp314-manylinux_2_28_s390x.whl", hash = "sha256:475848518d708e07d1987c3d94cb8ff53e2b3a69df32e39feda2779cafe400b0", size = 1855278, upload-time = "2026-07-22T18:29:11.517Z" }, + { url = "https://files.pythonhosted.org/packages/72/8b/2f36c0fa389fa2b7ceb5a8972b0e7da7bc770f9135315cf4246c607ca5fc/gevent-26.7.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:0f8ed457dd616bfe6682569f92730f9ab45aafb1aeca5e80eb2f6b9a2ce26d11", size = 2136155, upload-time = "2026-07-22T16:48:33.865Z" }, + { url = "https://files.pythonhosted.org/packages/b5/98/09f2cfaa23dbce48e3271e95b0d003f93acece6b5cfd40f4cebe3850d79b/gevent-26.7.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:15373c68cf1fa14114bec2f09b16e2c65374bd5309e897e0a28740b09ce329e0", size = 1822108, upload-time = "2026-07-22T18:07:15.397Z" }, + { url = "https://files.pythonhosted.org/packages/b0/d8/05a294165c17569f04284ad3c889684c8780544885b4cdf77b1432947d0c/gevent-26.7.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:73f3d53f2f390369e290c933b75bd87f1f2261f2f2f2175aa667c43ee3049bad", size = 2162814, upload-time = "2026-07-22T17:02:18.066Z" }, + { url = "https://files.pythonhosted.org/packages/59/89/58a545c4eda33e106d6887a0387adc2249abc14c779e3eb88bbfdf3768d6/gevent-26.7.0-cp314-cp314-win_amd64.whl", hash = "sha256:f11b558d544ad2249029ba023cd6519ec3a0eee54a3d027e6515c1eaa322422a", size = 1706971, upload-time = "2026-07-22T16:27:02.263Z" }, + { url = "https://files.pythonhosted.org/packages/bb/cd/413f293e54961e5c89c54235370e3603ec0f561e7ace8357980410efbf78/gevent-26.7.0-cp314-cp314-win_arm64.whl", hash = "sha256:3871f4ca59ec2328c3ef638a0fe01a28a825443a133368dc78eb5ceadcad7609", size = 1585078, upload-time = "2026-07-22T16:30:48.145Z" }, + { url = "https://files.pythonhosted.org/packages/21/3a/47f29f632aaa38aa12410f57f1732fc50bfd4d4006d2e7e022ce731cabc9/gevent-26.7.0-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:3e3d6e20a94239ad353b776e72b8ce18c35dbe4e98c279aef3932651553d8404", size = 2996208, upload-time = "2026-07-22T16:23:23.859Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b3/4620f1ce81ecec9890229806c73f07dd022e40f76a552bd430391e7316c4/gevent-26.7.0-cp315-cp315-manylinux_2_28_aarch64.whl", hash = "sha256:ddbd3cc76b9bc69df651a216c2a62fc6415ad463b3ac9c6cbbbb8b7b8224af17", size = 1811545, upload-time = "2026-07-22T18:11:32.224Z" }, + { url = "https://files.pythonhosted.org/packages/97/fd/d285212ffd5585d511299e13e61d76262def8e826e9f20c92cb85df406f2/gevent-26.7.0-cp315-cp315-manylinux_2_28_ppc64le.whl", hash = "sha256:01ceab7e608dc1b9859d9511a0a29d7ce2e7d909ab19fddc860e70a2ed5b10ce", size = 1910418, upload-time = "2026-07-22T18:10:47.709Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e0/c5d666e6065652918cfb6e6a3cf8f721d0e152c57cec217ad81792a1323b/gevent-26.7.0-cp315-cp315-manylinux_2_28_s390x.whl", hash = "sha256:2e6c917b2b8baeb6080797a6b25e35e1fd784319a05bb92b87c53546e5578eb2", size = 1857891, upload-time = "2026-07-22T18:29:13.13Z" }, + { url = "https://files.pythonhosted.org/packages/a1/67/e945ed458fa98b34572876bfd0d35fe4fa3f1159f43660b71d982b7cb63e/gevent-26.7.0-cp315-cp315-manylinux_2_28_x86_64.whl", hash = "sha256:df75a1748b26030f2f7f10042cc45640b22954d9d0dc6b4b6f0dbe0b6751a2d4", size = 2138121, upload-time = "2026-07-22T16:48:35.358Z" }, + { url = "https://files.pythonhosted.org/packages/99/55/622468fa1a3c4cf51f20e14813f5cc1592fe6e44a42ccca9195a0b18c769/gevent-26.7.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:ee1b389587e5d5c1eb19d0455b5b4d7a0fb5c5287af4e226ec66d9dfd2548107", size = 1825114, upload-time = "2026-07-22T18:07:16.667Z" }, + { url = "https://files.pythonhosted.org/packages/0f/7a/151a2afcacf487ca25faf8b1bdd6c5b4ace2f7c1e6b4eaffe0a5e6e1df61/gevent-26.7.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:c2918641ba756f46aa01ab9dd82d6dfceec403c77c2787298746b411dcf0288e", size = 2165990, upload-time = "2026-07-22T17:02:19.817Z" }, +] + +[[package]] +name = "greenlet" +version = "3.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/74/b13368064b09053253555d3f2839cc2684d22d5aed0d2ccffbf7a6736558/greenlet-3.5.4.tar.gz", hash = "sha256:0232ae1de90a8e07867bb127d7a6ba2301e859145489f25cda8a6096dabe1d20", size = 206538, upload-time = "2026-07-22T12:47:14.468Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/9d/58f80897f4121f5c218bb931cf6d3b6514873f02ad0b729f744352926b9f/greenlet-3.5.4-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:ac5bf81d79d2c8eeb2ef6359b2e1687a1e9ebf46c2b1f970da9a9255df51d190", size = 293072, upload-time = "2026-07-22T11:38:14.299Z" }, + { url = "https://files.pythonhosted.org/packages/dd/9f/b4bc9bbd6a7855cbd8ad8a83c874eeeca56c24de9132b3323f81c03a30ba/greenlet-3.5.4-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89f3738167bab8c1084b94e23023d41d247117ac149fa0fbcb5bd4cf6262b353", size = 609393, upload-time = "2026-07-22T12:26:37.964Z" }, + { url = "https://files.pythonhosted.org/packages/05/0e/744b5e063af127d2e3c74fe0f1aef15573064c83b6066883524f5b258b17/greenlet-3.5.4-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e9a5e3406e3ed8125ae1a3b37c12f3434e2b1f0fa053197c5557895b4fb09606", size = 622750, upload-time = "2026-07-22T12:28:59.546Z" }, + { url = "https://files.pythonhosted.org/packages/5d/5c/53d6b94742a6f1ee1877c7ff76262c909e137f3f7383ce96a8ab78e1ae31/greenlet-3.5.4-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a2d614cb2372c7101a12ea8b96dd56f81c986d247c5a73db67063f3ed1ca4a52", size = 629659, upload-time = "2026-07-22T12:43:40.451Z" }, + { url = "https://files.pythonhosted.org/packages/d6/6b/d78ea2908e8e08985348f28ac396c2950be7ab66321dfe0054c73bd1f456/greenlet-3.5.4-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4ab9f0704bccf6d3b38e0d2130b7b33271cff11453690da074fa280c3aa8e8e7", size = 622920, upload-time = "2026-07-22T11:51:06.83Z" }, + { url = "https://files.pythonhosted.org/packages/f2/34/957fc5577180ef2f57be82580ee1f59fdefad4f6c623c7d5e1b6980a76fb/greenlet-3.5.4-cp310-cp310-manylinux_2_39_riscv64.whl", hash = "sha256:188e4d142f243051d92a1f5c244a741da02dddc070a0620c842804d7b56d008c", size = 425580, upload-time = "2026-07-22T12:39:48.326Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e4/3ce7009c948920b01527f8d9da29f501a31ac3d98318829e981fd879b850/greenlet-3.5.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2cdaadc3d31445a8f782bde3cd37e49a2c2a9c6da6daf76a3e34c683b271a3c7", size = 1582262, upload-time = "2026-07-22T12:25:01.131Z" }, + { url = "https://files.pythonhosted.org/packages/f2/4c/0408366102a33829f7bdd6a992dad75abbf75e86cc1e76caf19e57311d29/greenlet-3.5.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:70bdfacdc183dac838b2a0aaff2dd6134a457c52fe68a9c6bbab435483d2b9df", size = 1648906, upload-time = "2026-07-22T11:51:08.627Z" }, + { url = "https://files.pythonhosted.org/packages/13/52/ebfe8f6a1aeb8e430540b406c844ecc4e3367072b0192f69dcb85eeeec2b/greenlet-3.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:69173331fbc5d64bfac0065d7e22c39cfcd089e9b18d125bdcd5079363b09616", size = 246036, upload-time = "2026-07-22T11:38:30.073Z" }, + { url = "https://files.pythonhosted.org/packages/61/16/71eefcf68267bbf06a9b6bff57d0b222e49432326e85d74348b67694b8d4/greenlet-3.5.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e883de250e299654b1f1680f72a1a9f9ba62c9bd1bce84099c90657349a8dfbb", size = 294266, upload-time = "2026-07-22T11:37:56.142Z" }, + { url = "https://files.pythonhosted.org/packages/36/ea/a0b19adfc35d07e10acb626e9d22a3893b95f1309c42c4a20161dec16800/greenlet-3.5.4-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32802705c2c1ff25e8237b3bdacf2594fa02be80af8a66703eb7853ea7e68686", size = 613712, upload-time = "2026-07-22T12:26:39.375Z" }, + { url = "https://files.pythonhosted.org/packages/54/76/a121978b3337407d05a1ce5f79b4aa5998a43a9d8422f9726029b90b4471/greenlet-3.5.4-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:57aa201b351f7c7c75627c60d29e4d5b97a07d37efeb62b903466fca42c097d7", size = 625582, upload-time = "2026-07-22T12:29:00.814Z" }, + { url = "https://files.pythonhosted.org/packages/d1/4a/f301f1d85c69a86b90b5d581a73e8927bba4e79450037e6e2cbca05eb4fd/greenlet-3.5.4-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9667862a2e38ad379f11b845daeda22c8989186def44f06962c9c4c05e556da7", size = 633429, upload-time = "2026-07-22T12:43:42.073Z" }, + { url = "https://files.pythonhosted.org/packages/34/c2/080f16cf870e929e592f55767f01d6c98d2ee83bfdc36c3b892f2d0459ab/greenlet-3.5.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c3fe76c2cac86b4f7a1e92865ac0a54384deb05c92986287c1a7110d9bd53071", size = 624663, upload-time = "2026-07-22T11:51:08.016Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2e/26884072b0eb343a4d5fee903341bfe5171b32b7f14553886e2b6349135a/greenlet-3.5.4-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:ae53534b5dec0f4c2ec26f898f538dc8ea1ca3ef2927d597a9439e40a09da937", size = 428238, upload-time = "2026-07-22T12:39:49.973Z" }, + { url = "https://files.pythonhosted.org/packages/9e/bb/8f3ca88370b817369008faeceeee85970adc16c92a70a3e5fe5fea495a57/greenlet-3.5.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1e1a4a684b16c45ba324e60b32a4386a87722bcb815d2a149d2182f9b401ca72", size = 1585010, upload-time = "2026-07-22T12:25:02.539Z" }, + { url = "https://files.pythonhosted.org/packages/51/c2/45877154689709ebce9a0b83c2235e6ca0f31577889b02af308c8cc5f8fb/greenlet-3.5.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e849e6e139b9671adeac505f72fc05f4af7fd1921faef40295e214fc3b361b59", size = 1651283, upload-time = "2026-07-22T11:51:10.408Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7d/8711a75cb61d85246277c07ff6e1a6504621ba473d808c11ad225ffca43f/greenlet-3.5.4-cp311-cp311-win_amd64.whl", hash = "sha256:dc418cf4c873357964d6624445ed09472e50def990c65dd4e76fc3ba8cd9cef6", size = 246434, upload-time = "2026-07-22T11:43:15.557Z" }, + { url = "https://files.pythonhosted.org/packages/00/62/e290b3bce433da8f0324ac02da0b128d683482229f1a8b789fa47818a4cd/greenlet-3.5.4-cp311-cp311-win_arm64.whl", hash = "sha256:c38c902a0986eba1f6e7ba1ab39ad5195926abde90f3fe080e08212db62176da", size = 244990, upload-time = "2026-07-22T11:39:22.626Z" }, + { url = "https://files.pythonhosted.org/packages/f3/04/81bd731d6d1e3a469d9a4c36f5eb069bcf0cbb2d5d342c9fec22245b91fc/greenlet-3.5.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3d66250e8b09f182ede05490998c818b5961f7a3640332d44c4927caec7bbfe4", size = 295909, upload-time = "2026-07-22T11:38:09.261Z" }, + { url = "https://files.pythonhosted.org/packages/cc/dd/f5f22903a6ae70f5ea328ed0beaec92ad903f0e3b7d2845133b354abc4b8/greenlet-3.5.4-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c90e930c9c192e5b3ee9fb8bcd920ea3926155e2e3ded39fc697323addecee17", size = 612011, upload-time = "2026-07-22T12:26:40.69Z" }, + { url = "https://files.pythonhosted.org/packages/8e/10/92a4a88d12b915d74ea5b6d288e4afefda4771647caa34442c156f7a454f/greenlet-3.5.4-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:791fdfeeb9c6e0c7b10fa151bf110d2a6974866f13dcb5b1c7efae698245893a", size = 624299, upload-time = "2026-07-22T12:29:02.089Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f9/03e26be3487c5238e81f2b84714959a86ea8515a869828cf41f4fc54b34e/greenlet-3.5.4-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b7c895310363f310361e0fe2072af85269d2a2a285cd04c0c59e79a5e3670dcf", size = 629603, upload-time = "2026-07-22T12:43:43.456Z" }, + { url = "https://files.pythonhosted.org/packages/50/6d/0b14bb9db2989f32cd9fe7f76afedea01ee8bee3f87c07e69f24adfe7e63/greenlet-3.5.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f88193799d43dbf8c8a806d6405c9c52fe2af40bf75072a606357b33cc336c7f", size = 621541, upload-time = "2026-07-22T11:51:09.464Z" }, + { url = "https://files.pythonhosted.org/packages/57/6b/7c55ca72ef80d57c16c4a55210f82582622462dc4485799a30f4ec6f3372/greenlet-3.5.4-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:13b980043cb1b3134e81ea469da1250ddcc6bfe6d245bbaa59168d9cdc8f228f", size = 432554, upload-time = "2026-07-22T12:39:51.379Z" }, + { url = "https://files.pythonhosted.org/packages/48/3d/25e9a2d9eb6b2e8b7ca4e80a3a26cb887cce6c8e0a87c921164f11bc5574/greenlet-3.5.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b7a5f095767c4493afcd06067f2bb3b8716e3f3f9e92b99c88e7e99f885b3d4d", size = 1581444, upload-time = "2026-07-22T12:25:03.818Z" }, + { url = "https://files.pythonhosted.org/packages/b9/96/4c9bf2e2c408dcc0556edce69efa9f802e82223573c53240136a086821f1/greenlet-3.5.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:42afdc1ab5f66da8c586c32af9224a74a706b4f0ea0dc3a4188a0860a09c65c9", size = 1645842, upload-time = "2026-07-22T11:51:12.295Z" }, + { url = "https://files.pythonhosted.org/packages/b5/41/303ecb26a3a56122c0f4d4073ee078881847bd6b6f463ae0ec57ec20223b/greenlet-3.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:60149df8f462d1b230038e6590c23c3b4768bb5d6c022b3b6e82532b34b0b8a3", size = 247169, upload-time = "2026-07-22T11:38:19.893Z" }, + { url = "https://files.pythonhosted.org/packages/a4/e3/ef56864b4c35fcb3eb3b41b869f6cc46f4cd3f5e2c68e74acde8ac433951/greenlet-3.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:77d6ce04fed0d9aeed42e0f37923cc43eba9b027bdd9c34546bb4ccd143d0fe0", size = 245565, upload-time = "2026-07-22T11:38:27.061Z" }, + { url = "https://files.pythonhosted.org/packages/c0/9a/e51225dcd58713f16ccbdcc501a8da21098ea14515b7870f1f94459e5ff5/greenlet-3.5.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:24e61b88cb7e1b1d794b32a10cc346ac779681d6d74ff137a3e0a444d2bf1f02", size = 294831, upload-time = "2026-07-22T11:38:53.389Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ea/de50a50fadf979713ab18b46f22ad5ff5f2dcfc637a3ebdecf669801e1a5/greenlet-3.5.4-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:870d730fec833f5a06906a32596cc099b9161594642a92a520b7a88911c95356", size = 614619, upload-time = "2026-07-22T12:26:42.282Z" }, + { url = "https://files.pythonhosted.org/packages/db/c7/2aae27fea41205b8650294c301f042a2a4bb6155eea48c995b890a92f2c1/greenlet-3.5.4-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ec5ff0d1878df6af3bf9b638a5a92a7d5693291de77c91bff10fa48519c604ef", size = 627021, upload-time = "2026-07-22T12:29:03.445Z" }, + { url = "https://files.pythonhosted.org/packages/1b/80/fb4d4788bbc8e54761f1fc88533af9523a6e86299fa113d6e8a8503ed9fc/greenlet-3.5.4-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:07bd44616608d873d06735b63ef1a88191d6ca57c8d291d6559c71bc14c0893c", size = 632845, upload-time = "2026-07-22T12:43:45.19Z" }, + { url = "https://files.pythonhosted.org/packages/eb/56/79fd826f9ccaae0b84e1b4ef68dabba5e105bb044ffcd448a0b782fcba9a/greenlet-3.5.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d84d993f6e575c950d91a23c1345d18fe1a4310d447bf630849d7809196b52f0", size = 624002, upload-time = "2026-07-22T11:51:11.391Z" }, + { url = "https://files.pythonhosted.org/packages/42/e3/6086fa578ebb72772722cdc4bcd628459814b42e0c2db1e3cbd6552b3271/greenlet-3.5.4-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:3529a8a933582ad19e224792cac7372489526576b75b4c124e8e4f29948f4861", size = 435053, upload-time = "2026-07-22T12:39:52.715Z" }, + { url = "https://files.pythonhosted.org/packages/0a/1a/27319f97e731298513dcba1a2e91b63e9d8811d9de22130f960b129b1bf1/greenlet-3.5.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:58023945f421093de5e6fa108c0985a8659d43f49e0216da25099369a121bcbd", size = 1581533, upload-time = "2026-07-22T12:25:05.322Z" }, + { url = "https://files.pythonhosted.org/packages/b1/6d/24240bf562e9786dd2799ee0a4a4dadb4ded22510f41b20245099159ac8c/greenlet-3.5.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bae2728e1897aa8df8cb1af38cd48b3a743aefe29372de7b8b7a9f532501e69f", size = 1645781, upload-time = "2026-07-22T11:51:14.805Z" }, + { url = "https://files.pythonhosted.org/packages/c1/5a/442ab1a9ef7ca6bf7210e5397a95972206a91a31033a03c8900866a10039/greenlet-3.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:ca5726c0b08ca35ae873557266a78b2c3f3b2b7d7401aa5ff886c2045dd0111c", size = 247133, upload-time = "2026-07-22T11:39:20.661Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e6/9160210222386b1a378ff94db846b9508ca24a121cf684991561fdb69280/greenlet-3.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:7c1303791d603080cac6fc3b34df51c3b75b723739c282c8029e48a0d241672f", size = 245500, upload-time = "2026-07-22T11:40:22.185Z" }, + { url = "https://files.pythonhosted.org/packages/a5/a7/6ab1d4f9cd548d15ab90da29947f2076100130bb179b0bde59f795a459e3/greenlet-3.5.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:7e8afa5eac028f8140ceafe5ceec66e6aa127ddcb21452d2a564dcd2900b5f22", size = 295410, upload-time = "2026-07-22T11:40:35.747Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7a/422f63b4715cbc0b24385305407adf38b48f6bb68b3e6b04090e994d0f5a/greenlet-3.5.4-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73b37afe369021423ea53dd3123e04bffa7e93ac64429b9f50835b2e4fcae7cf", size = 661286, upload-time = "2026-07-22T12:26:43.8Z" }, + { url = "https://files.pythonhosted.org/packages/d0/31/5a1cac663bf5582190c5a714ef81364f03cde232227f39748f8ae4c11da5/greenlet-3.5.4-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3ef964f56dfcb6f9bbef2a190d9126795eac408716aeae47b5e7c73c32aafca9", size = 673517, upload-time = "2026-07-22T12:29:04.815Z" }, + { url = "https://files.pythonhosted.org/packages/9c/bf/250c2921c7b585dde12f5239e313ca2dcbc464d161ecca36e4e6ef21762d/greenlet-3.5.4-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cef589bc65fae02d10bca2ac341191c5b33acc2967892ebf4fcbd10eabb7a74c", size = 677968, upload-time = "2026-07-22T12:43:46.788Z" }, + { url = "https://files.pythonhosted.org/packages/15/4a/2a82a1e3f8aaca020853ac8d12211280ca2b231aa08ea39f636f1060c319/greenlet-3.5.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c53ff01a5c53a40f2c16820ebc56d7c61a77f5fbe009dadd96292d5682f80f8", size = 670917, upload-time = "2026-07-22T11:51:13.589Z" }, + { url = "https://files.pythonhosted.org/packages/18/40/10bfcf6513558d82f7b95dd728001c63bd388259fe27d3e30ae01f103430/greenlet-3.5.4-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:dfc41ae893d9ceaf22c824f2153a88b30651b20e8758c2cd9ac143f23640563c", size = 480643, upload-time = "2026-07-22T12:39:54.149Z" }, + { url = "https://files.pythonhosted.org/packages/68/b0/e379a152b17bfdfa95795af4049e37c0fd1b4d81f020d426db104ed07c77/greenlet-3.5.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ecca4d80d55a01ad6b23b33262662956149fbb7b2c6be2910f1705921958cbf3", size = 1628478, upload-time = "2026-07-22T12:25:06.678Z" }, + { url = "https://files.pythonhosted.org/packages/5e/43/bffdfa64f7317f954c5c1230b5dd5922676ce198689a68c1ac1ed4b1b1a5/greenlet-3.5.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ffbc533e0eaf8e80d8471411646ab88fe58f641d508c0b02b24494479f4d9ec", size = 1692021, upload-time = "2026-07-22T11:51:17.008Z" }, + { url = "https://files.pythonhosted.org/packages/d0/11/f799f9637e2c6e9b0b716015e339040598b058cf7654dfc0d67468b177ed/greenlet-3.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:305f69e6c4523d7f6979ed001cff4e5853c063e5da04880296603aa0227e544c", size = 248031, upload-time = "2026-07-22T11:40:11.007Z" }, + { url = "https://files.pythonhosted.org/packages/05/75/625bcdd74d5e6b2dca1ecba3c3ac77bcf8a026c21a649a46cef23e421f97/greenlet-3.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:f260930bbbbcf9caee661211235a5111c86dfe5832fdf6ae4570da1e0995320f", size = 246892, upload-time = "2026-07-22T11:40:27.357Z" }, + { url = "https://files.pythonhosted.org/packages/ec/69/35c62ed49c320cb4d98e14698ccca5467d3bfe683984172be9cb564d9ce3/greenlet-3.5.4-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:41ddab54e4b238f4a6c323f39b4e59e176affd5a94d461a9fb7583dac74240a3", size = 305571, upload-time = "2026-07-22T11:40:31.659Z" }, + { url = "https://files.pythonhosted.org/packages/5b/6c/64d60216b3640dcb0b62d913dd9e0d80030c09115bb2e4ba70c95d10ca45/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b3dabe3e2809013052c68bdf0b7fa5f5f2859c43a80803131ad61af9cabd7867", size = 672568, upload-time = "2026-07-22T12:26:45.298Z" }, + { url = "https://files.pythonhosted.org/packages/5c/de/ba3ab0a96292e53039530333b0d2ae18d9e508f3a325cd7bf15f8172944c/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:27d3f00718634d4520a3a150154ac5da36f257869d41321953375b90bfbbc72c", size = 680076, upload-time = "2026-07-22T12:29:06.125Z" }, + { url = "https://files.pythonhosted.org/packages/ae/db/24a10af12bf8e639cec46c38b9ce1a282543ba42ff4fb0b31a970f1ab603/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:39169a11d87a6a263afda3e9a27d1df16d0f919d40a4837cc73986c9884c0dd8", size = 681690, upload-time = "2026-07-22T12:43:48.109Z" }, + { url = "https://files.pythonhosted.org/packages/a5/be/aeada79083c6f1c15f45d77a332f9c441af263ee298e3eb17522cd337d22/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbd60b5763c6543c1827e48faaf14ea9bfbad245f52b1a4d76a2a2d8884c6c66", size = 676733, upload-time = "2026-07-22T11:51:16.027Z" }, + { url = "https://files.pythonhosted.org/packages/f4/60/44a2eca7b9fd71ae0fae7ff184da1cd3169d176652b97aa1cffcbb0ef961/greenlet-3.5.4-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:bd3d1145f603b2db19feb9078c2e6855eb7c67e15580c010ed815cee519b86fd", size = 510263, upload-time = "2026-07-22T12:39:55.678Z" }, + { url = "https://files.pythonhosted.org/packages/e9/10/2392fc3a98948652ef5fd1e7275c04f861dd13f74b78a2b4309f4ee4d090/greenlet-3.5.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f00f910f0e7b35416c63b23ad78b769aeccfc1775f712b43c4ee525624a2eef7", size = 1637327, upload-time = "2026-07-22T12:25:07.879Z" }, + { url = "https://files.pythonhosted.org/packages/55/c6/e7237a3dfa1f205ed0d9ea1e46d70bd2811b32d516266399fb59d28ab90a/greenlet-3.5.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:91c26423753b92caf41ab3f98fd547d7374d4d9fc2d85be041886c1579d9255e", size = 1697493, upload-time = "2026-07-22T11:51:19.214Z" }, + { url = "https://files.pythonhosted.org/packages/55/e3/4ba8154ba2a3d43729e499f72471b4b5c993f3826d3e24da81d5f06d6572/greenlet-3.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:ee032b91fd8ec29ec6c4cea2b8c561b178435134bd0752c7334b94e9c736c132", size = 251637, upload-time = "2026-07-22T11:40:37.44Z" }, + { url = "https://files.pythonhosted.org/packages/90/03/e3f96dfc100261a29545ddc8270cafe58f9195b6651466910e820910de77/greenlet-3.5.4-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:178111881dd7a6c946471fda85485ec796e1043c2b939f694b096e2ecf986809", size = 296076, upload-time = "2026-07-22T11:39:38.364Z" }, + { url = "https://files.pythonhosted.org/packages/a4/3d/da52d208e5c977bce8667e784729e584e38b5785f4c1ec0f4c836e9a1c42/greenlet-3.5.4-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d92df08dd65fede97fc37aad36c2e9dcda3b31c467f8e0c2c096456cb818e927", size = 666870, upload-time = "2026-07-22T12:26:46.691Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8a/7e6dee25cb8a8cf9b362c8e597cc269593378bd916f16c736c059a52e85a/greenlet-3.5.4-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:99e8f8c4ebc4fd80aa26c1280ae9ad43a0976e786349703a181cf0bae60413e5", size = 677678, upload-time = "2026-07-22T12:29:07.508Z" }, + { url = "https://files.pythonhosted.org/packages/51/a7/dafc7415d430b0a43a16396eb49ecb3b62fd720877fb259cc4dcfaf5f31e/greenlet-3.5.4-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1f17e362d78e37559e0506c5a7d066bdd45073c36a0127a543e8a0df27242ff3", size = 681428, upload-time = "2026-07-22T12:43:49.623Z" }, + { url = "https://files.pythonhosted.org/packages/6c/21/5a38699fa45de749e3857d93b8f07e4c20489e77c2d35d915a2e1c456606/greenlet-3.5.4-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:394de08dad5ffcb1f50c2159d93e398d9d2da3ed437645eaa54771fa720db9f0", size = 676067, upload-time = "2026-07-22T11:51:18.163Z" }, + { url = "https://files.pythonhosted.org/packages/2e/d9/6298f3432de301d4718766cf934bd73c418c73f81fbb77247319364b0d96/greenlet-3.5.4-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:cd320d998cbaa032932830448e39abf3c6a12901295e386e8114db926e10cffb", size = 487446, upload-time = "2026-07-22T12:39:57.044Z" }, + { url = "https://files.pythonhosted.org/packages/5e/ba/863116ab8ff1ca7a729e327800268939d182db47aa433db70e216e7d9194/greenlet-3.5.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:c883d61f2282d72c767a14936641b3efcbde9d82f1080712aaea0b1d3126cb88", size = 1633489, upload-time = "2026-07-22T12:25:09.605Z" }, + { url = "https://files.pythonhosted.org/packages/fa/06/7466ced82818d6132462d7f26b3f83c66ea15d2b193a6c0088d558ed7d95/greenlet-3.5.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2a924f15d17957e252a810acefcb5942f5ca712298e8b6fcaed9a307d357522c", size = 1696584, upload-time = "2026-07-22T11:51:21.304Z" }, + { url = "https://files.pythonhosted.org/packages/f9/4d/55b638489260065de9ffce606c8b5d04507bef705de4b212a0c3d6a1a0df/greenlet-3.5.4-cp315-cp315-win_amd64.whl", hash = "sha256:ed17e5f3420360d5b459de8462efb52060399a5326a613d4cde31cef63ef95da", size = 248297, upload-time = "2026-07-22T11:42:04.055Z" }, + { url = "https://files.pythonhosted.org/packages/bb/08/9dd4ae635da93d41dc268bc34bd62a9d711ed8b8825c5d22ac910c7d6e6d/greenlet-3.5.4-cp315-cp315-win_arm64.whl", hash = "sha256:f908898d6fa484ce4b6f447ce70ea99b52c503fee419e53cf74d60a16bc9e667", size = 247423, upload-time = "2026-07-22T11:44:00.764Z" }, + { url = "https://files.pythonhosted.org/packages/19/66/7c87ed9cdbf1d49c2c6cd1c7b9dd4d16c33b24235ca03972293a1876b30c/greenlet-3.5.4-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:1833637f17d5e7472548a48575c394fe39f1b1890d676d162d86593610f44d8c", size = 306487, upload-time = "2026-07-22T11:41:25.118Z" }, + { url = "https://files.pythonhosted.org/packages/5b/05/0a4201e7c0054866eefc05da234f236dd4c950d0fbf9ca0517141f01b269/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:12cda9122e03341f1cb6b8207a19d7a9d375e52f1b4e9243918375f40fd7b4b9", size = 676479, upload-time = "2026-07-22T12:26:48.129Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c0/4b6b8c5a3aec70f0649cd89662d120fdd6421e2bdc8e3b15c3ab5ec568d8/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d83ae0e32d14957ab7170785a20f582635c8474deab1bfbb552b17e769a6ce25", size = 684321, upload-time = "2026-07-22T12:29:08.925Z" }, + { url = "https://files.pythonhosted.org/packages/88/15/0b167aeea95285b0e654ddce651922f666c089363c2ec528ca8b9a9ba74f/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:123aa379c962ed5fe90a880327e0c3066124ac64ec99e12a238be9fd8eb3db3d", size = 685995, upload-time = "2026-07-22T12:43:50.993Z" }, + { url = "https://files.pythonhosted.org/packages/24/c9/b49c31c9a972eee91e260445770e922244a7efc542697f51a012ec046d0f/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f1467de1bb767f75db0aa34c195e3a496d8d1278c796e70c24ce205d3e99cde", size = 681293, upload-time = "2026-07-22T11:51:20.43Z" }, + { url = "https://files.pythonhosted.org/packages/de/90/c023ec337f32ff505be7db759c80d98f0532bb94d0c6fa13645efe9bee2e/greenlet-3.5.4-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:adf2244d7f69409925a8f22ed22cc5f93cdfe5c9dc87ff3476be2c2aaae61a05", size = 516928, upload-time = "2026-07-22T12:39:58.359Z" }, + { url = "https://files.pythonhosted.org/packages/95/6f/7f2d4653770500eee667866016d42d7a68e3d3462f80df6b8e3fcd48a0eb/greenlet-3.5.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:0fa53040b78b578120eecdc0265e3f1051487cc425d11a2b7c761daadf4feaa8", size = 1642474, upload-time = "2026-07-22T12:25:10.819Z" }, + { url = "https://files.pythonhosted.org/packages/5a/d8/8cba31036a4caae448087ba5d150660ab03b4a0f54d9150f6495a3be7262/greenlet-3.5.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:60e0bc961d367df506660e9ac0177a76bc6d81305300704b0977d1634f76efe2", size = 1701012, upload-time = "2026-07-22T11:51:23.17Z" }, + { url = "https://files.pythonhosted.org/packages/e3/cd/3f77a4cce3bae631b08eb52f53a82a976669600337e21dfdba811cb50267/greenlet-3.5.4-cp315-cp315t-win_amd64.whl", hash = "sha256:f680e549edb3eaf21eea4e7fe101e15ec180c74b7879ab46adc080f22d4015d2", size = 251977, upload-time = "2026-07-22T11:41:38.125Z" }, + { url = "https://files.pythonhosted.org/packages/93/e8/65e8707d00fe2a49bf12f609a9b2b39ba6dd23c2810eacad877c4fc94bfe/greenlet-3.5.4-cp315-cp315t-win_arm64.whl", hash = "sha256:08fc36de8442d5c3e95b044550dbea9bf144d31ec0cc58e36fb241cb6ef6a994", size = 250538, upload-time = "2026-07-22T11:40:17.985Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1158,3 +1307,60 @@ sdist = { url = "https://files.pythonhosted.org/packages/3f/50/bad581df71744867e wheels = [ { url = "https://files.pythonhosted.org/packages/b7/1a/7e4798e9339adc931158c9d69ecc34f5e6791489d469f5e50ec15e35f458/zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931", size = 9630, upload-time = "2024-11-10T15:05:19.275Z" }, ] + +[[package]] +name = "zope-event" +version = "6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/93/41/faa10af34d48d9cd6fa0249a1162943ad84a9590bd1a06939981e6640416/zope_event-6.2.tar.gz", hash = "sha256:b97d5d6327067ee6b9dfcbdf606ade9ade70991e19c162e808ea39e5fcf0f8d3", size = 18958, upload-time = "2026-04-28T06:24:10.578Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/33/848922889e946d4befc415c219fe516af75c49555d8e736e183bfd30db42/zope_event-6.2-py3-none-any.whl", hash = "sha256:5e755153ac4faf64c10a4b6dd3307680166a3edf65b38df22df592610f8fa874", size = 6525, upload-time = "2026-04-28T06:24:09.176Z" }, +] + +[[package]] +name = "zope-interface" +version = "8.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/dc/50550cfcbb2ea3cbca5f1d7ed05c8aa840f831a0f2d63aec0a953f7c590e/zope_interface-8.5.tar.gz", hash = "sha256:7a3ba1c5877f0f3e3906b02ddf793abed2becc2948116414ce0e1dd820b68d6d", size = 257957, upload-time = "2026-05-26T06:50:14.574Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/43/9cd98bee951d23848de690ba2809f87e3b22c67c370987acc960da15ad37/zope_interface-8.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0c8aa2bf8f3911ef37b87deb1bbe225a310e6eb6522a16d77f5d8330c4f6fbe", size = 210951, upload-time = "2026-05-26T06:49:00.178Z" }, + { url = "https://files.pythonhosted.org/packages/17/0f/8f1a29966bcf863e3a2121edcafb81c55715de7886bcc9544749cc79e7da/zope_interface-8.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:efe234a0fafb4b6b1602e9be9245b97c2bf06d67c07af5a4bc3c0438978b555c", size = 211309, upload-time = "2026-05-26T06:49:02.732Z" }, + { url = "https://files.pythonhosted.org/packages/9f/9f/37e564eaaf85e3abc1ada40a79fa43f2ab45bdb67431b0ec0fe29e4763e2/zope_interface-8.5-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:dabeb6fe1228d411994f300811edc6866fff0cdcbc9cef98a78f05ea0da42e37", size = 254881, upload-time = "2026-05-26T06:49:04.303Z" }, + { url = "https://files.pythonhosted.org/packages/06/61/e6501d8ea7a2cac3217e03f404e1f98c1df7191d83cfe86b1895fbba5dac/zope_interface-8.5-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:147a9442dcc2b7339ecdb1be2b3cdb098e90462e39425054053ebfb50d99125a", size = 259811, upload-time = "2026-05-26T06:49:06.373Z" }, + { url = "https://files.pythonhosted.org/packages/91/15/bfa25ef480b02af6e9452c478483fec75e87c9e2b60c407fd0b1f6054b9c/zope_interface-8.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a17e681224267880707c9ec9e730ad9a1ad2d65c371256843efba6cf48711b58", size = 260358, upload-time = "2026-05-26T06:49:08.317Z" }, + { url = "https://files.pythonhosted.org/packages/64/51/2b518072fea76242da64451d501c69b7b5ccdef9b57fead584ccf1c180d5/zope_interface-8.5-cp310-cp310-win_amd64.whl", hash = "sha256:d178968a1a611df30549a717d1624cb38ca810347339e3e37b7baa6f6781a170", size = 214822, upload-time = "2026-05-26T06:49:10.441Z" }, + { url = "https://files.pythonhosted.org/packages/ce/f1/83ad110fb847413affe71609bb50e59e1aa082e1236030122227c7c283d3/zope_interface-8.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:afc66ccaef2a3c0bef6ca02aad40d29a39276389dad16a8eac36f9f385e4d057", size = 211426, upload-time = "2026-05-26T06:49:12.595Z" }, + { url = "https://files.pythonhosted.org/packages/bb/a7/6b6e0c31ac240cb9fc015ae9ed45ca54be886c18fcf7bfa2377a4d7a8785/zope_interface-8.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c28044972187245d7a309e4699319bfdbd2ffcbf7176d1d4ddf5adffb2dea80f", size = 211850, upload-time = "2026-05-26T06:49:14.474Z" }, + { url = "https://files.pythonhosted.org/packages/37/36/7599ecabcf80ce4fef2e1ef3c5ac0d4696b61f03f724cc44022f4d226af9/zope_interface-8.5-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:03bbecc7982af713d7499d4084bc03916413d17ffd45f89009348cc0c1d9e376", size = 260711, upload-time = "2026-05-26T06:49:16.568Z" }, + { url = "https://files.pythonhosted.org/packages/03/3e/1774b0ee46ccbb5498ee3c33ece40315b6ef58bc71957be94bd345340bc1/zope_interface-8.5-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bf917009a4a7457c7290225a019f4a0aa706d96accd2cfdba2418d3bc1fcde2f", size = 265277, upload-time = "2026-05-26T06:49:18.656Z" }, + { url = "https://files.pythonhosted.org/packages/b6/09/e533b2ffabaae4e5d5730d6768a591cf335defe8e37bec2ad905d09be656/zope_interface-8.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:31cff25b2aaedb5267e6e77b1e9be6b0ec4f622032de8a069202b8ffacda7dc2", size = 266369, upload-time = "2026-05-26T06:49:20.174Z" }, + { url = "https://files.pythonhosted.org/packages/49/4a/3ebe6a4c122b2d5340db45cbe7e490663d3228b172710ec71060cd5d541e/zope_interface-8.5-cp311-cp311-win_amd64.whl", hash = "sha256:17a3114bbdddb5e75e5784cdf318944636190cbbc72d357ef9fb1a8b0351f955", size = 215161, upload-time = "2026-05-26T06:49:21.799Z" }, + { url = "https://files.pythonhosted.org/packages/d2/59/056ad97af5b16db1975ee98ec7ab03d2ce3f3355efad904ced1dbce0e39f/zope_interface-8.5-cp311-cp311-win_arm64.whl", hash = "sha256:aab6bb5bee10f38ea688b95ba054396b67f613552d2c8378be7fcb2d2fba7646", size = 213481, upload-time = "2026-05-26T06:49:25.085Z" }, + { url = "https://files.pythonhosted.org/packages/97/cc/b84123a948f3162a34623e188922827cd845244fdd043ed20f8d02228caa/zope_interface-8.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8e6ee90c2e6de7c37058d5fa41f123c8b13a312db8d1e0fb5840d7f4bcdff9c9", size = 212165, upload-time = "2026-05-26T06:49:26.566Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/cbceec44f1b27208a76c1a688c131302685852406a23df5aab68324109cc/zope_interface-8.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c1adc90d3576b3b4c4de4953e6002c37bef28b78d7fa54c1bbfd0c50f022fe7c", size = 212341, upload-time = "2026-05-26T06:49:28.182Z" }, + { url = "https://files.pythonhosted.org/packages/e1/c3/005032195ff3b210c139b7c560ed5c534e844b0907d8e44d2b3d8919305e/zope_interface-8.5-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:e6347b8d8d12c5eca6502450a92be30079b7acfade2c4f693efa0deb8871b06e", size = 265296, upload-time = "2026-05-26T06:49:29.741Z" }, + { url = "https://files.pythonhosted.org/packages/c5/66/1036543d6a66bc04c19df3cf650f3ad938a002ab0a443c24e23e8de5e8b9/zope_interface-8.5-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5e970dabea777a24b0b0bbf9dae3ab75ce8b2d8e948edf4875627034b21f3560", size = 270689, upload-time = "2026-05-26T06:49:31.767Z" }, + { url = "https://files.pythonhosted.org/packages/30/4c/8b56259558cace4414e753ca6740396a1f59d4a95ddb55b4658600408670/zope_interface-8.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f0b48ccadaa9839e09ff81e969703cecb3f402c813bfe8b958652e699bea69f5", size = 270280, upload-time = "2026-05-26T06:49:33.489Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ea/649908c83aa8fdb7faf2ddca4d3cf6fb8f2157121267dc56e8f72681e26c/zope_interface-8.5-cp312-cp312-win_amd64.whl", hash = "sha256:e0e311f1277468c08fd59a2b41f71b43d25dff639789d364747acd1705c0df6e", size = 215019, upload-time = "2026-05-26T06:49:35.607Z" }, + { url = "https://files.pythonhosted.org/packages/9f/97/da13037b4c563e4df32eedbc819f8c00b754af494f68211e3dffd48d52da/zope_interface-8.5-cp312-cp312-win_arm64.whl", hash = "sha256:652b73107a04159ec6c020db6c1543d4f1e8f4d069bd2aac88a947820923517b", size = 213569, upload-time = "2026-05-26T06:49:37.317Z" }, + { url = "https://files.pythonhosted.org/packages/f4/8c/4c15755d701f2ec0e80d64a18e1ebaf5be2c584c0ec153fd516f5d13eada/zope_interface-8.5-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:28e80457c134d1fa57a7d758004dece348654e1b1467ac22dcdc20fc1d127c52", size = 212512, upload-time = "2026-05-26T06:49:38.996Z" }, + { url = "https://files.pythonhosted.org/packages/9a/2e/4360c54c465db042cc8fbeeec92abac28b4cedbf6ba63c1f092fd08a190f/zope_interface-8.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:09495ce9d559c06b70f2d4855b3e4f48a822a9ddc8be1d30c5b4e5be14ae1ace", size = 212541, upload-time = "2026-05-26T06:49:41.186Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a5/692a2b8d70f78e848793231d5fae5fecbf8d0cccd73430fdc34802a6d3c1/zope_interface-8.5-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:7849ad8fa90763cc1087f4dda78ca3a233e950b3e08fac7079297c9cafbbd7bb", size = 265191, upload-time = "2026-05-26T06:49:43.449Z" }, + { url = "https://files.pythonhosted.org/packages/70/8d/454a9cfc7a050c394ab4f11b3371f7897828b7415e096afff724637e65e0/zope_interface-8.5-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5578c9421ca409a1f39f153d6f7803e4cde01da592ec75a9ac5e1b777d18d33b", size = 270626, upload-time = "2026-05-26T06:49:45.425Z" }, + { url = "https://files.pythonhosted.org/packages/51/8c/db8409cfa3575b8e9b4800babd7d49f8228433cd1f0c56814bd0ada49c33/zope_interface-8.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e1bd7d96b4ca5fa311f54c9eac16dce4886b428c1531dbe06067763ccdf123b4", size = 270444, upload-time = "2026-05-26T06:49:47.025Z" }, + { url = "https://files.pythonhosted.org/packages/4a/df/a386940e41469ef615e100a216d8b386521e9e598817147f87932ca203c4/zope_interface-8.5-cp313-cp313-win_amd64.whl", hash = "sha256:0c8123d2a4dfde2a613c7cb772605477724782c20bc2e0ad1d9435376a6a44a3", size = 215021, upload-time = "2026-05-26T06:49:48.478Z" }, + { url = "https://files.pythonhosted.org/packages/89/75/477eb5669b6b2a7a843decd1a075e9b1971a8720017654143a7183abd3d9/zope_interface-8.5-cp313-cp313-win_arm64.whl", hash = "sha256:6d02be14f3173c6c7288bc2fdf530090c01c3cf8764ad46c68024686f364278e", size = 213610, upload-time = "2026-05-26T06:49:50.01Z" }, + { url = "https://files.pythonhosted.org/packages/d4/19/5032e954827fdf02db2d2f49737ac4378bb9cfc2cd95a8f2e2a5ae2ec01a/zope_interface-8.5-cp314-cp314-macosx_10_9_x86_64.whl", hash = "sha256:ffaecf013251a89d0de6feb49a46eba48ad8cbbf8a40aeb6045e459e7bec6784", size = 212597, upload-time = "2026-05-26T06:49:51.63Z" }, + { url = "https://files.pythonhosted.org/packages/f1/53/3ef644012cf8a6a234a2d6134aab5a5c65ac5467c86296865501d4fbc406/zope_interface-8.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:126fa9d1c52295ae076d4cf968634f0a1826afa408a20808b57ff72877b8f69f", size = 212626, upload-time = "2026-05-26T06:49:53.236Z" }, + { url = "https://files.pythonhosted.org/packages/32/67/bc8b4f465d388039255003e230c284a175cedf1203c692f23cb7bff64efe/zope_interface-8.5-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:3090e3a663d20194756a59a272e0c8508b889341e31d5894223331fe6b4f9b21", size = 266827, upload-time = "2026-05-26T06:49:54.873Z" }, + { url = "https://files.pythonhosted.org/packages/a7/eb/37d05b935ede53d79690fecc8d201440084418e590bcfc05f384451c7593/zope_interface-8.5-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9342fb74e2afefdb081bf1df727d209ea56995c6e13f5a0540e6d7aff4beafb8", size = 270139, upload-time = "2026-05-26T06:49:57.116Z" }, + { url = "https://files.pythonhosted.org/packages/8b/0b/fd0c54579e2ce8dc6cf1a757903f3374bc6fbda929a46af9e0f53cb0e5f0/zope_interface-8.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6c54725d818f1b57a7efb8b16528326e1f3c257b602b32393fd255c45af8799d", size = 270338, upload-time = "2026-05-26T06:49:58.698Z" }, + { url = "https://files.pythonhosted.org/packages/c1/1d/c420dcd777bb761067ea92879ac766694a5ca78608185f1aecea64cbfc11/zope_interface-8.5-cp314-cp314-win_amd64.whl", hash = "sha256:29d74febbae1afeb6834c4ccbf42e242a673c860060f09e53142825270456140", size = 215789, upload-time = "2026-05-26T06:50:00.405Z" }, + { url = "https://files.pythonhosted.org/packages/62/94/50b5eb8f94e527edceac14f9955e58917424ea79bb572ddc18548561cbc2/zope_interface-8.5-cp314-cp314-win_arm64.whl", hash = "sha256:633c8c49396f38df030340797c533e9fe460d1b5d1e42d88e55e938e525f548c", size = 213757, upload-time = "2026-05-26T06:50:01.973Z" }, + { url = "https://files.pythonhosted.org/packages/17/6f/5d5f32c4dfcdb16ce2ec5363da686840f13c13e1a1214cb70b49e1cd6d9f/zope_interface-8.5-cp314-cp314t-macosx_10_9_x86_64.whl", hash = "sha256:133999820fdbae513c36c03d6f29ef87317aaa3edef39112222b155083664714", size = 213591, upload-time = "2026-05-26T06:50:03.529Z" }, + { url = "https://files.pythonhosted.org/packages/f3/55/de0c3459ff717fce3342f9a29464c281fdeb0d36c3171ee88d119d5f0650/zope_interface-8.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8bd75c96966e573232f0599deaff717564828031c7f05563ccc1ac35c5ee0304", size = 213733, upload-time = "2026-05-26T06:50:05.101Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/d97430abd5ae9677e8b9295b58720c0064a5b557dbb6b8bf5928484cf0d8/zope_interface-8.5-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:14b0e9799351d4c34fe99afd67f0cdd76e55ba15c66a98699d5fc22ea8241e08", size = 294905, upload-time = "2026-05-26T06:50:07.384Z" }, + { url = "https://files.pythonhosted.org/packages/41/ec/a0f8f3dad6e74992f4654bdd94802be0929eabca7b871cac3b6fbb5e961b/zope_interface-8.5-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cd6a732ac84b94eb1ef9222a117347a27efd294ee16810ffdf7ecd307677ed5", size = 300885, upload-time = "2026-05-26T06:50:08.997Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/6881b48803a0ee8d23eb5efa30fce3ed218a2bd9de5758ce489d224fee81/zope_interface-8.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:798b7c87d0e59a7d5d086d642208d0d8700ff0d55c4029134b3c479c3bfb110f", size = 304672, upload-time = "2026-05-26T06:50:10.563Z" }, + { url = "https://files.pythonhosted.org/packages/2e/0e/b4c01320859ff1d585438bc231fd60bd258d096359bccf6654fecdf0cffb/zope_interface-8.5-cp314-cp314t-win_amd64.whl", hash = "sha256:0fc3a9d45f114d27eaa1e53beeb144533689edca8a9f66505b1e8e8b3f075e42", size = 217241, upload-time = "2026-05-26T06:50:12.171Z" }, +] From 5df051ce664f023213af9c36b1078748321972b8 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:23:28 +0200 Subject: [PATCH 40/91] docs: record the landed boundary rethink (P1-P5) in the handoff Co-Authored-By: Claude Fable 5 --- handoff-boundary-protocol-rethink.md | 48 +++++++++++++++++++--------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/handoff-boundary-protocol-rethink.md b/handoff-boundary-protocol-rethink.md index 52336693..59cd1141 100644 --- a/handoff-boundary-protocol-rethink.md +++ b/handoff-boundary-protocol-rethink.md @@ -98,21 +98,39 @@ portal ingress + `OneShot`/`Mailbox` on an `AsyncioWakener`. Full asyncio-native API over the trio host loop, all transports, no anyio port. Phase E (anyio-native core) becomes optional purity/perf work. -## Staging - -1. **P1 — boundary kit**: Wakener/Mailbox/OneShot + ThreadWakener; port - SyncReceiver, write-acks, `_done_sync` onto it. Pure refactor. -2. **P2 — channel unification** (big one): sync Channel onto - RawChannel+Mailbox, delete classic dispatch. Straight to the rebase, - no intermediate primitive-swap (suite pins semantics either way). - Canary: `uv run pytest testing/ -n 12` + xdist suite mid-step. -3. **P3 — retirement**: ExecModel/WorkerPool out, preset shims in, - `wait=` joins the C.1 spec axes + worker CLI config. -4. **P4 — `execnet.aio`**. -5. **P5 — gevent wakener** + opt-in CI job (gevent in a dev extra). - -Then Phase C `loop=main` / `exec=task` on the smaller core, and Phase D -docs cover the three namespaces + `execnet.aio` + the axes/presets. +## Staging — ALL LANDED 2026-07-26 (commits 2c78416..b01d009) + +1. **P1 — boundary kit** (`2c78416`): Wakener/Mailbox/OneShot + + ThreadWakener; SyncReceiver, write-acks, `_done_sync` ported. +2. **P2 — channel unification** (`2a40a55`): sync Channel onto + RawChannel(+set_consumer)+Mailbox, classic dispatch deleted. The kit + lives in trio-free `execnet._boundary` (portal re-exports) so + `import execnet` stays trio-free. Fix worth knowing: an unconsumed + remotely-closed RawChannel stays registered until a consumer claims + it — call-site deserialization means a passed channel can bind after + its data AND close arrived (was a payload-loss race, latent in the + async core's `open_channel(id)` too). +3. **P3 — retirement** (`811fd99`): ExecModel ABC/WorkerPool/Reply gone; + one deprecated ExecModel preset KEEPS the stdlib-delegate members + because pytest-xdist's remote worker calls + `channel.gateway.execmodel.RLock()/Event()`; `wait=` axis end to end + (spec validation, worker CLI config, `BaseGateway._new_wakener`, + `_boundary` registry + Flag); safe_terminate on daemon threads; + test_threadpool.py deleted. +4. **P4 — `execnet.aio`** (`845510e`): implemented as a per-call + _HostBridge (host task + `loop.call_soon_threadsafe` future), NOT the + originally sketched AsyncioWakener/Mailbox — real awaitables over the + trio-native AsyncGroup/AsyncChannel, simpler and semantically exact. + The "asyncio" wait= registry entry is therefore unused/unregistered. + v1 limitation: cancelling a bridged await abandons it aio-side only. +5. **P5 — gevent wakener** (`b01d009`): `execnet._gevent_support`, + lazily imported by `make_wakener("gevent")`; hub-bound pieces created + in the first waiter's hub (carriers may be constructed on the host + loop); tests behind the `gevent` dependency group. + +Next: Phase C `loop=main` / `exec=task` on the smaller core +(handoff-phase-c-worker-axes.md), and Phase D docs cover the four +namespaces + the axes/presets. ## Invariants (unchanged, re-mapped) From 7969f7e37daea12c5b6543c0ed2680db96d81275 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 15:47:43 +0200 Subject: [PATCH 41/91] docs: record the decided Phase C plan (axes matrix, trio-only core) Co-Authored-By: Claude Fable 5 --- handoff-phase-c-worker-axes.md | 41 +++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index 351f6c16..40c222b6 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -71,7 +71,46 @@ File map (src/execnet/): `threading.Event`/`queue.Queue` so KeyboardInterrupt can interrupt them; `portal.run` (KI-deferred) is only for management ops. -## Phase C — what is missing (nothing of C exists yet) +## Phase C — DECIDED PLAN (Ronny, 2026-07-26, after the boundary rethink) + +Predates below gap analysis; where they conflict, this section wins. +Context: the boundary rethink (`handoff-boundary-protocol-rethink.md`) +landed first — `wait=` axis exists, ExecModel is a preset, the sync +Channel is a facade over the async core. + +**Worker matrix** (`loop=` = main-thread role, `exec=` = placement). +The main thread is the scarce resource (signals, interrupt_main, GUI, +tests calling asyncio.run/trio.run themselves): + +| loop= | exec= | main thread | exec'd code | channel | consumer | +|---|---|---|---|---|---| +| main | thread | host loop | worker threads | sync | default preset for execmodel=thread; no parked thread | +| main | task | host loop | tasks on the loop | AsyncChannel | async-native sources (C4) | +| thread | main | exec'd code (serialized) | true main thread | sync | pytest/xdist (forces main_thread_only today), GUI, signals | +| thread | thread | parked | worker threads | sync | valid, non-default (today's shape) | +| thread | task | parked | tasks on side loop | AsyncChannel | valid, niche | +| main | main | — | — | — | invalid | + +Interaction rule: exec'd code may start its own event loop only when it +does not share a thread with ours (exec=thread / exec=main); exec=task +sources are ``async def`` and join our loop — a sync source under +exec=task is a hard error. ``wait=`` composes orthogonally. + +**Backend decisions**: the core STAYS trio-only and trio stays the +default and a hard dependency (pure-python wheels; deployment cost +accepted). The anyio/asyncio-core port ("C0") is REJECTED for now; +asyncio apps use the ``execnet.aio`` bridge. ``backend=`` still lands +as a per-gateway spec axis, but reserved: only ``trio`` validates +(exactly how ``wait=`` landed with only ``thread``), keeping an +asyncio-core door open without paying for it. Consequence: exec=task +sources are trio-typed. + +Sequencing: C1 axes (`backend=`/`loop=`/`exec=` + validity matrix + +presets) → C2 `loop=main` worker restructure → C3 `exec=main` re-home → +C4 `exec=task`. Presets: `execmodel=thread` → `loop=main, exec=thread`; +`execmodel=main_thread_only` → `loop=thread, exec=main`. + +## Phase C — original gap analysis (pre-decision record) Target axes: `loop=thread|main` × `exec=thread|main|task`. Compat mapping: `execmodel=thread` → `loop=main` + `exec=thread`; From b2f43c38d0bb53265d481064e36f6216c5542b5e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 16:35:31 +0200 Subject: [PATCH 42/91] refactor: extract worker exec placement strategies behind the FIFO pump C1 of the profile plan (approved 2026-07-26): TrioWorkerExec becomes a pure admission pump (message-arrival FIFO preserved) delegating placement to a strategy object -- PoolExec (worker threads, today's "thread") and MainExec (serialized main thread + deadlock guard, today's "main_thread_only"), selected via WORKER_EXEC_STRATEGIES by execmodel name. _run_worker keys main-thread integration off strategy.needs_primary_thread instead of a hardcoded flag. No behavior change; upcoming profiles (trio task exec, classic hybrid, gevent greenlets, later subinterpreters) slot in as new strategies. Co-Authored-By: Claude Fable 5 --- src/execnet/_gevent_support.py | 2 +- src/execnet/_trio_worker.py | 177 +++++++++++++++++++++++---------- src/execnet/multi.py | 5 +- 3 files changed, 129 insertions(+), 55 deletions(-) diff --git a/src/execnet/_gevent_support.py b/src/execnet/_gevent_support.py index cd761571..a999ab96 100644 --- a/src/execnet/_gevent_support.py +++ b/src/execnet/_gevent_support.py @@ -63,7 +63,7 @@ def wait(self, timeout: float | None = None) -> bool: # a notify may have fired before the watcher existed if self._notified and not event.is_set(): event.set() - return event.wait(timeout) + return bool(event.wait(timeout)) def clear(self) -> None: with self._lock: diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 89aad122..c5152cc2 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -6,6 +6,7 @@ import os import sys import threading +from collections.abc import Callable from typing import TYPE_CHECKING from typing import Any @@ -26,40 +27,128 @@ ExecItem = tuple[Any, ...] +class PoolExec: + """Exec strategy: run each request on a worker thread (trio's pool). + + The building block of the ``thread`` profile; exec'd code may freely + start its own event loops (it never shares a thread with ours). + """ + + #: whether _run_worker must hand this strategy the process main thread + needs_primary_thread = False + + def __init__(self, gateway: WorkerGateway) -> None: + self.gateway = gateway + + async def admit(self, channel: Channel, item: ExecItem) -> bool: + """FIFO admission gate; always open for pool placement.""" + return True + + async def run(self, channel: Channel, item: ExecItem) -> None: + await trio.to_thread.run_sync( + self.gateway.executetask, + (channel, item), + abandon_on_cancel=True, + ) + + def integrate_as_primary_thread(self) -> None: + raise RuntimeError("pool exec strategy does not use the main thread") + + def trigger_shutdown(self) -> None: + pass + + +class MainExec: + """Exec strategy: serialize each request onto the process main thread. + + The ``main_thread_only`` profile (GUI/signal-safe: pytest under xdist + runs this way). Admission waits for the previous request to finish and + closes the channel with the deadlock text when it cannot within a + second (a second concurrent remote_exec would deadlock the requester). + """ + + needs_primary_thread = True + + def __init__(self, gateway: WorkerGateway) -> None: + self.gateway = gateway + self._primary: Mailbox[tuple[Channel, ExecItem, threading.Event] | None] = ( + Mailbox() + ) + + async def admit(self, channel: Channel, item: ExecItem) -> bool: + complete = self.gateway._executetask_complete + assert complete is not None + wait_slot = functools.partial(complete.wait, timeout=1) + if not await trio.to_thread.run_sync(wait_slot, abandon_on_cancel=True): + channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) + return False + complete.clear() + return True + + async def run(self, channel: Channel, item: ExecItem) -> None: + done = threading.Event() + self._primary.put((channel, item, done)) + await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) + + def integrate_as_primary_thread(self) -> None: + """Block the main thread running exec tasks until shutdown.""" + while True: + task = self._primary.get() + if task is None: + break + channel, item, done = task + try: + self.gateway.executetask((channel, item)) + finally: + done.set() + + def trigger_shutdown(self) -> None: + self._primary.put(None) + + +# execmodel profile -> exec strategy. Placement strategies to come: +# "trio" (tasks on a main-thread loop), classic hybrid for "thread" +# (primary main + pool overflow), "gevent" (greenlets on a main-thread +# hub), and eventually subinterpreters. +WORKER_EXEC_STRATEGIES: dict[str, Callable[[WorkerGateway], Any]] = { + "thread": PoolExec, + "main_thread_only": MainExec, +} + + class TrioWorkerExec: - """Schedule ``remote_exec`` work from the Trio host nursery. + """FIFO admission pump feeding an exec placement strategy. - * ``thread``: run ``executetask`` via ``trio.to_thread`` (concurrent). - * ``main_thread_only``: hand off to the process main thread (GUI-safe). + Exec requests flow through a single pump task so admission happens + strictly in message-arrival order (trio task scheduling order is + deliberately unordered, so per-request tasks would race for e.g. the + main_thread_only slot). Where an admitted request runs is the + strategy's business (:data:`WORKER_EXEC_STRATEGIES`). """ def __init__( self, host: _trio_host.TrioHost, gateway: WorkerGateway, - *, - main_thread_only: bool, + strategy: Any, ) -> None: self.host = host self.gateway = gateway - self.main_thread_only = main_thread_only + self.strategy = strategy self._lock = threading.Lock() self._running = 0 self._shutting_down = False self._idle = threading.Event() self._idle.set() - self._primary: Mailbox[tuple[Channel, ExecItem, threading.Event] | None] = ( - Mailbox() - ) - # Exec requests flow through a single pump task so admission happens - # strictly in message-arrival order (trio task scheduling order is - # deliberately unordered, so per-request tasks would race for the - # main_thread_only slot). self._pending_send: trio.MemorySendChannel[tuple[Channel, ExecItem]] self._pending_recv: trio.MemoryReceiveChannel[tuple[Channel, ExecItem]] self._pending_send, self._pending_recv = trio.open_memory_channel(float("inf")) self._pump_started = False + @property + def needs_primary_thread(self) -> bool: + return bool(self.strategy.needs_primary_thread) + def active_count(self) -> int: with self._lock: return self._running @@ -78,7 +167,7 @@ def _track_finish(self) -> None: def schedule(self, channel: Channel, sourcetask: bytes) -> None: """Called from the session dispatch on the Trio host thread. - Must not block: deadlock checks and exec run in a nursery task. + Must not block: admission checks and exec run in a nursery task. """ item = loads_internal(sourcetask) assert isinstance(item, tuple) @@ -95,48 +184,23 @@ def schedule(self, channel: Channel, sourcetask: bytes) -> None: async def _pump(self) -> None: """Admit queued exec requests in FIFO order, then run each as a task.""" async for channel, item in self._pending_recv: - if self.main_thread_only: - complete = self.gateway._executetask_complete - assert complete is not None - wait_slot = functools.partial(complete.wait, timeout=1) - if not await trio.to_thread.run_sync(wait_slot, abandon_on_cancel=True): - channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) - continue - complete.clear() - self.host.start_soon(self._run_exec, channel, item) + if await self.strategy.admit(channel, item): + self.host.start_soon(self._run_exec, channel, item) async def _run_exec(self, channel: Channel, item: ExecItem) -> None: self._track_start() try: - if self.main_thread_only: - done = threading.Event() - self._primary.put((channel, item, done)) - await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) - else: - await trio.to_thread.run_sync( - self.gateway.executetask, - (channel, item), - abandon_on_cancel=True, - ) + await self.strategy.run(channel, item) finally: self._track_finish() def integrate_as_primary_thread(self) -> None: - """Block the main thread running main_thread_only exec tasks.""" - while True: - task = self._primary.get() - if task is None: - break - channel, item, done = task - try: - self.gateway.executetask((channel, item)) - finally: - done.set() + self.strategy.integrate_as_primary_thread() def trigger_shutdown(self) -> None: with self._lock: self._shutting_down = True - self._primary.put(None) + self.strategy.trigger_shutdown() def waitall(self, timeout: float | None = None) -> bool: return self._idle.wait(timeout) @@ -202,30 +266,37 @@ def kill(self) -> None: def _build_worker_gateway( host: _trio_host.TrioHost, id: str, model: ExecModel, wait: str = "thread" -) -> tuple[WorkerGateway, TrioWorkerExec, bool]: - """Construct the WorkerGateway + Trio exec pool (no IO yet).""" +) -> tuple[WorkerGateway, TrioWorkerExec]: + """Construct the WorkerGateway + exec pump/strategy (no IO yet).""" trace(f"creating workergateway on trio id={id!r}") io_stub = _WorkerIOStub(model) gateway = WorkerGateway(io=io_stub, id=id, _startcount=2) gateway._wait_backend = wait - main_thread_only = model.backend == "main_thread_only" - trio_exec = TrioWorkerExec(host, gateway, main_thread_only=main_thread_only) - # Duck-type as WorkerPool for STATUS / _terminate_execution. + try: + strategy_factory = WORKER_EXEC_STRATEGIES[model.backend] + except KeyError: + raise ValueError( + f"execmodel {model.backend!r} has no worker exec strategy " + f"(known: {sorted(WORKER_EXEC_STRATEGIES)})" + ) from None + strategy = strategy_factory(gateway) + trio_exec = TrioWorkerExec(host, gateway, strategy) + # Duck-type as the exec pool for STATUS / _terminate_execution. gateway._execpool = trio_exec gateway._trio_exec = trio_exec gateway._executetask_complete = None - if main_thread_only: + if strategy.needs_primary_thread: gateway._executetask_complete = threading.Event() gateway._executetask_complete.set() - return gateway, trio_exec, main_thread_only + return gateway, trio_exec def _run_worker( host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel, wait: str = "thread" ) -> None: """Attach ``io`` as the gateway session and serve until shutdown.""" - gateway, trio_exec, main_thread_only = _build_worker_gateway(host, id, model, wait) + gateway, trio_exec = _build_worker_gateway(host, id, model, wait) async def _start() -> _trio_host.SyncBridgeGateway: # The bridge attaches itself to the gateway before serving starts, @@ -235,7 +306,7 @@ async def _start() -> _trio_host.SyncBridgeGateway: host.call(_start) try: - if main_thread_only: + if trio_exec.needs_primary_thread: trace("integrating as primary thread (trio worker)") trio_exec.integrate_as_primary_thread() gateway.join() diff --git a/src/execnet/multi.py b/src/execnet/multi.py index ba73bf62..4bbfa7cf 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -154,7 +154,10 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: id= specifies the gateway id python= specifies which python interpreter to execute - execmodel=model 'thread' or 'main_thread_only' execution model + execmodel=name worker profile: where exec'd code runs relative + to the worker's protocol loop. 'thread' (pool + threads) or 'main_thread_only' (serialized on + the worker main thread, GUI/signal-safe). wait=backend wakener for blocking waits ('thread' default) chdir= specifies to which directory to change nice= specifies process priority of new process From d8f22c5f269536bac36a8de2f2d503a150b870eb Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 16:44:20 +0200 Subject: [PATCH 43/91] feat!: answer info natively and apply chdir/nice/env from worker config C2 of the profile plan: coordinator bookkeeping no longer goes through remote_exec, so it can never claim an exec slot -- with main-thread profiles an info call used to occupy the main thread (or trip the main_thread_only deadlock guard) and push the real workload (pytest) onto a worker thread. - New Message.GATEWAY_INFO (code 10), answered by the dispatch loop like STATUS with the sys/env payload (gateway_base.gateway_info); Gateway._rinfo() sends it instead of executing rinfo_source, which is deleted. Handled in both the sync bridge and the async core. - chdir/nice/env ship in the worker config JSON (worker_cli_arg; the via spawn_request inherits it) and are applied by the worker before serving; the post-start remote_exec setup block in multi.makegateway is gone. This also makes setup valid for the upcoming pure-async profile, which cannot run sync sources at all. Regression test: _rinfo(update=True) while an exec occupies the worker, for both execmodels. Co-Authored-By: Claude Fable 5 --- src/execnet/_provision.py | 23 +++++++++++++++-------- src/execnet/_trio_gateway.py | 6 ++++++ src/execnet/_trio_host.py | 8 ++++++++ src/execnet/_trio_worker.py | 20 ++++++++++++++++++++ src/execnet/gateway.py | 34 ++++++++++++---------------------- src/execnet/gateway_base.py | 19 +++++++++++++++++++ src/execnet/multi.py | 21 ++------------------- testing/test_basics.py | 16 ---------------- testing/test_gateway.py | 13 +++++++++++++ 9 files changed, 95 insertions(+), 65 deletions(-) diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index f5114ad8..ffe9ffaa 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -164,14 +164,21 @@ def worker_cli_arg(spec: Any) -> str: """ import execnet - return json.dumps( - { - "id": f"{spec.id}-worker", - "execmodel": spec.execmodel, - "wait": spec.wait or "thread", - "coordinator_version": execnet.__version__, - } - ) + config: dict[str, Any] = { + "id": f"{spec.id}-worker", + "execmodel": spec.execmodel, + "wait": spec.wait or "thread", + "coordinator_version": execnet.__version__, + } + # Startup setup applied by the worker before serving (never through + # remote_exec: an exec slot must not be claimed by bookkeeping). + if spec.chdir: + config["chdir"] = spec.chdir + if spec.nice: + config["nice"] = int(spec.nice) + if spec.env: + config["env"] = spec.env + return json.dumps(config) def _worker_tokens(config: str) -> list[str]: diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index fc364772..66a892e8 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -46,6 +46,7 @@ from .gateway_base import TimeoutError from .gateway_base import Unserializer from .gateway_base import dumps_internal +from .gateway_base import gateway_info from .gateway_base import loads_internal from .gateway_base import trace @@ -678,6 +679,11 @@ def _dispatch(self, message: Message) -> None: } self._send_nowait(Message.CHANNEL_DATA, channelid, dumps_internal(status)) self._send_nowait(Message.CHANNEL_CLOSE, channelid) + elif code == Message.GATEWAY_INFO: + self._send_nowait( + Message.CHANNEL_DATA, channelid, dumps_internal(gateway_info()) + ) + self._send_nowait(Message.CHANNEL_CLOSE, channelid) elif code == Message.RECONFIGURE: data = loads_internal(message.data) assert isinstance(data, tuple) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index e29af4d2..9a1f1338 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -37,6 +37,7 @@ from .gateway_base import Message from .gateway_base import RemoteError from .gateway_base import dumps_internal +from .gateway_base import gateway_info from .gateway_base import loads_internal from .gateway_base import trace from .portal import LoopPortal @@ -177,6 +178,13 @@ def _dispatch(self, message: Message) -> None: try: if code == Message.STATUS: self._answer_status(message) + elif code == Message.GATEWAY_INFO: + gateway._send( + Message.CHANNEL_DATA, + message.channelid, + dumps_internal(gateway_info()), + ) + gateway._send(Message.CHANNEL_CLOSE, message.channelid) elif code == Message.CHANNEL_EXEC: channel = gateway._channelfactory.new(message.channelid) gateway._local_schedulexec(channel=channel, sourcetask=message.data) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index c5152cc2..56e64ef3 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -377,6 +377,25 @@ def _rough_version(version: str) -> tuple[int, ...]: return tuple(parts) +def _apply_worker_setup(config: dict[str, Any]) -> None: + """Apply chdir/nice/env from the worker config, before serving starts. + + This replaces the classic post-start ``remote_exec`` setup: valid for + every profile (a trio worker cannot run sync sources) and never + claims an exec slot. + """ + path = config.get("chdir") + if path: + if not os.path.exists(path): + os.mkdir(path) + os.chdir(path) + nice = config.get("nice") + if nice and hasattr(os, "nice"): + os.nice(nice) + for name, value in config.get("env", {}).items(): + os.environ[name] = value + + def _check_version(coordinator_version: str) -> None: """Warn on a real (major/minor) execnet version mismatch across the wire. @@ -416,6 +435,7 @@ def _main() -> None: config = json.loads(ns.config) _check_version(config["coordinator_version"]) + _apply_worker_setup(config) if ns.socket_fd is not None: serve_socket_trio( config["id"], diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index bec4c778..672a0830 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -26,7 +26,6 @@ "RemoteStatus", "_find_non_builtin_globals", "_source_of_function", - "rinfo_source", ] @@ -94,13 +93,19 @@ def reconfigure( self._send(Message.RECONFIGURE, data=data) def _rinfo(self, update: bool = False) -> RInfo: - """Return some sys/env information from remote.""" + """Return some sys/env information from remote. + + A native protocol request (like ``remote_status``): it never + touches the exec machinery, so it cannot claim an exec slot on + main-thread-shaped workers. + """ if update or not hasattr(self, "_cache_rinfo"): - ch = self.remote_exec(rinfo_source) - try: - self._cache_rinfo = RInfo(ch.receive()) - finally: - ch.waitclose() + channel = self.newchannel() + self._send(Message.GATEWAY_INFO, channel.id) + self._cache_rinfo = RInfo(channel.receive()) + # the other side didn't actually instantiate a channel + # so we just delete the internal id/channel mapping + self._channelfactory._local_close(channel.id) return self._cache_rinfo def hasreceiver(self) -> bool: @@ -166,18 +171,3 @@ def __getattr__(self, name: str) -> Any: ... RemoteStatus = RInfo - - -def rinfo_source(channel) -> None: - import os - import sys - - channel.send( - dict( - executable=sys.executable, - version_info=sys.version_info[:5], - platform=sys.platform, - cwd=os.getcwd(), - pid=os.getpid(), - ) - ) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 6a286302..c0ac5252 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -187,6 +187,7 @@ class only carries the framing and the code constants. CHANNEL_LAST_MESSAGE = 7 GATEWAY_START_SOCKET = 8 GATEWAY_START_SUB = 9 + GATEWAY_INFO = 10 # message code -> name _types: dict[int, str] = { @@ -200,6 +201,7 @@ class only carries the framing and the code constants. CHANNEL_LAST_MESSAGE: "CHANNEL_LAST_MESSAGE", GATEWAY_START_SOCKET: "GATEWAY_START_SOCKET", GATEWAY_START_SUB: "GATEWAY_START_SUB", + GATEWAY_INFO: "GATEWAY_INFO", } def __init__(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: @@ -243,6 +245,23 @@ def __repr__(self) -> str: return f"" +def gateway_info() -> dict[str, object]: + """Payload for ``Message.GATEWAY_INFO``: sys/env facts about this side. + + Answered natively by the dispatch loop -- an info request never + touches the exec machinery, so it cannot claim an exec slot (with + main-thread profiles, an info call stealing the primary slot used to + push the real workload onto a worker thread). + """ + return { + "executable": sys.executable, + "version_info": tuple(sys.version_info[:5]), + "platform": sys.platform, + "cwd": os.getcwd(), + "pid": os.getpid(), + } + + class FrameDecoder: """Incremental decoder for the 9-byte-header Message framing. diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 4bbfa7cf..ac88b181 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -183,25 +183,8 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: gw = _trio_host.makegateway_trio(self, spec) gw.spec = spec self._register(gw) - if spec.chdir or spec.nice or spec.env: - channel = gw.remote_exec( - """ - import os - path, nice, env = channel.receive() - if path: - if not os.path.exists(path): - os.mkdir(path) - os.chdir(path) - if nice and hasattr(os, 'nice'): - os.nice(nice) - if env: - for name, value in env.items(): - os.environ[name] = value - """ - ) - nice = (spec.nice and int(spec.nice)) or 0 - channel.send((spec.chdir, nice, spec.env)) - channel.waitclose() + # chdir/nice/env travel in the worker config and are applied at + # worker startup -- no remote_exec, so no exec slot is claimed. return gw def allocate_id(self, spec: XSpec) -> None: diff --git a/testing/test_basics.py b/testing/test_basics.py index c58e4728..470bec6d 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -153,22 +153,6 @@ def test_io_message(checker: Checker) -> None: assert "all passed" in out.stdout -def test_rinfo_source(checker: Checker) -> None: - out = checker.run_check( - f""" -class Channel: - def send(self, data): - assert eval(repr(data), {{}}) == data -channel = Channel() -{inspect.getsource(gateway.rinfo_source)} -print ('all passed') -""" - ) - - print(out.stdout) - assert "all passed" in out.stdout - - def test_geterrortext(checker: Checker) -> None: out = checker.run_check( standalone_gateway_base_source() diff --git a/testing/test_gateway.py b/testing/test_gateway.py index da519d4e..ae51cf0d 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -285,6 +285,19 @@ def test__rinfo(self, gw: Gateway) -> None: gw._cache_rinfo = rinfo gw.remote_exec("import os ; os.chdir(%r)" % old).waitclose() + def test__rinfo_while_exec_busy(self, gw: Gateway) -> None: + # info is a native protocol request: it must work (and not claim + # an exec slot) while an exec occupies the worker -- under + # main_thread_only an info-by-remote_exec used to either steal + # the main thread or trip the concurrency deadlock guard. + channel = gw.remote_exec("channel.send(channel.receive())") + try: + rinfo = gw._rinfo(update=True) + assert rinfo.pid != os.getpid() + finally: + channel.send("done") + assert channel.receive(TESTTIMEOUT) == "done" + class TestPopenGateway: gwtype = "popen" From cd2c8717fa481dd60a41e62da1439783558fa95b Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 16:55:17 +0200 Subject: [PATCH 44/91] feat: pure-async worker profile (execmodel=trio) C3 of the profile plan: the whole worker is one trio run owning the process main thread -- no host thread, no sync bridge, no sync channels. AsyncGateway gains a pluggable CHANNEL_EXEC handler (and live numexecuting for STATUS); TaskExec runs each source as a task on the worker nursery with an AsyncChannel. Sources must be async: strings execute with top-level await (PyCF_ALLOW_TOP_LEVEL_AWAIT), functions must be async def -- sync sources are rejected with a RemoteError before they can starve the loop. Gateway termination cancels running exec tasks. execmodel values are validated coordinator-side in makegateway against the profile registry (EXECMODEL_PROFILES); the worker CLI branches to the async entry (stdio and inherited-socket transports). Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_gateway.py | 11 ++- src/execnet/_trio_worker.py | 139 +++++++++++++++++++++++++++++++-- src/execnet/gateway_base.py | 10 ++- src/execnet/multi.py | 6 ++ testing/test_execmodel_trio.py | 124 +++++++++++++++++++++++++++++ 5 files changed, 283 insertions(+), 7 deletions(-) create mode 100644 testing/test_execmodel_trio.py diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 66a892e8..81ea76d0 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -461,6 +461,12 @@ class AsyncGateway: _error: BaseException | None = None #: where the peer lives, when the transport knows (ssh host, socket addr) remoteaddress: str | None = None + #: CHANNEL_EXEC handler ``(gateway, channelid, data) -> None`` -- set by + #: an exec strategy (the pure-async worker's TaskExec); without one, + #: exec requests are rejected. + _exec_handler: Callable[[AsyncGateway, int, bytes], None] | None = None + #: the exec strategy (for STATUS numexecuting), when serving as a worker + _task_exec: Any = None def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None: self._stream = stream @@ -671,10 +677,13 @@ def _dispatch(self, message: Message) -> None: self._channel_for(channelid)._close_from_remote(None, sendonly=True) elif code == Message.GATEWAY_TERMINATE: raise GatewayReceivedTerminate(self) + elif code == Message.CHANNEL_EXEC and self._exec_handler is not None: + self._exec_handler(self, channelid, message.data) elif code == Message.STATUS: + task_exec = self._task_exec status = { "numchannels": len(self._channels), - "numexecuting": 0, + "numexecuting": task_exec.active_count() if task_exec else 0, "execmodel": "trio", } self._send_nowait(Message.CHANNEL_DATA, channelid, dumps_internal(status)) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 56e64ef3..49e53a69 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -15,6 +15,7 @@ from .gateway_base import MAIN_THREAD_ONLY_DEADLOCK_TEXT from .gateway_base import WorkerGateway from .gateway_base import get_execmodel +from .gateway_base import geterrortext from .gateway_base import loads_internal from .gateway_base import trace from .portal import Mailbox @@ -106,16 +107,93 @@ def trigger_shutdown(self) -> None: self._primary.put(None) -# execmodel profile -> exec strategy. Placement strategies to come: -# "trio" (tasks on a main-thread loop), classic hybrid for "thread" -# (primary main + pool overflow), "gevent" (greenlets on a main-thread -# hub), and eventually subinterpreters. +# execmodel profile -> exec strategy for the sync-facade worker; the +# "trio" profile serves a plain AsyncGateway instead (TaskExec below). +# Placement strategies to come: classic hybrid for "thread" (primary +# main + pool overflow), "gevent" (greenlets on a main-thread hub), and +# eventually subinterpreters. WORKER_EXEC_STRATEGIES: dict[str, Callable[[WorkerGateway], Any]] = { "thread": PoolExec, "main_thread_only": MainExec, } +class TaskExec: + """Exec strategy for the pure-async profile (``execmodel=trio``). + + Sources run as tasks on the worker's own loop, in the one and only + thread of the process, and receive an ``AsyncChannel``. Sources must + be async: a plain function or a source string without top-level + ``await`` is rejected before it can starve the loop. Gateway + termination cancels running exec tasks (``trio.Cancelled`` inside the + source). + """ + + def __init__(self, gateway: Any, nursery: trio.Nursery) -> None: + self.gateway = gateway + self.nursery = nursery + self._running = 0 + + def active_count(self) -> int: + return self._running + + def handle_exec(self, gateway: Any, channelid: int, data: bytes) -> None: + """CHANNEL_EXEC hook; runs inline on the dispatch task.""" + item = loads_internal(data) + assert isinstance(item, tuple) + channel = gateway.open_channel(channelid) + self.nursery.start_soon(self._run_exec, channel, item) + + async def _run_exec(self, channel: Any, item: ExecItem) -> None: + import ast + import inspect + + source, file_name, call_name, kwargs = item + self._running += 1 + try: + trace(f"async execution starts[{channel.id}]: {source[:50]!r}") + loc: dict[str, Any] = {"channel": channel, "__name__": "__channelexec__"} + co = compile( + source + "\n", + file_name or "", + "exec", + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, + ) + toplevel_await = bool(co.co_flags & inspect.CO_COROUTINE) + if not toplevel_await and not call_name: + await channel.aclose( + "sync source under execmodel=trio: the source must use" + " top-level await (or pass an async function)" + ) + return + if toplevel_await: + await eval(co, loc) + else: + exec(co, loc) # define the function (no top-level awaits) + if call_name: + function = loc[call_name] + result = function(channel, **kwargs) + if not hasattr(result, "__await__"): + await channel.aclose( + f"sync function {call_name!r} under execmodel=trio:" + " remote_exec functions must be async" + ) + return + await result + except trio.Cancelled: + raise + except EOFError: + trace("ignoring EOFError from async exec") + except BaseException as exc: + trace(f"async exec got exception: {exc!r}") + await channel.aclose(geterrortext(exc)) + return + finally: + self._running -= 1 + trace("async execution finished") + await channel.aclose() + + class TrioWorkerExec: """FIFO admission pump feeding an exec placement strategy. @@ -326,6 +404,51 @@ async def _make_fd_io(read_fd: int, write_fd: int) -> Any: return _trio_gateway.staple_fd_stream(read_fd, write_fd) +async def _serve_async_worker(stream: Any, id: str) -> None: + """Serve a plain AsyncGateway with task-based exec (execmodel=trio). + + The whole worker is this one trio run on the process main thread: the + dispatch loop and every exec'd source share it. Termination cancels + running exec tasks. + """ + from ._trio_gateway import AsyncGateway + + gateway = AsyncGateway(stream, id=id, _startcount=2) + async with trio.open_nursery() as nursery: + task_exec = TaskExec(gateway, nursery) + gateway._exec_handler = task_exec.handle_exec + gateway._task_exec = task_exec + await nursery.start(gateway._serve) + await gateway.wait_closed() + nursery.cancel_scope.cancel() + + +def serve_popen_async(id: str) -> None: + """Serve the pure-async worker over the stdio pipes (execmodel=trio).""" + from ._trio_gateway import staple_fd_stream + + read_fd, write_fd = _prepare_protocol_fds() + os.write(write_fd, b"1") + + async def main() -> None: + await _serve_async_worker(staple_fd_stream(read_fd, write_fd), id) + + trio.run(main) + os._exit(0) + + +def serve_socket_async(id: str, socket_fd: int) -> None: + """Serve the pure-async worker over an inherited socket fd.""" + from . import _trio_host + + async def main() -> None: + stream = await _trio_host.adopt_socket(socket_fd) + await _serve_async_worker(stream, id) + + trio.run(main) + os._exit(0) + + def serve_popen_trio(id: str, execmodel: str = "thread", wait: str = "thread") -> None: """Serve a WorkerGateway over the stdio pipes (popen / ssh worker).""" from . import _trio_host @@ -436,7 +559,13 @@ def _main() -> None: config = json.loads(ns.config) _check_version(config["coordinator_version"]) _apply_worker_setup(config) - if ns.socket_fd is not None: + if config["execmodel"] == "trio": + # pure-async profile: one thread, the loop owns the main thread + if ns.socket_fd is not None: + serve_socket_async(config["id"], ns.socket_fd) + else: + serve_popen_async(config["id"]) + elif ns.socket_fd is not None: serve_socket_trio( config["id"], config["execmodel"], diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index c0ac5252..77fd81df 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -123,10 +123,18 @@ def Event(self) -> threading.Event: return threading.Event() +#: worker profiles: where exec'd code runs relative to the protocol loop +EXECMODEL_PROFILES = ( + "thread", # exec on pool threads, loop on a side thread + "main_thread_only", # exec serialized on the main thread (GUI/pytest) + "trio", # pure async: loop owns the main thread, async sources as tasks +) + + def get_execmodel(backend: str | ExecModel) -> ExecModel: if isinstance(backend, ExecModel): return backend - if backend in ("thread", "main_thread_only"): + if backend in EXECMODEL_PROFILES: return ExecModel(backend) raise ValueError(f"unknown execmodel {backend!r}") diff --git a/src/execnet/multi.py b/src/execnet/multi.py index ac88b181..87b17786 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -24,6 +24,7 @@ from typing import overload from ._boundary import wakener_names +from .gateway_base import EXECMODEL_PROFILES from .gateway_base import Channel from .gateway_base import ExecModel from .gateway_base import get_execmodel @@ -172,6 +173,11 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: self.allocate_id(spec) if spec.execmodel is None: spec.execmodel = self.remote_execmodel.backend + elif spec.execmodel not in EXECMODEL_PROFILES: + raise ValueError( + f"unknown execmodel {spec.execmodel!r}" + f" (known profiles: {list(EXECMODEL_PROFILES)})" + ) if spec.wait is not None and spec.wait not in wakener_names(): raise ValueError( f"unknown wait backend {spec.wait!r} (known: {wakener_names()})" diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py new file mode 100644 index 00000000..d23a72cf --- /dev/null +++ b/testing/test_execmodel_trio.py @@ -0,0 +1,124 @@ +"""The pure-async worker profile (execmodel=trio). + +One single thread in the worker: the trio loop owns the main thread and +exec'd sources run as tasks on it, talking through AsyncChannels. Sync +sources are rejected before they can starve the loop. +""" + +from __future__ import annotations + +from collections.abc import Callable + +import pytest +import trio as trio_lib + +import execnet +import execnet.trio +from execnet.gateway import Gateway + +TESTTIMEOUT = 10.0 + + +@pytest.fixture +def trio_gw(makegateway: Callable[[str], Gateway]) -> Gateway: + return makegateway("popen//execmodel=trio") + + +class TestSyncCoordinator: + def test_top_level_await_roundtrip(self, trio_gw: Gateway) -> None: + channel = trio_gw.remote_exec( + "await channel.send(await channel.receive() + 1)" + ) + channel.send(41) + assert channel.receive(TESTTIMEOUT) == 42 + + def test_async_function_source(self, trio_gw: Gateway) -> None: + async def source(channel, delta) -> None: + value = await channel.receive() + await channel.send(value + delta) + + channel = trio_gw.remote_exec(source, delta=5) + channel.send(37) + assert channel.receive(TESTTIMEOUT) == 42 + + def test_iteration_and_eof(self, trio_gw: Gateway) -> None: + channel = trio_gw.remote_exec( + """ + for i in range(3): + await channel.send(i * 2) + """ + ) + assert list(channel) == [0, 2, 4] + + def test_single_thread_on_main(self, trio_gw: Gateway) -> None: + channel = trio_gw.remote_exec( + """ + import threading + await channel.send( + ( + threading.active_count(), + threading.current_thread() is threading.main_thread(), + ) + ) + """ + ) + active, on_main = channel.receive(TESTTIMEOUT) + assert on_main + assert active == 1 + + def test_concurrent_execs_cooperate(self, trio_gw: Gateway) -> None: + # two execs run as tasks on one loop: the first parks in receive + # while the second completes -- no threads involved. + blocked = trio_gw.remote_exec( + "await channel.send(await channel.receive())" + ) + side = trio_gw.remote_exec("await channel.send('side')") + assert side.receive(TESTTIMEOUT) == "side" + blocked.send("go") + assert blocked.receive(TESTTIMEOUT) == "go" + + def test_sync_source_rejected(self, trio_gw: Gateway) -> None: + channel = trio_gw.remote_exec("x = 40 + 2") + with pytest.raises(channel.RemoteError, match="sync source"): + channel.receive(TESTTIMEOUT) + + def test_sync_function_rejected(self, trio_gw: Gateway) -> None: + def source(channel) -> None: + pass + + channel = trio_gw.remote_exec(source) + with pytest.raises(channel.RemoteError, match="must be async"): + channel.receive(TESTTIMEOUT) + + def test_remote_error_traceback(self, trio_gw: Gateway) -> None: + async def source(channel) -> None: + raise ValueError(17) + + channel = trio_gw.remote_exec(source) + with pytest.raises(channel.RemoteError, match="ValueError"): + channel.receive(TESTTIMEOUT) + + def test_status_and_rinfo(self, trio_gw: Gateway) -> None: + assert trio_gw.remote_status().execmodel == "trio" + rinfo = trio_gw._rinfo() + assert rinfo.pid + assert rinfo.version_info + + +def test_unknown_execmodel_rejected(makegateway: Callable[[str], Gateway]) -> None: + with pytest.raises(ValueError, match="unknown execmodel"): + makegateway("popen//execmodel=nope") + + +def test_trio_native_coordinator() -> None: + async def main() -> None: + async with execnet.trio.AsyncGroup() as group: + gateway = await group.makegateway("popen//execmodel=trio") + channel = await gateway.remote_exec( + "await channel.send(await channel.receive() * 2)" + ) + await channel.send(21) + with trio_lib.fail_after(TESTTIMEOUT): + assert await channel.receive() == 42 + + trio_lib.run(main) From 36e904e52dd22e0a8aea0e3f8761107d0363cc76 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 17:18:34 +0200 Subject: [PATCH 45/91] feat!: restore classic hybrid exec for execmodel=thread C4 of the profile plan: the thread profile regains its pre-port shape -- a request arriving while the worker main thread is idle claims it (the primary workload gets a true main thread again), and requests arriving while it is busy overflow to pool threads instead of queueing. The claim is decided during FIFO admission, so the first exec always lands on the main thread. HybridExec composes MainExec (primary mailbox + integrate loop) with PoolExec overflow; main-thread workers now integrate as primary instead of parking in join(). Together with C2 (native info/setup) this cannot be starved by coordinator bookkeeping: only real remote_exec work claims the slot. xdist is unaffected (it forces main_thread_only); the -n 12 suite runs are the live canary. Co-Authored-By: Claude Fable 5 --- src/execnet/_trio_worker.py | 45 +++++++++++++++++++++++++++++++--- testing/test_execmodel_trio.py | 8 ++---- testing/test_gateway.py | 32 ++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 49e53a69..c6042bb8 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -107,13 +107,50 @@ def trigger_shutdown(self) -> None: self._primary.put(None) +class HybridExec(MainExec): + """Exec strategy: primary on the main thread, overflow on pool threads. + + The classic ``thread`` execmodel shape: a request arriving while the + main thread is idle claims it (pytest and friends get a true main + thread); requests arriving while it is busy run on worker threads + instead of queueing. The claim is decided during FIFO admission so + the *first* request always gets the main thread. + """ + + def __init__(self, gateway: WorkerGateway) -> None: + super().__init__(gateway) + self._pool = PoolExec(gateway) + self._claim_lock = threading.Lock() + self._primary_busy = False + self._claimed: set[int] = set() + + async def admit(self, channel: Channel, item: ExecItem) -> bool: + with self._claim_lock: + if not self._primary_busy: + self._primary_busy = True + self._claimed.add(channel.id) + return True + + async def run(self, channel: Channel, item: ExecItem) -> None: + with self._claim_lock: + claimed = channel.id in self._claimed + self._claimed.discard(channel.id) + if not claimed: + await self._pool.run(channel, item) + return + try: + await super().run(channel, item) + finally: + with self._claim_lock: + self._primary_busy = False + + # execmodel profile -> exec strategy for the sync-facade worker; the # "trio" profile serves a plain AsyncGateway instead (TaskExec below). -# Placement strategies to come: classic hybrid for "thread" (primary -# main + pool overflow), "gevent" (greenlets on a main-thread hub), and -# eventually subinterpreters. +# Placement strategies to come: "gevent" (greenlets on a main-thread +# hub), and eventually subinterpreters. WORKER_EXEC_STRATEGIES: dict[str, Callable[[WorkerGateway], Any]] = { - "thread": PoolExec, + "thread": HybridExec, "main_thread_only": MainExec, } diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py index d23a72cf..84585ba9 100644 --- a/testing/test_execmodel_trio.py +++ b/testing/test_execmodel_trio.py @@ -26,9 +26,7 @@ def trio_gw(makegateway: Callable[[str], Gateway]) -> Gateway: class TestSyncCoordinator: def test_top_level_await_roundtrip(self, trio_gw: Gateway) -> None: - channel = trio_gw.remote_exec( - "await channel.send(await channel.receive() + 1)" - ) + channel = trio_gw.remote_exec("await channel.send(await channel.receive() + 1)") channel.send(41) assert channel.receive(TESTTIMEOUT) == 42 @@ -69,9 +67,7 @@ def test_single_thread_on_main(self, trio_gw: Gateway) -> None: def test_concurrent_execs_cooperate(self, trio_gw: Gateway) -> None: # two execs run as tasks on one loop: the first parks in receive # while the second completes -- no threads involved. - blocked = trio_gw.remote_exec( - "await channel.send(await channel.receive())" - ) + blocked = trio_gw.remote_exec("await channel.send(await channel.receive())") side = trio_gw.remote_exec("await channel.send('side')") assert side.receive(TESTTIMEOUT) == "side" blocked.send("go") diff --git a/testing/test_gateway.py b/testing/test_gateway.py index ae51cf0d..6b6d5cf5 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -9,6 +9,7 @@ import shutil import signal import sys +import time from collections.abc import Callable from textwrap import dedent @@ -285,6 +286,37 @@ def test__rinfo(self, gw: Gateway) -> None: gw._cache_rinfo = rinfo gw.remote_exec("import os ; os.chdir(%r)" % old).waitclose() + def test_hybrid_primary_then_overflow( + self, makegateway: Callable[[str], Gateway] + ) -> None: + # classic thread-model shape: the first exec claims the true main + # thread; while it is busy, further execs overflow to worker + # threads; once released the main thread is claimable again. + gw = makegateway("popen//execmodel=thread") + report = """ + import threading + channel.send(threading.current_thread() is threading.main_thread()) + channel.receive() + """ + first = gw.remote_exec(report) + assert first.receive(TESTTIMEOUT) is True + second = gw.remote_exec(report) + assert second.receive(TESTTIMEOUT) is False + second.send(None) + second.waitclose(TESTTIMEOUT) + first.send(None) + first.waitclose(TESTTIMEOUT) + # the primary slot frees shortly after the exec finishes + for _ in range(50): + probe = gw.remote_exec(report) + on_main = probe.receive(TESTTIMEOUT) + probe.send(None) + probe.waitclose(TESTTIMEOUT) + if on_main: + break + time.sleep(0.05) + assert on_main + def test__rinfo_while_exec_busy(self, gw: Gateway) -> None: # info is a native protocol request: it must work (and not claim # an exec slot) while an exec occupies the worker -- under From cbce1830630cd02388992e8d9773b0639693b8ae Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 19:21:54 +0200 Subject: [PATCH 46/91] feat: gevent worker profile and gevent-safe management ops C5 of the profile plan, completing the gevent/loop integration: - execmodel=gevent: GreenletExec runs the hub on the worker main thread and spawns a greenlet per remote_exec; the worker's wait backend defaults to gevent so channel operations park greenlets, letting concurrent execs cooperate on one thread. The integrate loop itself waits on a gevent wakener (a thread-blocking get would stall the hub). - Provisioning: execnet grows a [gevent] extra; every uv launcher (popen python=, ssh, via sub-spawn) derives an additional `--with gevent` from the worker config when the profile or wait backend needs it. - Coordinator side: TrioHost.call_pending resolves a OneShot from a host task; makegateway (wait!=thread), SyncIOHandle.wait/kill, and Group.terminate wait on it with the gateway's wakener so a gevent app's management ops park only the calling greenlet. wait=thread keeps the KI-deferred portal.run path unchanged. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 3 ++ src/execnet/_provision.py | 23 +++++++++-- src/execnet/_trio_host.py | 63 +++++++++++++++++++++++++++-- src/execnet/_trio_worker.py | 54 ++++++++++++++++++++++++- src/execnet/gateway_base.py | 3 +- src/execnet/multi.py | 24 ++++++++++- testing/test_gevent.py | 81 +++++++++++++++++++++++++++++++++++++ uv.lock | 6 ++- 8 files changed, 246 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index aedd6b59..6e12c8da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,9 @@ classifiers = [ execnet-socketserver = "execnet.script.socketserver:main" [project.optional-dependencies] +gevent = [ + "gevent", +] testing = [ "pre-commit", "pytest>8.0", diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index ffe9ffaa..4da4f689 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -164,10 +164,12 @@ def worker_cli_arg(spec: Any) -> str: """ import execnet + # the gevent profile parks its greenlets on gevent wakeners + default_wait = "gevent" if spec.execmodel == "gevent" else "thread" config: dict[str, Any] = { "id": f"{spec.id}-worker", "execmodel": spec.execmodel, - "wait": spec.wait or "thread", + "wait": spec.wait or default_wait, "coordinator_version": execnet.__version__, } # Startup setup applied by the worker before serving (never through @@ -203,13 +205,28 @@ def _uv_tokens(python: str | None) -> list[str]: return prefix +def _extra_with_tokens(config: str) -> list[str]: + """Additional ``--with`` requirements the worker env needs. + + Derived from the worker config itself so every uv launcher (popen, + ssh, via sub-spawn) provisions the same: the gevent profile / wait + backend needs gevent importable in the worker. + """ + parsed = json.loads(config) + if parsed.get("execmodel") == "gevent" or parsed.get("wait") == "gevent": + return ["--with", "gevent"] + return [] + + def uv_worker_argv(spec: Any) -> list[str]: """``uv run`` argv to launch the Trio worker locally (wheel path is local).""" + config = worker_cli_arg(spec) return [ *_uv_tokens(spec.python), "--with", coordinator_requirement(), - *worker_module_tokens(spec), + *_extra_with_tokens(config), + *_worker_tokens(config), ] @@ -228,7 +245,7 @@ def _remote_shell_command( wheel bytes are returned as the preamble to stream before the protocol. """ worker = _worker_tokens(config) - uv = _uv_tokens(python) + uv = [*_uv_tokens(python), *_extra_with_tokens(config)] if wheel is None: assert requirement is not None return shlex.join([*uv, "--with", requirement, *worker]), b"" diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 9a1f1338..597cbdb4 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -111,7 +111,7 @@ async def _wait() -> int | None: return code try: - return self._session.host.call(_wait) + return self._session.host_call(_wait) except Exception: return process.returncode @@ -126,7 +126,7 @@ async def _kill() -> None: await process.wait() try: - self._session.host.call(_kill) + self._session.host_call(_kill) except Exception as exc: trace("ERROR killing trio process:", exc) @@ -293,6 +293,21 @@ def release_channel(self, channelid: int) -> None: with suppress(trio.RunFinishedError): self.host.portal.post(self._forget_channel, channelid) + def host_call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: + """Blocking host call that parks correctly for the wait backend. + + ``wait=thread`` keeps the KI-deferred ``portal.run`` path; other + backends (gevent) wait on a OneShot with the gateway's wakener so + only the calling greenlet parks, not the whole hub. + """ + gateway = self.sync_gateway + if gateway._wait_backend == "thread": + return self.host.call(async_fn, *args) + pending = self.host.call_pending( + async_fn, *args, wakener=gateway._new_wakener() + ) + return pending.wait() + def run_on_loop(self, sync_fn: Callable[[], T]) -> T: """Run ``sync_fn`` on the host loop, excluding dispatch interleaving. @@ -428,6 +443,39 @@ async def _main(self) -> None: def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: return self.portal.run(async_fn, *args) + def call_pending( + self, + async_fn: Callable[..., Awaitable[T]], + *args: Any, + wakener: Any = None, + ) -> OneShot[T]: + """Run ``async_fn`` as a host task, resolving a :class:`OneShot`. + + The non-blocking counterpart of :meth:`call` for consumers that + must not block their OS thread (a gevent hub: waiting on the + OneShot with a gevent wakener parks only the calling greenlet). + Unlike ``portal.run`` the wait is KeyboardInterrupt-interruptible. + """ + result: OneShot[T] = OneShot(wakener) + + async def runner() -> None: + try: + value = await async_fn(*args) + except trio.Cancelled: + if not result.is_set(): + result.set_error(RuntimeError("trio host was shut down")) + raise + except BaseException as exc: + result.set_error(exc) + else: + result.set(value) + + def spawn() -> None: + self.start_soon(runner) + + self.portal.post(spawn) + return result + def call_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: return self.portal.run_sync(sync_fn, *args) @@ -556,7 +604,16 @@ def makegateway_trio(group: Group, spec: Any) -> Gateway: """Create a sync-facade Gateway for ``spec`` on the group's Trio host.""" host: TrioHost = group._ensure_trio_host() async_group: FacadeAsyncGroup = group._ensure_async_group() - bridge = host.call(async_group.makegateway, spec) + if spec.wait and spec.wait != "thread": + # e.g. a gevent app: wait on a OneShot so only the calling + # greenlet parks while the gateway comes up, not the whole hub. + from ._boundary import make_wakener + + bridge = host.call_pending( + async_group.makegateway, spec, wakener=make_wakener(spec.wait) + ).wait() + else: + bridge = host.call(async_group.makegateway, spec) assert isinstance(bridge, SyncBridgeGateway) gw: Gateway = bridge.sync_gateway # type: ignore[assignment] gw._io = SyncIOHandle( diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index c6042bb8..d8c6cc0e 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -145,13 +145,63 @@ async def run(self, channel: Channel, item: ExecItem) -> None: self._primary_busy = False +class GreenletExec: + """Exec strategy: greenlets on a gevent hub owning the main thread. + + ``execmodel=gevent``: each request runs as a greenlet spawned by the + integrate loop; the worker's ``wait=gevent`` wakeners make channel + operations park the greenlet, so concurrent remote_execs cooperate on + the one main thread. Requires gevent in the worker environment + (provisioning adds the ``gevent`` requirement automatically). + """ + + needs_primary_thread = True + + def __init__(self, gateway: WorkerGateway) -> None: + from ._boundary import make_wakener + + self.gateway = gateway + # The integrate loop blocks in get() on the hub thread: a gevent + # wakener parks only its root greenlet, letting exec greenlets run. + self._primary: Mailbox[tuple[Channel, ExecItem, threading.Event] | None] = ( + Mailbox(make_wakener("gevent")) + ) + + async def admit(self, channel: Channel, item: ExecItem) -> bool: + return True + + async def run(self, channel: Channel, item: ExecItem) -> None: + done = threading.Event() + self._primary.put((channel, item, done)) + await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) + + def integrate_as_primary_thread(self) -> None: + """Run the hub on the main thread, spawning a greenlet per exec.""" + import gevent + + def run_exec(channel: Channel, item: ExecItem, done: threading.Event) -> None: + try: + self.gateway.executetask((channel, item)) + finally: + done.set() + + while True: + task = self._primary.get() + if task is None: + break + gevent.spawn(run_exec, *task) + + def trigger_shutdown(self) -> None: + self._primary.put(None) + + # execmodel profile -> exec strategy for the sync-facade worker; the # "trio" profile serves a plain AsyncGateway instead (TaskExec below). -# Placement strategies to come: "gevent" (greenlets on a main-thread -# hub), and eventually subinterpreters. +# Future placement strategies slot in here (e.g. subinterpreters). WORKER_EXEC_STRATEGIES: dict[str, Callable[[WorkerGateway], Any]] = { "thread": HybridExec, "main_thread_only": MainExec, + "gevent": GreenletExec, } diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 77fd81df..2b183c1e 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -125,9 +125,10 @@ def Event(self) -> threading.Event: #: worker profiles: where exec'd code runs relative to the protocol loop EXECMODEL_PROFILES = ( - "thread", # exec on pool threads, loop on a side thread + "thread", # hybrid: primary on the main thread, overflow on pool threads "main_thread_only", # exec serialized on the main thread (GUI/pytest) "trio", # pure async: loop owns the main thread, async sources as tasks + "gevent", # greenlets on a main-thread hub, one per remote_exec ) diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 87b17786..7c0810a7 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -248,13 +248,35 @@ def terminate(self, timeout: float | None = None) -> None: # each with a GATEWAY_TERMINATE + timeout grace, then kill; # bounded at roughly twice the timeout (issues #43 / #221). try: - self._trio_host.call(self._async_group.terminate, timeout) + self._host_terminate(timeout) except Exception as exc: trace("group terminate error:", exc) for gw in self._gateways_to_join: gw.join() self._gateways_to_join[:] = [] + def _host_terminate(self, timeout: float | None) -> None: + """Terminate the async group, parking correctly for wait backends. + + A member gateway created with a non-thread ``wait=`` implies the + caller may be a greenlet: wait on a OneShot instead of blocking + the OS thread (which would stall the hub for the whole grace). + """ + backends = {gw._wait_backend for gw in self._gateways_to_join} | { + gw._wait_backend for gw in self + } + backends.discard("thread") + if backends: + from ._boundary import make_wakener + + self._trio_host.call_pending( + self._async_group.terminate, + timeout, + wakener=make_wakener(backends.pop()), + ).wait() + else: + self._trio_host.call(self._async_group.terminate, timeout) + def remote_exec( self, source: str | types.FunctionType | Callable[..., object] | types.ModuleType, diff --git a/testing/test_gevent.py b/testing/test_gevent.py index e95edfc7..91f74ef4 100644 --- a/testing/test_gevent.py +++ b/testing/test_gevent.py @@ -84,3 +84,84 @@ def test_waitclose_and_endmarker(self, gevent_gw: execnet.Gateway) -> None: channel = gevent_gw.remote_exec("channel.send(1)") assert gevent.spawn(channel.receive, TESTTIMEOUT).get(timeout=TESTTIMEOUT) == 1 gevent.spawn(channel.waitclose, TESTTIMEOUT).get(timeout=TESTTIMEOUT) + + def test_makegateway_parks_greenlet_not_hub(self) -> None: + # management ops (makegateway/terminate) from a greenlet must not + # stall the hub: they wait on a OneShot with a gevent wakener. + group = execnet.Group() + progressed: list[int] = [] + + def other() -> None: + for i in range(5): + progressed.append(i) + gevent.sleep(0.01) + + try: + ticker = gevent.spawn(other) + maker = gevent.spawn(group.makegateway, "popen//wait=gevent") + gw = maker.get(timeout=TESTTIMEOUT) + channel = gw.remote_exec("channel.send(42)") + assert gevent.spawn(channel.receive, TESTTIMEOUT).get(TESTTIMEOUT) == 42 + ticker.get(timeout=TESTTIMEOUT) + assert progressed == [0, 1, 2, 3, 4] + finally: + gevent.spawn(group.terminate, 5.0).get(timeout=TESTTIMEOUT) + + +class TestGeventWorkerProfile: + """execmodel=gevent: exec'd code runs as greenlets on the main-thread hub.""" + + @pytest.fixture + def worker_gw(self): + group = execnet.Group() + try: + yield group.makegateway("popen//execmodel=gevent") + finally: + group.terminate(timeout=5.0) + + def test_execs_are_greenlets_on_main_thread(self, worker_gw) -> None: + report = """ + import threading + channel.send(threading.current_thread() is threading.main_thread()) + channel.receive() + """ + first = worker_gw.remote_exec(report) + second = worker_gw.remote_exec(report) + # both run concurrently on the one main thread: greenlets + assert first.receive(TESTTIMEOUT) is True + assert second.receive(TESTTIMEOUT) is True + first.send(None) + second.send(None) + first.waitclose(TESTTIMEOUT) + second.waitclose(TESTTIMEOUT) + + def test_execs_cooperate_via_gevent(self, worker_gw) -> None: + # the first exec parks in channel.receive() (gevent wakener) while + # the second completes -- with a blocked hub this would deadlock. + blocked = worker_gw.remote_exec("channel.send(channel.receive())") + side = worker_gw.remote_exec( + """ + import gevent + gevent.sleep(0.01) + channel.send('side') + """ + ) + assert side.receive(TESTTIMEOUT) == "side" + blocked.send("go") + assert blocked.receive(TESTTIMEOUT) == "go" + + def test_status_reports_gevent(self, worker_gw) -> None: + assert worker_gw.remote_status().execmodel == "gevent" + + +def test_provisioning_adds_gevent_requirement() -> None: + from execnet._provision import _extra_with_tokens + from execnet._provision import worker_cli_arg + from execnet.xspec import XSpec + + spec = XSpec("popen//id=g1//execmodel=gevent") + config = worker_cli_arg(spec) + assert '"wait": "gevent"' in config + assert _extra_with_tokens(config) == ["--with", "gevent"] + plain = worker_cli_arg(XSpec("popen//id=g2//execmodel=thread")) + assert _extra_with_tokens(plain) == [] diff --git a/uv.lock b/uv.lock index b885a4b7..3b8bb46e 100644 --- a/uv.lock +++ b/uv.lock @@ -387,6 +387,9 @@ dependencies = [ ] [package.optional-dependencies] +gevent = [ + { name = "gevent" }, +] testing = [ { name = "asyncssh" }, { name = "hatch" }, @@ -411,6 +414,7 @@ testing = [ [package.metadata] requires-dist = [ { name = "asyncssh", marker = "extra == 'testing'" }, + { name = "gevent", marker = "extra == 'gevent'" }, { name = "hatch", marker = "extra == 'testing'" }, { name = "pre-commit", marker = "extra == 'testing'" }, { name = "pytest", marker = "extra == 'testing'", specifier = ">8.0" }, @@ -419,7 +423,7 @@ requires-dist = [ { name = "trio", specifier = ">=0.32" }, { name = "uv", marker = "extra == 'testing'" }, ] -provides-extras = ["testing"] +provides-extras = ["gevent", "testing"] [package.metadata.requires-dev] dev = [{ name = "pytest-xdist", specifier = ">=3.8.0" }] From 87c58430e31f7c849db7cef2fbb35d788a6324be Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sun, 26 Jul 2026 19:35:25 +0200 Subject: [PATCH 47/91] docs: record the landed Phase C worker profiles in the handoff Co-Authored-By: Claude Fable 5 --- handoff-phase-c-worker-axes.md | 74 ++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index 40c222b6..01382e39 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -71,44 +71,48 @@ File map (src/execnet/): `threading.Event`/`queue.Queue` so KeyboardInterrupt can interrupt them; `portal.run` (KI-deferred) is only for management ops. -## Phase C — DECIDED PLAN (Ronny, 2026-07-26, after the boundary rethink) +## Phase C — LANDED: use-case worker profiles on `execmodel=` (2026-07-26) -Predates below gap analysis; where they conflict, this section wins. -Context: the boundary rethink (`handoff-boundary-protocol-rethink.md`) -landed first — `wait=` axis exists, ExecModel is a preset, the sync -Channel is a facade over the async core. +Where this conflicts with anything below or elsewhere, this section +wins. The axes framing (`loop=`/`exec=` as spec keys, a reserved +`backend=` axis) was **dropped** in the final rethink with Ronny: +different use-cases get named profiles, `execmodel=` is the public mode +key (xdist already passes it), and `wait=` is the only other public +knob. Implemented in commits `b2f43c3..cbce183`: -**Worker matrix** (`loop=` = main-thread role, `exec=` = placement). -The main thread is the scarce resource (signals, interrupt_main, GUI, -tests calling asyncio.run/trio.run themselves): - -| loop= | exec= | main thread | exec'd code | channel | consumer | +| `execmodel=` | loop thread | exec'd code runs | channel | worker wait | extra deps | |---|---|---|---|---|---| -| main | thread | host loop | worker threads | sync | default preset for execmodel=thread; no parked thread | -| main | task | host loop | tasks on the loop | AsyncChannel | async-native sources (C4) | -| thread | main | exec'd code (serialized) | true main thread | sync | pytest/xdist (forces main_thread_only today), GUI, signals | -| thread | thread | parked | worker threads | sync | valid, non-default (today's shape) | -| thread | task | parked | tasks on side loop | AsyncChannel | valid, niche | -| main | main | — | — | — | invalid | - -Interaction rule: exec'd code may start its own event loop only when it -does not share a thread with ours (exec=thread / exec=main); exec=task -sources are ``async def`` and join our loop — a sync source under -exec=task is a hard error. ``wait=`` composes orthogonally. - -**Backend decisions**: the core STAYS trio-only and trio stays the -default and a hard dependency (pure-python wheels; deployment cost -accepted). The anyio/asyncio-core port ("C0") is REJECTED for now; -asyncio apps use the ``execnet.aio`` bridge. ``backend=`` still lands -as a per-gateway spec axis, but reserved: only ``trio`` validates -(exactly how ``wait=`` landed with only ``thread``), keeping an -asyncio-core door open without paying for it. Consequence: exec=task -sources are trio-typed. - -Sequencing: C1 axes (`backend=`/`loop=`/`exec=` + validity matrix + -presets) → C2 `loop=main` worker restructure → C3 `exec=main` re-home → -C4 `exec=task`. Presets: `execmodel=thread` → `loop=main, exec=thread`; -`execmodel=main_thread_only` → `loop=thread, exec=main`. +| `thread` (default) | side thread | **classic hybrid restored**: primary on the main thread, overflow on pool threads (claim decided during FIFO admission) | sync | thread | — | +| `main_thread_only` | side thread | main thread, serialized (deadlock guard) | sync | thread | — | +| `trio` (new) | **main thread** | async sources as tasks — one single thread total; top-level await or async def; sync sources rejected; termination cancels tasks | AsyncChannel | (loop) | — | +| `gevent` (revived) | side thread | greenlets on a main-thread hub, one per remote_exec | sync | gevent (derived) | `execnet[gevent]`, auto-added by uv provisioning | + +Architecture: `TrioWorkerExec` is a pure FIFO admission pump delegating +to strategy objects (`WORKER_EXEC_STRATEGIES` in `_trio_worker.py`: +PoolExec building block, MainExec, HybridExec, GreenletExec; TaskExec +serves a plain AsyncGateway via its pluggable `_exec_handler` — no sync +bridge at all in the trio profile). Subinterpreters: future strategy +slot, not built. `EXECMODEL_PROFILES` (gateway_base) validates +coordinator-side in makegateway. + +**Native info/setup** (the pytest fix): `Message.GATEWAY_INFO` (code 10) +answers `_rinfo()` from the dispatch loop; chdir/nice/env ship in the +worker config JSON and apply at startup. Coordinator bookkeeping can no +longer claim an exec slot (previously an info call could occupy the main +thread so pytest landed on a worker thread) — `rinfo_source` and the +post-start remote_exec setup block are gone. + +Coordinator-gevent integration: `TrioHost.call_pending` (OneShot from a +host task) backs makegateway / `SyncIOHandle.wait/kill` / +`Group.terminate` whenever the wait backend is not `thread`, so a gevent +app's management ops park only the calling greenlet. `wait=thread` +keeps the KI-deferred `portal.run` path. + +Still decided/standing: core stays trio-only (anyio/asyncio-core port +rejected; asyncio apps use `execnet.aio`); eventlet stays dead; exec'd +code may start its own loop in every profile except `trio`. + +## Phase C — original gap analysis (pre-decision record, superseded) ## Phase C — original gap analysis (pre-decision record) From 3031810da1bcf6d907d067565c6e9125a5438383 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 10:00:04 +0200 Subject: [PATCH 48/91] feat!: run channel callbacks in an off-loop consumer task setcallback no longer delivers inline on the loop/reader thread and no longer pins the channel in a ChannelFactory._callback_channels registry. Instead Channel.setcallback -> BaseGateway._start_channel_consumer -> SyncBridgeGateway.attach_consumer starts a per-channel consumer *task* on the host loop that drains the channel into an inbox and runs each callback in a trio.to_thread pool thread (bounded by TrioHost.callback_limiter). The task holds a strong reference to the channel, so a callback channel's lifecycle is bound to consumption (and to GC once the stream closes) rather than to a strong-ref registry. Delivery keeps flowing through the existing _sync_payload -> Channel._deliver_payload path (diverted to the inbox once a consumer is attached), so nothing rebinds the raw channel and there is no race with the bind posted at newchannel(). waitclose() now waits on a new Channel._consumer_done flag for callback channels, so it still returns only after every callback -- including the endmarker -- has run. A slow callback no longer blocks the reader, and callbacks for different channels run concurrently (per-channel order kept). Deleted: Channel._callback/_endmarker/_fire_endmarker, ChannelFactory. _callback_channels/_register_callback_channel, and the callback branch of _deliver_payload (now uniformly mailbox.put); __del__'s CHANNEL_LAST_MESSAGE case folds away (a callback channel is never GC'd while open). Resolves the standing "callbacks on the loop thread" open decision (moved). Co-Authored-By: Claude Opus 4.8 (1M context) --- handoff-boundary-protocol-rethink.md | 6 +- handoff-phase-c-worker-axes.md | 54 +++++++- src/execnet/_trio_host.py | 181 +++++++++++++++++++++++++-- src/execnet/gateway_base.py | 181 +++++++++++++++------------ testing/test_channel.py | 44 +++++++ 5 files changed, 369 insertions(+), 97 deletions(-) diff --git a/handoff-boundary-protocol-rethink.md b/handoff-boundary-protocol-rethink.md index 59cd1141..70146c7d 100644 --- a/handoff-boundary-protocol-rethink.md +++ b/handoff-boundary-protocol-rethink.md @@ -139,8 +139,10 @@ namespaces + the axes/presets. one portal FIFO. - After close: `OSError("cannot send (already closed?)")`. - exec admission order = message arrival order (TrioWorkerExec._pump). -- Channel callbacks run on the loop thread (revisit-later stands; the - kit makes moving them cheap). +- Channel callbacks run off the loop thread: a per-channel consumer task + drains into a threadpool call (moved 2026-07-26; the kit made it cheap, + as predicted). See `handoff-phase-c-worker-axes.md` "Callback consumer + tasks". - `Group.terminate(timeout)` never hangs (~2×timeout). - Sync blocking waits stay KeyboardInterrupt-interruptible on the main thread (thread Wakener keeps the Event.wait + drain pattern). diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index 01382e39..102cb78f 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -63,8 +63,12 @@ File map (src/execnet/): - exec admission order = message arrival order (`TrioWorkerExec._pump`; `test_main_thread_only_concurrent_remote_exec_deadlock` guards this — trio shuffles its run batch, so never rely on task-spawn order). -- Channel callbacks run on the receiver (loop) thread — keep for now - (open decision: revisit in D). +- Channel callbacks now run in a threadpool thread driven by a per-channel + consumer *task* on the loop (RESOLVED 2026-07-26, was the "revisit in D" + open decision — moved off the loop). A slow callback no longer blocks the + reader; per-channel order is still strict, and `waitclose()` still returns + only after every callback (incl. the endmarker) has run. See "Callback + consumer tasks" below. - `Group.terminate(timeout)` never hangs (~2×timeout bound, issues #43/#221). - Sync blocking waits (send-ack, receive, waitclose, join) stay on @@ -167,8 +171,8 @@ Compat mapping: `execmodel=thread` → `loop=main` + `exec=thread`; this branch plus real `-n` smoke runs (crash/endmarker tests, `main_thread_only` GUI case). Our own suite running `-n 12` green is necessary but not sufficient. -4. **Close the open decision**: channel callbacks on the loop thread — - keep or move. +4. **Open decision CLOSED (2026-07-26)**: channel callbacks moved off the + loop thread — see "Callback consumer tasks" below. 5. **xfail markers audit (done 2026-07-26, keep as-is)**: the 11 consistent XPASSes were investigated — trio's single-loop dispatch + FIFO admission makes them pass reliably when idle, but under @@ -185,6 +189,48 @@ compat mapping unblocks the ExecModel retirement, then `exec=task`; run the xdist verification (D.3) mid-C as an early canary rather than only at the end. +## Callback consumer tasks (`setcallback`, LANDED 2026-07-26) + +`setcallback` no longer delivers inline on the loop thread and no longer +pins the channel in a `ChannelFactory._callback_channels` registry. Instead +`Channel.setcallback` → `BaseGateway._start_channel_consumer` → +`SyncBridgeGateway.attach_consumer`, which on the loop: + +- moves any already-buffered mailbox items into a fresh per-channel inbox + (a `trio` memory channel), diverts future raw payloads there + (`raw.set_consumer(inbox.send_nowait, on_close)`), and nulls `_mailbox`; +- starts a consumer *task* on the host root nursery (`_run_consumer`) that is + handed a **strong** reference to the sync `Channel` — so the channel's + lifecycle is now bound to the task (and to GC once the stream closes), + replacing the strong-ref registry. + +`_run_consumer` does `async for data in inbox:` and runs each callback via +`trio.to_thread.run_sync(..., limiter=host.callback_limiter)` — off the loop, +one thread of a bounded pool (`DEFAULT_CALLBACK_THREADS=40`), sequential per +channel (order preserved), concurrent across channels. A raising callback +(or a `loads_internal` failure) → `CHANNEL_CLOSE_ERROR` + local close +(`_consumer_failed`). On EOF/close the endmarker fires (shielded, bounded by +`CONSUMER_ENDMARKER_GRACE`) and the task sets `channel._consumer_done`. + +Key invariant preserved: `waitclose()` waits on `_consumer_done` (not +`_receiveclosed`) for callback channels, so it still returns only after every +callback — including the endmarker — has run (`test_waiting_for_callbacks`, +`test_channel_endmarker_callback`). Deleted along the way: `_callback`, +`_endmarker`, `_fire_endmarker`, `_callback_channels`, +`_register_callback_channel`, and the callback branch of `_deliver_payload` +(now uniformly `mailbox.put`). `__del__`'s `CHANNEL_LAST_MESSAGE` case folds +away (a callback channel is never GC'd while open). + +Stress coverage: `testing/test_channel_stress.py` (Hypothesis) with a +`--stress=N` pytest option (profiles registered in +`testing/conftest.py::pytest_configure`; default quick profile). Hypothesis +also surfaced and we fixed a latent serializer bug: `_save_integral` only +bounds-checked the upper int4 limit, so a negative int below `-2**31` +overflowed `struct.pack('!i', …)` instead of taking the long path +(`FOUR_BYTE_INT_MIN` added; regression `test_serializer.test_int_boundaries`). + +`hypothesis` was added to the `testing` extra in `pyproject.toml`. + ## Invariants (do not regress) - No source shipping, ever. Workers import installed execnet+trio; diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 597cbdb4..e119ca68 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -10,6 +10,8 @@ import functools import itertools import json +import math +import queue as _queue import subprocess import sys import threading @@ -23,6 +25,7 @@ import trio +from ._boundary import Flag from ._trio_gateway import RECEIVE_CHUNK from ._trio_gateway import AsyncGateway from ._trio_gateway import AsyncGroup @@ -31,6 +34,8 @@ from ._trio_gateway import open_popen_process from ._trio_gateway import read_handshake_ack from ._trio_gateway import ssh_transport_args +from .gateway_base import ENDMARKER +from .gateway_base import NO_ENDMARKER_WANTED from .gateway_base import ExecModel from .gateway_base import FrameDecoder from .gateway_base import GatewayReceivedTerminate @@ -43,6 +48,17 @@ from .portal import LoopPortal from .portal import OneShot +#: default cap on concurrent threadpool threads running receiver callbacks +DEFAULT_CALLBACK_THREADS = 40 +#: bound on how long the endmarker callback may run during host shutdown +CONSUMER_ENDMARKER_GRACE = 10.0 + + +def _run_callback(callback: Callable[[Any], Any], data: bytes, channel: Any) -> None: + """Deserialize one payload and invoke the receiver callback (in a thread).""" + callback(loads_internal(data, channel)) + + if TYPE_CHECKING: from .gateway import Gateway from .gateway_base import BaseGateway @@ -255,16 +271,21 @@ def bind_sync_channel(self, channel: Any) -> None: """ ref = weakref.ref(channel) channelid = channel.id + with suppress(trio.RunFinishedError): + self.host.portal.post(self._install_sync_consumer, ref, channelid) - def bind() -> None: - raw = self._channel_for(channelid) - raw.set_consumer( - functools.partial(self._sync_payload, ref), - functools.partial(self._sync_close, ref, channelid), - ) + def _install_sync_consumer(self, ref: weakref.ref[Any], channelid: int) -> None: + """Route ``channelid``'s raw payloads/close to the sync channel (loop). - with suppress(trio.RunFinishedError): - self.host.portal.post(bind) + Idempotent: re-binding to the same hooks just re-flushes the (empty) + raw buffer, so ``attach_consumer`` can call this itself instead of + depending on the separately-posted bind having run first. + """ + raw = self._channel_for(channelid) + raw.set_consumer( + functools.partial(self._sync_payload, ref), + functools.partial(self._sync_close, ref, channelid), + ) def _sync_payload(self, ref: weakref.ref[Any], data: bytes) -> None: channel = ref() @@ -293,6 +314,141 @@ def release_channel(self, channelid: int) -> None: with suppress(trio.RunFinishedError): self.host.portal.post(self._forget_channel, channelid) + # -- receiver callbacks: a consumer task per channel -- + + def attach_consumer( + self, + channel: Any, + callback: Callable[[Any], Any], + endmarker: object, + ) -> None: + """Switch ``channel`` to callback mode: a loop task drains it. + + The switch runs on the loop (so it cannot interleave with delivery): + it moves any already-buffered items into the task's inbox, points the + channel's delivery/close at that inbox, and starts the consumer task. + It deliberately does *not* touch the raw channel's consumer (the sync + payload/close hooks bound by :meth:`bind_sync_channel` stay in place); + delivery keeps flowing through ``Channel._deliver_payload`` / + ``_close_from_remote``, which divert to the inbox once ``_has_consumer`` + is set. Rebinding the raw channel here would race the still-queued + ``bind()`` post (``run_sync`` and ``run_sync_soon`` are not mutually + ordered) and could be clobbered back to the mailbox. + + The task is handed a strong reference to ``channel`` and thus keeps it + alive for as long as it consumes -- the channel's lifecycle is bound to + the task (and to GC once the stream closes), not to a registry. + """ + + def switch() -> None: + mailbox = channel._mailbox + if mailbox is None: + raise OSError(f"{channel!r} has callback already registered") + inbox_send, inbox_recv = trio.open_memory_channel[bytes](math.inf) + + def feed(data: bytes) -> None: + with suppress(trio.BrokenResourceError, trio.ClosedResourceError): + inbox_send.send_nowait(data) + + def close_inbox() -> None: + with suppress(trio.ClosedResourceError): + inbox_send.close() + + # Drain items buffered before the switch into the task's inbox, + # preserving order. An ENDMARKER means the channel already closed. + saw_end = False + while True: + try: + item = mailbox.get_nowait() + except _queue.Empty: + break + if item is ENDMARKER: + saw_end = True + break + inbox_send.send_nowait(item) + channel._mailbox = None + done = Flag(channel.gateway._new_wakener()) + channel._consumer_done = done + channel._consumer_feed = feed + channel._consumer_close_inbox = close_inbox + + def stop() -> None: + # thread-safe: end the task's inbox from any thread (local close) + with suppress(trio.RunFinishedError, trio.ClosedResourceError): + self.host.portal.post(inbox_send.close) + + channel._consumer_stop = stop + channel._has_consumer = True + + # Guarantee the raw channel routes to us right now (idempotent with + # the bind posted at newchannel()): otherwise, if that bind has not + # run yet, inbound payloads would buffer unread in the raw channel + # and the consumer task would wait forever. + self._install_sync_consumer(weakref.ref(channel), channel.id) + + if saw_end: + close_inbox() + self.host.start_soon( + self._run_consumer, channel, inbox_recv, callback, endmarker, done + ) + + self.run_on_loop(switch) + + async def _run_consumer( + self, + channel: Any, + inbox: trio.MemoryReceiveChannel[bytes], + callback: Callable[[Any], Any], + endmarker: object, + done: Flag, + ) -> None: + """Drain ``inbox`` into ``callback`` (each call off the loop thread). + + Runs on the host loop; ``channel`` is held for the task's lifetime so + the channel stays alive while consuming. Items are delivered in order + and each callback runs in a threadpool thread. On completion (EOF, + local close, or a raising callback) the endmarker fires and ``done`` + is set -- which is what ``waitclose()`` waits on. + """ + limiter = self.host.callback_limiter + try: + async for data in inbox: + try: + await trio.to_thread.run_sync( + functools.partial(_run_callback, callback, data, channel), + limiter=limiter, + ) + except Exception as exc: + # trio.Cancelled is a BaseException and propagates past + # here (host shutdown); only a real callback/deserialize + # failure closes the channel with the error. + self._consumer_failed(channel, exc) + break + finally: + # Fire the endmarker and signal done even while the host is being + # torn down, but never let a stuck callback hang shutdown forever. + with trio.CancelScope(shield=True): + if endmarker is not NO_ENDMARKER_WANTED: + with ( + trio.move_on_after(CONSUMER_ENDMARKER_GRACE), + suppress(BaseException), + ): + await trio.to_thread.run_sync( + functools.partial(callback, endmarker), limiter=limiter + ) + done.set() + + def _consumer_failed(self, channel: Any, exc: BaseException) -> None: + """A callback (or its deserialization) raised: close with the error.""" + gateway = self.sync_gateway + gateway._trace("exception during callback: %s" % exc) + errortext = gateway._geterrortext(exc) + with suppress(OSError): + gateway._send( + Message.CHANNEL_CLOSE_ERROR, channel.id, dumps_internal(errortext) + ) + channel._close_from_remote(RemoteError(errortext), sendonly=False) + def host_call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: """Blocking host call that parks correctly for the wait backend. @@ -406,6 +562,7 @@ def __init__(self, name: str = "execnet-trio-host") -> None: self._ready = threading.Event() self._shutdown: trio.Event | None = None self._started = False + self._callback_limiter: trio.CapacityLimiter | None = None def start(self) -> None: if self._started: @@ -422,6 +579,13 @@ def portal(self) -> LoopPortal: raise RuntimeError("TrioHost is not running") return self._portal + @property + def callback_limiter(self) -> trio.CapacityLimiter: + """Bound on concurrent threadpool threads running receiver callbacks.""" + if self._callback_limiter is None: + raise RuntimeError("TrioHost is not running") + return self._callback_limiter + def is_host_thread(self) -> bool: return self._portal is not None and self._portal.is_loop_thread() @@ -431,6 +595,7 @@ def _run(self) -> None: async def _main(self) -> None: self._portal = LoopPortal() self._shutdown = trio.Event() + self._callback_limiter = trio.CapacityLimiter(DEFAULT_CALLBACK_THREADS) try: async with trio.open_nursery() as nursery: self._nursery = nursery diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 2b183c1e..8eaa770e 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -13,7 +13,6 @@ import builtins import os -import queue as _queue import struct import sys import threading @@ -371,6 +370,23 @@ class Channel: TimeoutError = TimeoutError _INTERNALWAKEUP = 1000 _executing = False + #: set once a receiver callback is attached. A consumer *task* on the + #: loop drains this channel and runs the callback in a threadpool thread; + #: the task holds the channel alive, so a callback channel's lifecycle is + #: bound to consumption (and to GC once the stream closes) rather than to + #: a strong registry. + _has_consumer = False + #: loop-side hooks installed while a consumer is attached: divert one + #: payload into / close the consumer task's inbox (set by the Trio session, + #: so they encapsulate the trio memory channel; gateway_base stays trio-free). + _consumer_feed: Callable[[bytes], None] | None = None + _consumer_close_inbox: Callable[[], None] | None = None + #: thread-safe "stop the consumer" hook -- ends the task's inbox. + _consumer_stop: Callable[[], None] | None = None + #: set by the consumer task once it has drained every item and fired the + #: endmarker; ``waitclose()`` waits on this (instead of ``_receiveclosed``) + #: so it still guarantees "all callbacks have run" before returning. + _consumer_done: Flag | None = None def __init__(self, gateway: BaseGateway, id: int) -> None: """:private:""" @@ -380,10 +396,8 @@ def __init__(self, gateway: BaseGateway, id: int) -> None: # XXX: defaults copied from Unserializer self._strconfig = getattr(gateway, "_strconfig", (True, False)) self.id = id - # serialized payloads (or ENDMARKER); None once a callback is set + # serialized payloads (or ENDMARKER); None once a consumer is attached self._mailbox: Mailbox[Any] | None = Mailbox(gateway._new_wakener()) - self._callback: Callable[[Any], Any] | None = None - self._endmarker: object = NO_ENDMARKER_WANTED self._closed = False self._receiveclosed = Flag(gateway._new_wakener()) self._remoteerrors: list[RemoteError] = [] @@ -398,37 +412,19 @@ def setcallback( ) -> None: """Set a callback function for receiving items. - All already-queued items will immediately trigger the callback. - Afterwards the callback will execute in the receiver (loop) thread - for each received data item and calls to ``receive()`` will - raise an error. - If an endmarker is specified the callback will eventually - be called with the endmarker when the channel closes. + A consumer task on the gateway's loop drains this channel and runs + ``callback`` for each received item in a threadpool thread (so a slow + callback never blocks the loop); items for one channel are delivered + strictly in order. Already-queued items are delivered first. After + this call ``receive()`` raises an error. + + The task keeps the channel alive for as long as it is consuming, so a + callback channel need not be referenced elsewhere. If an endmarker is + specified the callback is eventually called with it when the channel + closes, and ``waitclose()`` does not return until every callback + (including the endmarker) has run. """ - - def switch() -> None: - # Runs on the loop thread (inline without a session), so the - # switch-over cannot interleave with payload delivery. - mailbox = self._mailbox - if mailbox is None: - raise OSError(f"{self!r} has callback already registered") - self._mailbox = None - while 1: - try: - olditem = mailbox.get_nowait() - except _queue.Empty: - if not (self._closed or self._receiveclosed.is_set()): - self._callback = callback - self._endmarker = endmarker - self.gateway._channelfactory._register_callback_channel(self) - break - if olditem is ENDMARKER: - if endmarker is not NO_ENDMARKER_WANTED: - callback(endmarker) - break - callback(loads_internal(olditem, self)) - - self.gateway._run_on_loop(switch) + self.gateway._start_channel_consumer(self, callback, endmarker) def __repr__(self) -> str: flag = (self.isclosed() and "closed") or "open" @@ -454,17 +450,16 @@ def __del__(self) -> None: # in which case the process will go away and we probably # don't need to try to send a closing or last message # (and often it won't work anymore to send things out) + # A callback channel is held by its consumer task until the stream + # closes, so by the time __del__ runs it is never in the "opened" + # state -- this branch only ever fires for a plain receive channel. if Message is not None: - if self._mailbox is None: # has_callback - msgcode = Message.CHANNEL_LAST_MESSAGE - else: - msgcode = Message.CHANNEL_CLOSE with suppress(OSError, ValueError): # ignore problems with sending # Never wait during GC: post the close best-effort. send = getattr( self.gateway, "_send_nonblocking", self.gateway._send ) - send(msgcode, self.id) + send(Message.CHANNEL_CLOSE, self.id) with suppress(Exception): self.gateway._release_channel(self.id) @@ -482,46 +477,43 @@ def _getremoteerror(self): # loop-side delivery (called by the session's raw-channel consumer) # def _deliver_payload(self, data: bytes) -> None: - """Route one inbound serialized payload (loop thread).""" + """Route one inbound serialized payload (loop thread). + + A callback channel diverts payloads into its consumer task's inbox; a + plain channel queues them for ``receive()``. + """ if self._closed: return # late data for a locally closed channel: drop - callback = self._callback - if callback is None: - mailbox = self._mailbox - if mailbox is not None: - mailbox.put(data) - # no mailbox and no callback: closed for receiving -- drop - else: - try: - callback(loads_internal(data, self)) - except Exception as exc: - self.gateway._trace("exception during callback: %s" % exc) - errortext = self.gateway._geterrortext(exc) - self.gateway._send( - Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(errortext) - ) - self._close_from_remote(RemoteError(errortext)) + feed = self._consumer_feed + if feed is not None: + feed(data) + return + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(data) + # no consumer and no mailbox: closed for receiving -- drop def _close_from_remote(self, remoteerror=None, *, sendonly: bool = False) -> None: - """Close initiated by the peer or session shutdown (loop thread).""" + """Close initiated by the peer or session shutdown (loop thread). + + For a callback channel the consumer task ends separately (its inbox is + closed) and fires the endmarker; here we only record the state. + """ if remoteerror: self._remoteerrors.append(remoteerror) - mailbox = self._mailbox - if mailbox is not None: - mailbox.put(ENDMARKER) - self._fire_endmarker() + if self._has_consumer: + close_inbox = self._consumer_close_inbox + if close_inbox is not None: + close_inbox() # ends the consumer task (it fires the endmarker) + else: + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) self.gateway._channelfactory._no_longer_opened(self.id) if not sendonly: # otherwise #--> "sendonly" self._closed = True # --> "closed" self._receiveclosed.set() - def _fire_endmarker(self) -> None: - callback = self._callback - if callback is not None: - self._callback = None - if self._endmarker is not NO_ENDMARKER_WANTED: - callback(self._endmarker) - # # public API for channel objects # @@ -588,10 +580,16 @@ def close(self, error=None) -> None: self._remoteerrors.append(error) self._closed = True # --> "closed" self._receiveclosed.set() - mailbox = self._mailbox - if mailbox is not None: - mailbox.put(ENDMARKER) - self._fire_endmarker() + if self._has_consumer: + # End the consumer task's inbox; it drains any buffered items, + # fires the endmarker, and sets _consumer_done. + stop = self._consumer_stop + if stop is not None: + stop() + else: + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) self.gateway._channelfactory._no_longer_opened(self.id) self.gateway._release_channel(self.id) @@ -611,9 +609,16 @@ def waitclose(self, timeout: float | None = None) -> None: self.TimeoutError is raised after the specified number of seconds (default is None, i.e. wait indefinitely). """ - # wait for non-"opened" state - self._receiveclosed.wait(timeout=timeout) - if not self._receiveclosed.is_set(): + # For a callback channel wait on the consumer task finishing (so every + # callback, including the endmarker, has run); otherwise wait for the + # non-"opened" state directly. + signal = ( + self._consumer_done + if self._consumer_done is not None + else (self._receiveclosed) + ) + signal.wait(timeout=timeout) + if not signal.is_set(): raise self.TimeoutError("Timeout after %r seconds" % timeout) error = self._getremoteerror() if error: @@ -693,16 +698,14 @@ class ChannelFactory: Message routing lives in the Trio session (the sync channel binds a consumer on the session's raw channel); the factory only tracks live channels -- weakly, so dropping the last user reference triggers - ``Channel.__del__``'s close message -- and keeps channels with a - registered callback strongly alive until they close. + ``Channel.__del__``'s close message. A callback channel is kept alive by + its consumer task rather than by any registry here. """ def __init__(self, gateway: BaseGateway, startcount: int = 1) -> None: self._channels: weakref.WeakValueDictionary[int, Channel] = ( weakref.WeakValueDictionary() ) - # channels kept strongly alive while their callback is registered - self._callback_channels: dict[int, Channel] = {} self._writelock = threading.Lock() self.gateway = gateway self.count = startcount @@ -739,12 +742,8 @@ def channels(self) -> list[Channel]: # # internal methods, called from the loop thread (or local close paths) # - def _register_callback_channel(self, channel: Channel) -> None: - self._callback_channels[channel.id] = channel - def _no_longer_opened(self, id: int) -> None: self._channels.pop(id, None) - self._callback_channels.pop(id, None) def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> None: """Close ``id`` as if the peer had closed it (no message is sent).""" @@ -885,6 +884,22 @@ def _run_on_loop(self, sync_fn: Callable[[], Any]) -> Any: return sync_fn() return session.run_on_loop(sync_fn) + def _start_channel_consumer( + self, + channel: Channel, + callback: Callable[[Any], Any], + endmarker: object, + ) -> None: + """Attach a receiver callback: hand the channel to a consumer task. + + The task drains the channel on the loop and runs ``callback`` in a + threadpool thread, holding the channel alive while it consumes. + """ + session = self._trio_session + if session is None: + raise OSError(f"cannot set callback on {channel!r}: no active session") + session.attach_consumer(channel, callback, endmarker) + def _terminate_execution(self) -> None: pass diff --git a/testing/test_channel.py b/testing/test_channel.py index f06b8632..f1b0b454 100644 --- a/testing/test_channel.py +++ b/testing/test_channel.py @@ -305,6 +305,50 @@ def f(item): channel.send(1) channel.waitclose() + @needs_early_gc + def test_callback_channel_collected_after_close(self, gw: Gateway) -> None: + # A callback channel is kept alive only by its consumer task: while it + # is consuming it survives with no user reference, and once the stream + # closes and the callback has run the object becomes collectable. + import gc + import weakref + + received: list[int] = [] + channel = gw.remote_exec("channel.send(1); channel.send(2)") + channel.setcallback(received.append) + ref = weakref.ref(channel) + del channel # only the consumer task holds it now + + deadline = time.time() + TESTTIMEOUT + while ref() is not None and time.time() < deadline: + gc.collect() + time.sleep(0.05) + assert received == [1, 2] + assert ref() is None # consumer finished -> no strong refs -> reclaimed + + def test_callbacks_run_off_the_loop_thread(self, gw: Gateway) -> None: + # A slow callback on one channel must not block delivery to another: + # callbacks run in threadpool threads, not inline on the loop. + import threading + + release = threading.Event() + fast_ran = threading.Event() + + def slow(item: object) -> None: + release.wait(TESTTIMEOUT) + + chan_slow = gw.remote_exec("channel.send(1); channel.receive()") + chan_fast = gw.remote_exec("channel.send(1)") + chan_slow.setcallback(slow) + chan_fast.setcallback(lambda item: fast_ran.set()) + + # the fast callback fires even while the slow one is still blocking + assert fast_ran.wait(TESTTIMEOUT) + release.set() + chan_slow.send(0) # let the remote finish and close + chan_slow.waitclose(TESTTIMEOUT) + chan_fast.waitclose(TESTTIMEOUT) + class TestChannelFile: def test_channel_file_write(self, gw: Gateway) -> None: From 34c9953dadc2c1eba3888e23de3e2faf39086772 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 10:00:55 +0200 Subject: [PATCH 49/91] fix: serialize ints below the signed int4 minimum via the long opcode _save_integral only bounds-checked the upper int4 limit, so a negative int below -2**31 took the short (INT/LONG) path and overflowed struct.pack('!i', ...) with a struct.error instead of using the arbitrary-precision long opcode. Gate the short path on both ends (FOUR_BYTE_INT_MIN..MAX). Found by the new channel-callback Hypothesis stress tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/execnet/gateway_base.py | 5 ++++- testing/test_serializer.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 8eaa770e..c59f4437 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1051,6 +1051,7 @@ def bchr(n: int) -> bytes: DUMPFORMAT_VERSION = bchr(2) FOUR_BYTE_INT_MAX = 2147483647 +FOUR_BYTE_INT_MIN = -2147483648 FLOAT_FORMAT = "!d" FLOAT_FORMAT_SIZE = struct.calcsize(FLOAT_FORMAT) @@ -1407,7 +1408,9 @@ def _write_byte_sequence(self, bytes_: bytes) -> None: self._write(bytes_) def _save_integral(self, i: int, short_op: bytes, long_op: bytes) -> None: - if i <= FOUR_BYTE_INT_MAX: + # The short op packs a signed 4-byte int; anything outside that range + # (in either direction) goes through the arbitrary-precision long op. + if FOUR_BYTE_INT_MIN <= i <= FOUR_BYTE_INT_MAX: self._write(short_op) self._write_int4(i) else: diff --git a/testing/test_serializer.py b/testing/test_serializer.py index cb2d422c..87adf04c 100644 --- a/testing/test_serializer.py +++ b/testing/test_serializer.py @@ -126,6 +126,26 @@ def test_long(load, dump) -> None: assert v == really_big +@pytest.mark.parametrize( + "value", + [ + "2147483647", # int4 max: short path + "-2147483648", # int4 min: short path + "2147483648", # just over max: long path + "-2147483649", # just under min: long path (used to crash in struct.pack) + "9223372036854775807324234", + "-9223372036854775807324234", + ], +) +def test_int_boundaries(value, dump, load) -> None: + # regression: negative ints below the signed int4 minimum must take the + # arbitrary-precision long path instead of overflowing the 4-byte pack. + p = dump(value) + tp, v = load(p) + assert tp == "int" + assert v == value + + def test_bytes(dump, load) -> None: p = dump("b'hi'") tp, v = load(p) From 0d36f7fd8a05524e3dbba03e4f669d7921e7b8dc Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 10:01:35 +0200 Subject: [PATCH 50/91] test: hypothesis stress tests for channel callbacks with a --stress knob testing/test_channel_stress.py hammers the setcallback consumer-task path with randomised traffic: ordering/completeness, many-channel isolation, receive-then-switch, endmarker-last, and GC-reclaim-after-close. A --stress=N pytest option scales the number of Hypothesis examples (profiles registered in conftest.pytest_configure; a quick profile runs by default). hypothesis added to the testing extra. Co-Authored-By: Claude Opus 4.8 (1M context) --- pyproject.toml | 1 + testing/conftest.py | 37 ++++++++++ testing/test_channel_stress.py | 123 +++++++++++++++++++++++++++++++++ uv.lock | 75 ++++++++++++++++++++ 4 files changed, 236 insertions(+) create mode 100644 testing/test_channel_stress.py diff --git a/pyproject.toml b/pyproject.toml index 6e12c8da..f29b24d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ testing = [ "pre-commit", "pytest>8.0", "pytest-timeout", + "hypothesis", "tox", "hatch", "uv", diff --git a/testing/conftest.py b/testing/conftest.py index 7c3ed7a6..becf10f2 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -69,6 +69,43 @@ def pytest_addoption(parser: pytest.Parser) -> None: "page on invalid addresses" ), ) + group.addoption( + "--stress", + action="store", + dest="stress", + default=None, + metavar="N", + help=( + "how hard the Hypothesis stress tests try: number of examples " + "per test (e.g. --stress=500). Without it a quick profile runs." + ), + ) + + +def pytest_configure(config: pytest.Config) -> None: + # Register Hypothesis profiles scaled by --stress. The stress tests reuse + # a function-scoped gateway across examples on purpose (spawning one per + # example would dominate the runtime), and each round-trip can be slow, so + # the health checks for those are suppressed. + try: + from hypothesis import HealthCheck + from hypothesis import settings + except ImportError: + return + common = dict( + deadline=None, + suppress_health_check=[ + HealthCheck.function_scoped_fixture, + HealthCheck.too_slow, + ], + ) + settings.register_profile("execnet-quick", max_examples=15, **common) + stress = config.getoption("stress") + if stress is not None: + settings.register_profile("execnet-stress", max_examples=int(stress), **common) + settings.load_profile("execnet-stress") + else: + settings.load_profile("execnet-quick") @pytest.fixture diff --git a/testing/test_channel_stress.py b/testing/test_channel_stress.py new file mode 100644 index 00000000..8090367d --- /dev/null +++ b/testing/test_channel_stress.py @@ -0,0 +1,123 @@ +"""Hypothesis stress tests for the channel callback (consumer-task) machinery. + +These hammer ``setcallback`` -- the loop-task-plus-threadpool consumer -- with +randomised traffic to check the invariants that must hold no matter the timing: + +* every item reaches the callback, exactly once and in send order; +* many callback channels run concurrently without cross-talk; +* switching to a callback after some ``receive()`` calls loses nothing; +* the endmarker is always delivered last; +* a callback channel with no user reference is kept alive by its consumer. + +Use ``--stress=N`` to raise the number of examples per test (default: a quick +profile registered in ``conftest.pytest_configure``). +""" + +from __future__ import annotations + +import gc +import weakref + +import pytest + +hypothesis = pytest.importorskip("hypothesis") +from hypothesis import given # noqa: E402 +from hypothesis import strategies as st # noqa: E402 + +from execnet.gateway import Gateway # noqa: E402 + +# High --stress levels replay one test many times; lift the per-test timeout +# well above the default so that only a real hang (bounded by TESTTIMEOUT on +# every blocking call below) fails, not sheer example count. +pytestmark = pytest.mark.timeout(600) + +TESTTIMEOUT = 10.0 + +# Varied payloads: unbounded ints exercise both the short and long serializer +# paths (and their boundaries), on top of the callback machinery itself. +payload_strategy = st.one_of( + st.integers(), + st.text(max_size=20), + st.booleans(), + st.none(), +) +items_strategy = st.lists(payload_strategy, max_size=40) + + +def _echo(channel, items): + """Remote: send every item, in order, then let the channel close.""" + for item in items: + channel.send(item) + + +class TestCallbackStress: + # popen only: fast to spawn and the callback path is transport-independent + gwtype = "popen" + + @given(data=items_strategy) + def test_callback_receives_all_in_order( + self, gw: Gateway, data: list[int] + ) -> None: + collected: list[int] = [] + channel = gw.remote_exec(_echo, items=data) + channel.setcallback(collected.append) + channel.waitclose(TESTTIMEOUT) + assert collected == data + + @given(batches=st.lists(items_strategy, min_size=1, max_size=6)) + def test_many_channels_stay_ordered_and_isolated( + self, gw: Gateway, batches: list[list[int]] + ) -> None: + results: list[list[int]] = [[] for _ in batches] + channels = [] + for index, items in enumerate(batches): + channel = gw.remote_exec(_echo, items=items) + channel.setcallback(results[index].append) + channels.append(channel) + for channel in channels: + channel.waitclose(TESTTIMEOUT) + assert results == batches + + @given(data=st.data()) + def test_receive_then_switch_loses_nothing( + self, gw: Gateway, data: st.DataObject + ) -> None: + items = data.draw(items_strategy) + split = data.draw(st.integers(min_value=0, max_value=len(items))) + channel = gw.remote_exec(_echo, items=items) + first = [channel.receive(TESTTIMEOUT) for _ in range(split)] + rest: list[int] = [] + channel.setcallback(rest.append) + channel.waitclose(TESTTIMEOUT) + assert first + rest == items + + @given(data=items_strategy) + def test_endmarker_is_always_last(self, gw: Gateway, data: list[int]) -> None: + endmarker = object() + collected: list[object] = [] + channel = gw.remote_exec(_echo, items=data) + channel.setcallback(collected.append, endmarker=endmarker) + channel.waitclose(TESTTIMEOUT) + assert collected == [*data, endmarker] + + @pytest.mark.skipif( + "not hasattr(sys, 'getrefcount')", reason="needs refcount GC semantics" + ) + @given(data=st.lists(payload_strategy, min_size=1, max_size=40)) + def test_callback_channel_kept_alive_then_collected( + self, gw: Gateway, data: list[object] + ) -> None: + collected: list[int] = [] + channel = gw.remote_exec(_echo, items=data) + channel.setcallback(collected.append) + ref = weakref.ref(channel) + del channel # only the consumer task holds it now + + import time + + deadline = time.time() + TESTTIMEOUT + while ref() is not None and time.time() < deadline: + gc.collect() + time.sleep(0.02) + assert collected == data + assert ref() is None # consumer finished -> reclaimed diff --git a/uv.lock b/uv.lock index 3b8bb46e..311eb0d8 100644 --- a/uv.lock +++ b/uv.lock @@ -393,6 +393,7 @@ gevent = [ testing = [ { name = "asyncssh" }, { name = "hatch" }, + { name = "hypothesis" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-timeout" }, @@ -416,6 +417,7 @@ requires-dist = [ { name = "asyncssh", marker = "extra == 'testing'" }, { name = "gevent", marker = "extra == 'gevent'" }, { name = "hatch", marker = "extra == 'testing'" }, + { name = "hypothesis", marker = "extra == 'testing'" }, { name = "pre-commit", marker = "extra == 'testing'" }, { name = "pytest", marker = "extra == 'testing'", specifier = ">8.0" }, { name = "pytest-timeout", marker = "extra == 'testing'" }, @@ -680,6 +682,79 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/aa/8caf6a0a3e62863cbb9dab27135660acba46903b703e224f14f447e57934/hyperlink-21.0.0-py2.py3-none-any.whl", hash = "sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4", size = 74638, upload-time = "2021-01-08T05:51:22.906Z" }, ] +[[package]] +name = "hypothesis" +version = "6.161.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/17/94/d208ced653376e7e0a2f0429ee5be864dd0b59393b98a8b41a35ceb4d035/hypothesis-6.161.5.tar.gz", hash = "sha256:ba73a3c3b68e63a0bee5ea1a8a13efce60bcc7ee5fc7e71df2954db39c225b95", size = 486653, upload-time = "2026-07-25T14:39:34.866Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/58/e48aa878d119474631fc097d511b5e70807dfe56be4b244cce0275f1805e/hypothesis-6.161.5-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:4c89e0c35d7cd70af2a0a5f0b5ad69d1898c369adf15115c6d4271d47bf1b280", size = 766970, upload-time = "2026-07-25T14:38:42.613Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ff/c83c1a4d5d3c4b6a4dc1fcb5069245171b7a30ad5dd31f44282e8881e089/hypothesis-6.161.5-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:1bb987e519a6d00675bab124c925000637e2e59384196b0ded5d108dc1851449", size = 762574, upload-time = "2026-07-25T14:38:15.427Z" }, + { url = "https://files.pythonhosted.org/packages/d2/04/7a5ba16cd14340009d1d8aa46083bf3a3a55c5b0d1ccf553e6f6748ee0e8/hypothesis-6.161.5-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5990e9e2e145ff5369c95ad684bd6eee7ebd2d4de37d37eea4c6ca5e339e2a52", size = 1091754, upload-time = "2026-07-25T14:38:38.562Z" }, + { url = "https://files.pythonhosted.org/packages/70/69/031913f7408ebb41e31a9b20e5bca75c5a64ab151d57ca75e944a09ba6e0/hypothesis-6.161.5-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2a5c6400c58c9c3d616ad7177ada12ae577e5103b3b0df6e79920729250bf100", size = 1120372, upload-time = "2026-07-25T14:38:18.962Z" }, + { url = "https://files.pythonhosted.org/packages/b9/95/360884e86ab99099340fca781531286c26d40ce32c853097e76537cc0726/hypothesis-6.161.5-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:71aa718e08bdfbacd1a5d8f86ffd55f1d86e64a57cec6a107144d883b522823e", size = 1141230, upload-time = "2026-07-25T14:39:03.757Z" }, + { url = "https://files.pythonhosted.org/packages/bc/5b/6f3c9fbe9191432f33c9aaf951cf5a5416533848999f2e2c6a20ec00098b/hypothesis-6.161.5-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:66912038467e1af791f76336bc079d669f7c8bbf7aeb85bdd52e83259d885122", size = 1096611, upload-time = "2026-07-25T14:38:34.688Z" }, + { url = "https://files.pythonhosted.org/packages/32/c7/d126b9f66295fed5d5745f98ad92f485c98dee303dd67ea7a53fe4a903aa/hypothesis-6.161.5-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:99d8b2380d0bba602df963d6e42d56bb50cf1734376c1b9b14dcab3cec21814e", size = 1133382, upload-time = "2026-07-25T14:38:20.27Z" }, + { url = "https://files.pythonhosted.org/packages/56/de/277a17091687f298079c197cadd806d64908c5a08c9c91bb27b834192503/hypothesis-6.161.5-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:19e57e621c6abd98123a91bd4b022bde7760ba709f1ca121f822d9aa291bd001", size = 1265592, upload-time = "2026-07-25T14:38:11.734Z" }, + { url = "https://files.pythonhosted.org/packages/d7/e5/3f15b1a43cb70b223427ce3a9a6afbe430bf1754b3346ac546a3df38b96b/hypothesis-6.161.5-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:3b74cced62995ed2e0096fa63d0b7babd1fa08ce1944cff41134275516848708", size = 1393424, upload-time = "2026-07-25T14:38:46.738Z" }, + { url = "https://files.pythonhosted.org/packages/1a/1d/a3cf1441550dbf5a362dcfa19c831721f3bb6a749eb53ecbdfc983959027/hypothesis-6.161.5-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:fd37a0257647288be8c27d56a78a31da2b65e3b6254511f03164f5e1c786187a", size = 1266197, upload-time = "2026-07-25T14:39:33.052Z" }, + { url = "https://files.pythonhosted.org/packages/57/e5/2616432d6144057b6c8f4409c95d95610bdbdf07fecb6618e98761861c24/hypothesis-6.161.5-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cf9dfba054164e481f05774c70b65ea88ba735b986cdc44f5210ef40871a32e9", size = 1308330, upload-time = "2026-07-25T14:39:27.86Z" }, + { url = "https://files.pythonhosted.org/packages/f7/cd/9e9c7a88858f84447e44e45c8b9fd1058120b4d5fa9bab9c3ceb9b9cf96b/hypothesis-6.161.5-cp310-abi3-win32.whl", hash = "sha256:8763938787e6c98e461f8c11139981597d083e6915a9956db1c26cc609bc7b19", size = 652819, upload-time = "2026-07-25T14:38:35.915Z" }, + { url = "https://files.pythonhosted.org/packages/03/23/aae37b5bb2014e0e01e372498f7f5977b9fd61a0179143854b7fb0538b1c/hypothesis-6.161.5-cp310-abi3-win_amd64.whl", hash = "sha256:79b5d069095a726dca5b6b7e4c5d268acc809a6595c46d7eee860708f960cb8f", size = 658970, upload-time = "2026-07-25T14:39:05.239Z" }, + { url = "https://files.pythonhosted.org/packages/70/77/fbda7005f891f534af0a74cb5acd7a67cb1a7fbee1ae170d176c7dcd5e0a/hypothesis-6.161.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:c73c86afe87b1bf81af586402d41bff69d93c8c34517a81d3a88a1e28e8662e2", size = 767688, upload-time = "2026-07-25T14:39:13.418Z" }, + { url = "https://files.pythonhosted.org/packages/d3/8d/f884e10029018504504df6100faf3795918e1dd14d4b6f4102c3fcb21acf/hypothesis-6.161.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7b289d33f0154850baed802d088564bba83dae58bae6fb9157e498c43d758afd", size = 763399, upload-time = "2026-07-25T14:38:10.49Z" }, + { url = "https://files.pythonhosted.org/packages/40/23/75d9c1f7ad8a52a14c5981c9ba97b1d4abfd263cd48ce7ffb036ce1f5a16/hypothesis-6.161.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e60afc424627a4119d89f02425049a97e500e022aef6a06d76e987e266e3d720", size = 1092278, upload-time = "2026-07-25T14:38:51.187Z" }, + { url = "https://files.pythonhosted.org/packages/be/52/efc7c45352636fc71d24590893e52f0e8b4ca42984aea870b47831720fef/hypothesis-6.161.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:defa4f24aa62b3a6d07271a4d54f31c7bb07ce1f3fa86cc0705890a071a58301", size = 1141822, upload-time = "2026-07-25T14:38:48.298Z" }, + { url = "https://files.pythonhosted.org/packages/6b/48/697b194412fed03cbf9757771b9c3b51ca062e1d78fe2b5ddf833a25b7f2/hypothesis-6.161.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4f5e8be93cf0da10b8a2f1044c12f2aa506e80c5071e84dffeec4311fddeb8d7", size = 1266206, upload-time = "2026-07-25T14:39:31.414Z" }, + { url = "https://files.pythonhosted.org/packages/2f/b8/56c7996272a8310737c44378fe181befbe3ab41f72ee3d7180e32269cb7c/hypothesis-6.161.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:46aa94fa4107f97760b63c83142cd2fe208f38a6708736494cabfcf219b137cb", size = 1308570, upload-time = "2026-07-25T14:39:09.772Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ba/ed5baa948085a08ed38362e0e88ddad7a4a524b0a2a1581c8138ce7b307a/hypothesis-6.161.5-cp310-cp310-win_amd64.whl", hash = "sha256:44e25c35123a2b77ebd2df356e1a78bf2c9abbf875222ff252b3012801bfb143", size = 658822, upload-time = "2026-07-25T14:38:45.402Z" }, + { url = "https://files.pythonhosted.org/packages/9d/82/60f7213dd5262863646bbf31ca28e1dba68730080c14f39744d110733932/hypothesis-6.161.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f6d55821d13890875d5a50a1433448c8a86f7f83e85103a527dad232d18c470e", size = 767446, upload-time = "2026-07-25T14:38:39.904Z" }, + { url = "https://files.pythonhosted.org/packages/20/b3/145fe198d155ac394cac067ae852074adbca2c015a7244fdb3df2f655c19/hypothesis-6.161.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ecb5878b81beb1dfda4e573375a816b1ffa7931d7899a1a2d7216afa5e1efb4f", size = 763215, upload-time = "2026-07-25T14:38:30.624Z" }, + { url = "https://files.pythonhosted.org/packages/fc/c0/0ea73426ba5b2504d4e3f4e6c7589f5808ad6eef4c922f15766c0537bbc2/hypothesis-6.161.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bff9062c1af02e391c0e48aa75f989393651c6e07f1b50eddeadaa8ca440f944", size = 1092115, upload-time = "2026-07-25T14:39:18.121Z" }, + { url = "https://files.pythonhosted.org/packages/dc/65/2a52daac1d39d0a1a88f96a602ff997e4665c5a9acc4d029e3168e536cd1/hypothesis-6.161.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:570ab5afc34dd7a78e4d1ab1a0ea4e026bbae6caee79dc04245d0347f1d548a7", size = 1141577, upload-time = "2026-07-25T14:38:55.841Z" }, + { url = "https://files.pythonhosted.org/packages/91/cd/77693dbb6766345d980fdfa41f7e7d3d0a676469d156c8e9455c1ebda67d/hypothesis-6.161.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6a7541433984c5b5fb4ad433a715808579cba16ebaca6a214a99d2f8c35ed49d", size = 1265879, upload-time = "2026-07-25T14:38:21.759Z" }, + { url = "https://files.pythonhosted.org/packages/ed/af/ea72a466210a43b35f3e9d86350aca94529a00d2aff698b4cdd490238955/hypothesis-6.161.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f3118cce6e66366e2c64389a5dd6e9bc49c697c366d62709e6fda0b3a8a7d997", size = 1308593, upload-time = "2026-07-25T14:38:23.221Z" }, + { url = "https://files.pythonhosted.org/packages/50/8c/d0ae6738baed9061ca29b18de506f5bc8dc84649b216b55213474ec7da9b/hypothesis-6.161.5-cp311-cp311-win_amd64.whl", hash = "sha256:d8cbd7c938b191d5f9bf846a79e81484135a239cd45bf194993949645495ee6f", size = 658671, upload-time = "2026-07-25T14:38:54.453Z" }, + { url = "https://files.pythonhosted.org/packages/bf/37/71a53545f2732574bdd7a45a6e073043bce3447fec4aa722211ebf742f2e/hypothesis-6.161.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:37016b6b4842993b06ee453dda4dedac5ef328cefa6daa36372d261abf12db43", size = 768565, upload-time = "2026-07-25T14:38:14.362Z" }, + { url = "https://files.pythonhosted.org/packages/e6/56/7f32ba1443d5819b324444e7d5ebcf207ba4a0f9219a1693a7994293fd3f/hypothesis-6.161.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d60a204b86936b29d8914f93a99c59a2ecd4e311be793bf32a3f94774d41e016", size = 760188, upload-time = "2026-07-25T14:38:07.894Z" }, + { url = "https://files.pythonhosted.org/packages/07/0d/699436929d2980103c9ddba153872ebed55cfcc13f0f78c1741ac6128205/hypothesis-6.161.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cee9f1a563830a9137b9d1505c9c4fac760bc43dfbfa5873bebefe36c8dca4ec", size = 1090570, upload-time = "2026-07-25T14:39:00.492Z" }, + { url = "https://files.pythonhosted.org/packages/8a/a5/2c28b58d16443fd8ad63e4f287440291a42135073748e5e511084f32a6f3/hypothesis-6.161.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7fcf44f9876b7b4fdc40b607342219c634cf6a1acc708dc0ff88e8e48ae8ea2", size = 1140601, upload-time = "2026-07-25T14:39:21.403Z" }, + { url = "https://files.pythonhosted.org/packages/4e/fe/f0f87c38dc741687bace80553478f8c9796ba5b6e68793a80990d79ec5ac/hypothesis-6.161.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:59088f6458d2a6ef04724a2a639418c97dd8b8c280bfd222fcc5566854560caf", size = 1263341, upload-time = "2026-07-25T14:38:41.257Z" }, + { url = "https://files.pythonhosted.org/packages/44/db/02e30bcdb3434f4eee8fdbebf5cf151fecd37d2a76f2fc19f2745c93ca38/hypothesis-6.161.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ea8e7e6bed5407ab902026d8cac74d10cac894a496d82e261a0beaca499114dc", size = 1307604, upload-time = "2026-07-25T14:39:19.867Z" }, + { url = "https://files.pythonhosted.org/packages/1d/70/1f6349a244f13c5a383a62ca674d3175018c7fc30fbaad10faa859d5c2cf/hypothesis-6.161.5-cp312-cp312-win_amd64.whl", hash = "sha256:a4ad11f4eafd561a672cb9fa977d0f58ab4ae9cf4b160dd3c12c351089f3e2be", size = 656103, upload-time = "2026-07-25T14:38:57.639Z" }, + { url = "https://files.pythonhosted.org/packages/52/07/86f98ebd945f3f8a821c1e7bf9b530902675145d93ed44fa56a309cc24fd/hypothesis-6.161.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3b4c308ff19741ab9f795cac61cce61227f55a1b9767ad525a34de8b01391d1d", size = 768455, upload-time = "2026-07-25T14:38:27.821Z" }, + { url = "https://files.pythonhosted.org/packages/70/11/93c00ba5c77b886017ea8eaa42de8ca937e319032c4251b0540fc1838ae1/hypothesis-6.161.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ef16fac46cd5675504a4d3824f443f6842b80fae8b6d15ecffa9c90e0fcf4522", size = 760099, upload-time = "2026-07-25T14:38:04.918Z" }, + { url = "https://files.pythonhosted.org/packages/86/13/c8787cfae81c816976c1b3aaa43294c3abd6fd055a0147122edd7357a349/hypothesis-6.161.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc4c6aab8dbabb98b9c5ca8da8fa9294d2e5173432a12abcb447e83e666fa430", size = 1090486, upload-time = "2026-07-25T14:39:26.243Z" }, + { url = "https://files.pythonhosted.org/packages/f8/be/48f4f56bfb24787aa27e5ae851d1aee5214ab6e95311cf8c88a56707941f/hypothesis-6.161.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f485e3c7ead1f76a8c060b16c6b50e5615264819e95b794c6fc6882a9cbfdc4", size = 1140433, upload-time = "2026-07-25T14:38:59.043Z" }, + { url = "https://files.pythonhosted.org/packages/2e/50/c119bfec24d267e4af5bdefda8fd490f7b3a7ee36a52d5ac0b88e07c10b3/hypothesis-6.161.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:75bb899b2db5c45dbfa6ef704c26259bfd27fb3418179c3bd5788b7f2ecce72d", size = 1263296, upload-time = "2026-07-25T14:38:06.719Z" }, + { url = "https://files.pythonhosted.org/packages/81/89/eeb97a9684e3eaae801dd538058202b0b65c2aaecea0211e17d79f731170/hypothesis-6.161.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:894e5d9e2cd97798c3dc2731699098c7f800a7905572db1677e0d27d3b41f541", size = 1307324, upload-time = "2026-07-25T14:38:24.881Z" }, + { url = "https://files.pythonhosted.org/packages/15/9d/e0c64a64721ba169dff5a6f12236d5102f8c76f661598d7837591ec7b375/hypothesis-6.161.5-cp313-cp313-win_amd64.whl", hash = "sha256:62452bb19d73496a74919d82afc6306bf9bd42b8cf1dfce21da3802c596a59b1", size = 656067, upload-time = "2026-07-25T14:38:37.157Z" }, + { url = "https://files.pythonhosted.org/packages/31/27/0c6884785ab6afb41305296b2a5fac2c5d1d26dba40e85c70909b8692fce/hypothesis-6.161.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:35b41648b547a233dd89bed37a3ce8c8298a454fd7133b27b0a37ced135eddfe", size = 768668, upload-time = "2026-07-25T14:39:15.033Z" }, + { url = "https://files.pythonhosted.org/packages/7b/12/17c63bba85201d5fa9db89841e8548c4fa6b240b2437c80d606a65889eb6/hypothesis-6.161.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:429d2179b01787a54f8ccd150c02077a6feb973f14ef31664227148c6a1fadff", size = 760240, upload-time = "2026-07-25T14:38:43.93Z" }, + { url = "https://files.pythonhosted.org/packages/a2/df/1084fd695a1516120a5ea26c026d9f0a071fdf9efcba774c94c09332b372/hypothesis-6.161.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b23fc5441ef297d56797dd372889dec38ddf3662468772b347db2fd8fd43eced", size = 1090987, upload-time = "2026-07-25T14:39:06.646Z" }, + { url = "https://files.pythonhosted.org/packages/a0/a3/bef99daeaf6d1d2db09790cd4ec9ce0cdcac52faaf1fd80b4eaba5d5ea5e/hypothesis-6.161.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f42dcc7fee0d4d56d011218b8c4d5afa6cdee5dbc690652d2a1a598d21969a3", size = 1140613, upload-time = "2026-07-25T14:38:26.284Z" }, + { url = "https://files.pythonhosted.org/packages/9f/09/ce1f6e29d897c7dcb7de1665a9e30d1e00500933caec697a4338db9e7bd0/hypothesis-6.161.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2825fac0d0428cb5e7826347aa4e60106d28f32948cd58c837097f3bd96dbf6c", size = 1263802, upload-time = "2026-07-25T14:38:33.277Z" }, + { url = "https://files.pythonhosted.org/packages/81/e5/7b33aae590bc457fffba70c4f19cc150e3ca356f8e15322f9224c5372a64/hypothesis-6.161.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e9f859ecfd3e00ebb8a36fbb36a8951513fd8e09103a56c645b102e0a51dd1bc", size = 1307642, upload-time = "2026-07-25T14:39:29.729Z" }, + { url = "https://files.pythonhosted.org/packages/d4/8b/4156a9e70bf8c69f54954539ca805e7311048516b665c5f87bee2c1955b7/hypothesis-6.161.5-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:b2647c2d5da341467c1ffaad0dd86d8612e27da2419e76b3b6ee79a33bba7d86", size = 600146, upload-time = "2026-07-25T14:38:17.815Z" }, + { url = "https://files.pythonhosted.org/packages/f9/db/48a518f3f2facd7b280592f81dbb0ba09109656be4ad3dbf0d95a02b4c02/hypothesis-6.161.5-cp314-cp314-win_amd64.whl", hash = "sha256:7c40dd1a3e99497d48a3cfd4c5d71bdb0cd70402a9e09eceb160f045edc92b3a", size = 655981, upload-time = "2026-07-25T14:39:16.52Z" }, + { url = "https://files.pythonhosted.org/packages/25/92/1e9f9f44ef75fc052cc0e3700cfc7d52fd4af1ed9e8b2945d19956bb83cb/hypothesis-6.161.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:6c1b7e1d508a3cc5c10ea7a7a4a7d3b0b56fe6291e8936bb9d052af530380b15", size = 767251, upload-time = "2026-07-25T14:38:52.608Z" }, + { url = "https://files.pythonhosted.org/packages/48/31/0c3f824bbb1fccc0338930c8ef855880065d9d58dddf9547a1b9ae4e664d/hypothesis-6.161.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b0926ba51452c24b92fbcbb30d28cc43fd6cd9e636ea2134fbc1cc2c9da109de", size = 758710, upload-time = "2026-07-25T14:38:09.156Z" }, + { url = "https://files.pythonhosted.org/packages/ab/6b/5811dbb4c1dc0a772519fb9af9c4089128bceeb5deb4c36a887e1e731920/hypothesis-6.161.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba8d0346308b558bfbb49f965ed1d9d5bb65758f4530cb30ce1f9bd911a130b4", size = 1089583, upload-time = "2026-07-25T14:38:49.83Z" }, + { url = "https://files.pythonhosted.org/packages/5f/29/f9cbfd9d0aaea5370a7ceb966f6c8d47e9101ee9a6623606e9f564a59c6e/hypothesis-6.161.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c64079ef4633ab7b540f532107188d75999ee5b342116636312e09ada15783e", size = 1139525, upload-time = "2026-07-25T14:38:29.262Z" }, + { url = "https://files.pythonhosted.org/packages/99/27/f56a03be4ee21f25828e8fba8a502d6840826aea4ff784ff54cbd1f51175/hypothesis-6.161.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ed3eede1c23689edb4773b4c2303ee91518e82056a0a4893fc8415b18e5c3e8b", size = 1261963, upload-time = "2026-07-25T14:38:31.931Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ea/72e11fe6832a68674271655be5cb6f035502dd328ea629b3ee510fb130a5/hypothesis-6.161.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:82b1088ebcad5ab6d146caf9a1552019472a8a9302727f1be5939e30460a101b", size = 1306382, upload-time = "2026-07-25T14:38:13.011Z" }, + { url = "https://files.pythonhosted.org/packages/df/8b/bb039d11a805db0977904c245fa3dcf6d266808482d9dfc6dad3e5f1b256/hypothesis-6.161.5-cp314-cp314t-win_amd64.whl", hash = "sha256:bbe8705ff394a573624cd53f2192dad6255c86346704adf9873cb3acb9b940e5", size = 656131, upload-time = "2026-07-25T14:38:16.712Z" }, + { url = "https://files.pythonhosted.org/packages/21/71/0b28e9ac10f692d9b85a0df74615d308ec1e77140c61773ab107070b82d3/hypothesis-6.161.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:64167d94bcd4a15c1b0aa44c3176f019614898ff10907a2065c142ed34acf828", size = 768375, upload-time = "2026-07-25T14:39:23Z" }, + { url = "https://files.pythonhosted.org/packages/6d/5c/f518bdc85ee5a9b9153aa01b3ed6a77304b8f573a461da1a6694a6992001/hypothesis-6.161.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:22bc641a290428cfd01c62b2b7fe7686007b104dfb5e8ed5420fd3d3a281ebb9", size = 764276, upload-time = "2026-07-25T14:39:02.167Z" }, + { url = "https://files.pythonhosted.org/packages/af/3b/6c93b03adb5804b01854d2801a1f74705ea73d685cef7861484ad8804856/hypothesis-6.161.5-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35527a795992396b82293fec2294ba5b9c0768350c8cb89085e3f5f76ed7d08e", size = 1093089, upload-time = "2026-07-25T14:39:11.442Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3d/6ebfbd76534a71c7ffdd48e0d42373a14152e45714feabf0c25bd7d6b10a/hypothesis-6.161.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ae81035628d2b80435320c43cb09f5b421bac4941bd8ea9b2778d278ae65996", size = 1142876, upload-time = "2026-07-25T14:39:24.674Z" }, + { url = "https://files.pythonhosted.org/packages/fd/07/b57cfd34f55c14a2e823284a796d4228c356e338e1195affc0546bca567b/hypothesis-6.161.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:20e3ab3b0dc300a84b58f41ddd7f5190757521282aad58701c4c76f28a3d981a", size = 659768, upload-time = "2026-07-25T14:39:08.166Z" }, +] + [[package]] name = "identify" version = "2.6.12" From f911ce7589d7e748725c8a5ff3639a4cb0c5cea9 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 10:01:35 +0200 Subject: [PATCH 51/91] refactor: name the main_thread_only deadlock-admission timeout Extract MAIN_THREAD_ONLY_ADMIT_TIMEOUT (unchanged, 1s) so the deadlock window is documented in one place. It stays small: a genuine second concurrent remote_exec never returns, so this only bounds how fast that is reported. main_thread_only is an xdist-only transitional profile; the whole guard goes away with it. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/execnet/_trio_worker.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index d8c6cc0e..4f3a65f2 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -59,13 +59,23 @@ def trigger_shutdown(self) -> None: pass +#: How long admission waits for the previous main-thread exec to finish +#: before declaring a deadlock. Kept small: a genuine second concurrent +#: remote_exec never returns, so this only affects how fast that is +#: reported. (Under heavy CPU contention a merely slow predecessor can be +#: misread as a deadlock, but main_thread_only is an xdist-only transitional +#: profile slated for removal, so the whole guard goes away with it.) +MAIN_THREAD_ONLY_ADMIT_TIMEOUT = 1.0 + + class MainExec: """Exec strategy: serialize each request onto the process main thread. The ``main_thread_only`` profile (GUI/signal-safe: pytest under xdist runs this way). Admission waits for the previous request to finish and - closes the channel with the deadlock text when it cannot within a - second (a second concurrent remote_exec would deadlock the requester). + closes the channel with the deadlock text when it cannot within + ``MAIN_THREAD_ONLY_ADMIT_TIMEOUT`` (a second concurrent remote_exec would + deadlock the requester). """ needs_primary_thread = True @@ -79,7 +89,9 @@ def __init__(self, gateway: WorkerGateway) -> None: async def admit(self, channel: Channel, item: ExecItem) -> bool: complete = self.gateway._executetask_complete assert complete is not None - wait_slot = functools.partial(complete.wait, timeout=1) + wait_slot = functools.partial( + complete.wait, timeout=MAIN_THREAD_ONLY_ADMIT_TIMEOUT + ) if not await trio.to_thread.run_sync(wait_slot, abandon_on_cancel=True): channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) return False From 8eee047f5e4b87f0dbce30a9c63d555bb4189804 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 14:40:29 +0200 Subject: [PATCH 52/91] feat!: hide the standalone serializer from the public API dumps/loads/dump/load are no longer re-exported on any public namespace (execnet, execnet.sync, execnet.trio, execnet.aio); they remain internal to execnet.gateway_base. A channel carries only simple builtin data (plus channel references) -- encoding rich objects is the caller's job, and execnet deliberately does not provide or manage a codec. The serializer error types (DataFormatError / DumpError / LoadError) stay exposed on every surface for `except` and documentation, and now spell out that hitting one is a caller error to resolve (encode the value to simple data first), pointing at pydantic model_dump/model_validate and pytest's report to/from-serializable hooks as the mechanisms to use. Docs: the stale "Cross-interpreter serialization" section (which documented the removed execnet.dumps, with an autofunction + doctest that would now break the build) becomes "Sending objects over a channel" guidance; the dumps/loads anchors are kept so existing refs resolve. BREAKING CHANGE: execnet.dumps, execnet.loads, execnet.dump and execnet.load are removed from the public API (use your own encoding, e.g. pydantic or the pytest report hooks). Shipping with the trio backend as a major release. Co-Authored-By: Claude Opus 4.8 (1M context) --- doc/basics.rst | 56 ++++++++++++++++++++++++++++--------- doc/index.rst | 8 +++--- src/execnet/__init__.py | 8 ------ src/execnet/aio.py | 14 +++------- src/execnet/gateway_base.py | 33 ++++++++++++++++++++-- src/execnet/sync.py | 8 ------ src/execnet/trio.py | 14 +++------- testing/test_basics.py | 24 ++++++++++------ testing/test_namespaces.py | 5 ++-- testing/test_serializer.py | 2 +- 10 files changed, 105 insertions(+), 67 deletions(-) diff --git a/doc/basics.rst b/doc/basics.rst index a6b256b7..94f43ad8 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -216,23 +216,53 @@ configure a tracing mechanism: .. _`dumps/loads`: .. _`dumps/loads API`: +.. _`serialization`: -Cross-interpreter serialization of Python objects +Sending objects over a channel ======================================================= -.. versionadded:: 1.1 +A channel carries only **simple builtin data**: ``None``, ``bool``, +``int``, ``float``, ``complex``, ``bytes``, ``str`` and arbitrarily nested +``list`` / ``tuple`` / ``set`` / ``frozenset`` / ``dict`` of those -- plus +**channel references**, which arrive as channels on the peer. That is the +entire contract. -Execnet exposes a function pair which you can safely use to -store and load values from different Python interpreters -(e.g. Python2 and Python3, PyPy and Jython). Here is -a basic example:: +execnet does **not** pickle and does **not** encode rich objects for you: +arbitrary instances, functions, ``datetime``, dataclasses, pydantic models, +numpy arrays, enums, etc. have no wire representation. This is deliberate; +encoded / rich-object channels are out of scope for execnet. - >>> import execnet - >>> dump = execnet.dumps([1,2,3]) - >>> execnet.loads(dump) - [1,2,3] +Sending an unsupported value raises ``DumpError`` (a subclass of +``DataFormatError``); a corrupt or protocol-mismatched payload on receive +raises ``LoadError``. These signal a **caller error to resolve** -- reduce +the value to simple data before sending -- not a transport failure. The +standalone serializer itself is an internal implementation detail +(``execnet.gateway_base``) and is not part of the public API. -For more examples see :ref:`dumps/loads examples`. +Encode rich objects yourself +------------------------------------------------------- -.. autofunction:: execnet.dumps(spec) -.. autofunction:: execnet.loads(spec) +Turning a rich object into simple data (and back) is the caller's job. Use +an established encoding mechanism rather than expecting the channel to do it: + +- **pydantic**: ``model.model_dump(mode="json")`` reduces a model to simple + data (``datetime`` -> ISO string, ``UUID`` / ``Enum`` / ``Decimal`` -> + primitives); ``Model.model_validate(...)`` rebuilds it on the other side. + ``TypeAdapter`` covers non-model types. + + :: + + channel.send(model.model_dump(mode="json")) + # peer: + model = MyModel.model_validate(channel.receive()) + +- **pytest** does exactly this above execnet: pytest-xdist ships + ``TestReport`` objects with the ``pytest_report_to_serializable`` / + ``pytest_report_from_serializable`` hooks (rich report <-> simple dict) + around ``channel.send`` / ``channel.receive``. + +- **stdlib**: ``dataclasses.asdict(obj)``, ``dt.isoformat()`` / + ``datetime.fromisoformat``, or ``json`` with a ``default=`` hook. + +Channels are the one non-builtin you *can* send: nested channel references +pass through intact, so callbacks and sub-streams need no encoding. diff --git a/doc/index.rst b/doc/index.rst index 19a20eb8..ce161b47 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -31,10 +31,10 @@ Features * Automatic bootstrapping: no manual remote installation. * Safe and simple serialization of Python builtin - types for sending/receiving structured data messages. - (New in 1.1) execnet offers a new :ref:`dumps/loads ` - API which allows cross-interpreter compatible serialization - of Python builtin types. + types for sending/receiving structured data messages; + see :ref:`sending objects over a channel `. + Encoding rich objects is the caller's job (execnet stays + builtin-types-only). * Flexible communication: synchronous send/receive as well as callback/queue mechanisms supported diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 66c61821..b81aaedd 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -34,10 +34,6 @@ from .sync import TimeoutError from .sync import XSpec from .sync import default_group -from .sync import dump -from .sync import dumps -from .sync import load -from .sync import loads from .sync import makegateway from .sync import set_execmodel @@ -56,10 +52,6 @@ "XSpec", "__version__", "default_group", - "dump", - "dumps", - "load", - "loads", "makegateway", "set_execmodel", ] diff --git a/src/execnet/aio.py b/src/execnet/aio.py index bbfe06e3..7151ab5d 100644 --- a/src/execnet/aio.py +++ b/src/execnet/aio.py @@ -23,8 +23,10 @@ async def main(): side only -- the host-side task runs to completion (a cancelled ``receive`` may still consume the next item). -The serialization helpers and error types are shared with -:mod:`execnet.sync` and :mod:`execnet.trio`. +The error types are shared with :mod:`execnet.sync` and +:mod:`execnet.trio`. Items you send must already be simple builtin data +(plus channels); the standalone serializer is intentionally not part of +the public API -- see ``DumpError``. """ from __future__ import annotations @@ -53,10 +55,6 @@ async def main(): from .gateway_base import LoadError from .gateway_base import RemoteError from .gateway_base import TimeoutError -from .gateway_base import dump -from .gateway_base import dumps -from .gateway_base import load -from .gateway_base import loads from .xspec import XSpec if TYPE_CHECKING: @@ -73,10 +71,6 @@ async def main(): "RemoteError", "TimeoutError", "XSpec", - "dump", - "dumps", - "load", - "loads", "open_popen_gateway", ] diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index c59f4437..75e5895a 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1033,15 +1033,42 @@ def executetask( class DataFormatError(Exception): - pass + """A value could not cross the channel in execnet's simple wire format. + + execnet only moves *simple* builtin data over a channel -- ``None``, + ``bool``, ``int``, ``float``, ``complex``, ``bytes``, ``str``, and + (arbitrarily nested) ``list``/``tuple``/``set``/``frozenset``/``dict`` of + those -- plus channel references, which pass through as channels. It does + **not** pickle: arbitrary instances, functions, ``datetime``, dataclasses, + pydantic models, numpy arrays, etc. have no wire representation. + + A ``DataFormatError`` therefore signals a caller error to resolve, not a + transport failure: reduce the value to simple data before sending (and + reconstruct it after receiving) with an encoding mechanism of your own -- + e.g. pydantic ``model_dump`` / ``model_validate`` or pytest's + ``pytest_report_to_serializable`` / ``pytest_report_from_serializable`` + hooks. See the docs, "Sending objects over a channel". + """ class DumpError(DataFormatError): - """Error while serializing an object.""" + """A value being **sent** is not simple wire data; convert it first. + + Raised by ``channel.send`` (and the internal serializer) when an object is + not one of execnet's simple wire types. Fix it at the call site by turning + the rich object into simple data -- e.g. ``dt.isoformat()``, + ``dataclasses.asdict(obj)``, ``model.model_dump(mode="json")`` -- rather + than expecting the channel to pickle it. Channels are the one non-builtin + that *is* sendable, so nested channel references are fine. + """ class LoadError(DataFormatError): - """Error while unserializing an object.""" + """Received bytes could not be turned back into an object. + + Raised while **receiving** (deserializing) -- a corrupted or + protocol-incompatible payload, or data produced by a mismatched peer. + """ def bchr(n: int) -> bytes: diff --git a/src/execnet/sync.py b/src/execnet/sync.py index 0d3c835f..e5e026c7 100644 --- a/src/execnet/sync.py +++ b/src/execnet/sync.py @@ -14,10 +14,6 @@ from .gateway_base import LoadError from .gateway_base import RemoteError from .gateway_base import TimeoutError -from .gateway_base import dump -from .gateway_base import dumps -from .gateway_base import load -from .gateway_base import loads from .multi import Group from .multi import MultiChannel from .multi import default_group @@ -40,10 +36,6 @@ "TimeoutError", "XSpec", "default_group", - "dump", - "dumps", - "load", - "loads", "makegateway", "set_execmodel", ] diff --git a/src/execnet/trio.py b/src/execnet/trio.py index df0ab493..4b712c0a 100644 --- a/src/execnet/trio.py +++ b/src/execnet/trio.py @@ -14,8 +14,10 @@ async def main(): trio.run(main) -The serialization helpers and error types are shared with the blocking -API in :mod:`execnet.sync`. +The error types are shared with the blocking API in :mod:`execnet.sync`. +Items you send must already be simple builtin data (plus channels); the +standalone serializer is intentionally not part of the public API -- see +``DumpError``. """ from ._trio_gateway import AsyncChannel @@ -32,10 +34,6 @@ async def main(): from .gateway_base import LoadError from .gateway_base import RemoteError from .gateway_base import TimeoutError -from .gateway_base import dump -from .gateway_base import dumps -from .gateway_base import load -from .gateway_base import loads from .xspec import XSpec __all__ = [ @@ -52,10 +50,6 @@ async def main(): "RemoteError", "TimeoutError", "XSpec", - "dump", - "dumps", - "load", - "loads", "open_popen_gateway", "serve_gateway", ] diff --git a/testing/test_basics.py b/testing/test_basics.py index 470bec6d..d5606610 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -28,37 +28,45 @@ ) +# The standalone serializer is an internal detail (execnet.gateway_base), +# not part of the public API -- see the docs, "Sending objects over a channel". @pytest.mark.parametrize("val", ["123", 42, [1, 2, 3], ["23", 25]]) class TestSerializeAPI: def test_serializer_api(self, val: object) -> None: - dumped = execnet.dumps(val) - val2 = execnet.loads(dumped) + dumped = gateway_base.dumps(val) + val2 = gateway_base.loads(dumped) assert val == val2 def test_mmap(self, tmp_path: Path, val: object) -> None: mmap = pytest.importorskip("mmap").mmap p = tmp_path / "data.bin" - p.write_bytes(execnet.dumps(val)) + p.write_bytes(gateway_base.dumps(val)) with p.open("r+b") as f: m = mmap(f.fileno(), 0) - val2 = execnet.load(m) + val2 = gateway_base.load(m) assert val == val2 def test_bytesio(self, val: object) -> None: f = BytesIO() - execnet.dump(f, val) + gateway_base.dump(f, val) read = BytesIO(f.getvalue()) - val2 = execnet.load(read) + val2 = gateway_base.load(read) assert val == val2 +def test_serializer_not_public() -> None: + # dumps/loads/dump/load are internal to gateway_base only. + for name in ("dumps", "loads", "dump", "load"): + assert not hasattr(execnet, name), name + + def test_serializer_api_version_error(monkeypatch: pytest.MonkeyPatch) -> None: bchr = gateway_base.bchr monkeypatch.setattr(gateway_base, "DUMPFORMAT_VERSION", bchr(1)) - dumped = execnet.dumps(42) + dumped = gateway_base.dumps(42) monkeypatch.setattr(gateway_base, "DUMPFORMAT_VERSION", bchr(2)) - pytest.raises(execnet.DataFormatError, lambda: execnet.loads(dumped)) + pytest.raises(execnet.DataFormatError, lambda: gateway_base.loads(dumped)) def test_errors_on_execnet() -> None: diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index a8bfef0c..0317bc2b 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -32,9 +32,10 @@ def test_trio_namespace_exposes_async_core() -> None: assert execnet.trio.AsyncGateway is _trio_gateway.AsyncGateway assert execnet.trio.AsyncChannel is _trio_gateway.AsyncChannel assert execnet.trio.open_popen_gateway is _trio_gateway.open_popen_gateway - # serialization + errors are shared with the sync surface + # error types are shared with the sync surface; the standalone serializer + # is intentionally not exposed on any public namespace assert execnet.trio.RemoteError is execnet.RemoteError - assert execnet.trio.dumps is execnet.dumps + assert not hasattr(execnet.trio, "dumps") def test_portal_namespace() -> None: diff --git a/testing/test_serializer.py b/testing/test_serializer.py index 87adf04c..49332c97 100644 --- a/testing/test_serializer.py +++ b/testing/test_serializer.py @@ -189,4 +189,4 @@ def test_tuple_nested_with_empty_in_between(dump, load) -> None: def test_py2_string_loads() -> None: """Regression test for #267.""" - assert execnet.loads(b"\x02M\x00\x00\x00\x01aQ") == b"a" + assert execnet.gateway_base.loads(b"\x02M\x00\x00\x00\x01aQ") == b"a" From 6a3371673d492d7a17cb835cd6c29105e6295067 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 14:40:29 +0200 Subject: [PATCH 53/91] test: type-check the callback stress tests under mypy Add hypothesis to the mypy pre-commit hook's dependencies so @given is typed, and pass Hypothesis settings.register_profile keywords explicitly (a homogeneous **dict tripped the arg-type check). Also a ruff-format tidy. Co-Authored-By: Claude Opus 4.8 (1M context) --- .pre-commit-config.yaml | 1 + testing/conftest.py | 17 +++++++++-------- testing/test_channel_stress.py | 4 +--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f7a9f86c..91db1f29 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,4 +32,5 @@ repos: additional_dependencies: - pytest - trio + - hypothesis - types-pywin32 diff --git a/testing/conftest.py b/testing/conftest.py index becf10f2..04dbe631 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -92,17 +92,18 @@ def pytest_configure(config: pytest.Config) -> None: from hypothesis import settings except ImportError: return - common = dict( - deadline=None, - suppress_health_check=[ - HealthCheck.function_scoped_fixture, - HealthCheck.too_slow, - ], + suppress = [HealthCheck.function_scoped_fixture, HealthCheck.too_slow] + settings.register_profile( + "execnet-quick", max_examples=15, deadline=None, suppress_health_check=suppress ) - settings.register_profile("execnet-quick", max_examples=15, **common) stress = config.getoption("stress") if stress is not None: - settings.register_profile("execnet-stress", max_examples=int(stress), **common) + settings.register_profile( + "execnet-stress", + max_examples=int(stress), + deadline=None, + suppress_health_check=suppress, + ) settings.load_profile("execnet-stress") else: settings.load_profile("execnet-quick") diff --git a/testing/test_channel_stress.py b/testing/test_channel_stress.py index 8090367d..12f69d47 100644 --- a/testing/test_channel_stress.py +++ b/testing/test_channel_stress.py @@ -55,9 +55,7 @@ class TestCallbackStress: gwtype = "popen" @given(data=items_strategy) - def test_callback_receives_all_in_order( - self, gw: Gateway, data: list[int] - ) -> None: + def test_callback_receives_all_in_order(self, gw: Gateway, data: list[int]) -> None: collected: list[int] = [] channel = gw.remote_exec(_echo, items=data) channel.setcallback(collected.append) From 12fdbbec13564e6576f4c8ab4f284b0b90aea983 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 14:57:26 +0200 Subject: [PATCH 54/91] refactor!: drop the injected remote shell script execnet.script.shell connected to a socket server, sent repr() of its own source as the first line, and expected the server to exec it with clientsock bound in globals. That handshake belonged to the pre-Trio socket server; the Trio one speaks the execnet frame protocol from the first byte and never execs an incoming line, so the module has been inert since that rewrite. Nothing imports it, no test covers it, and a remote prompt is better served as a remote_exec example than as a socket-protocol side channel. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 3 ++ src/execnet/script/shell.py | 91 ------------------------------------- 2 files changed, 3 insertions(+), 91 deletions(-) delete mode 100644 src/execnet/script/shell.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index eac3c8d6..3e99f1d5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,9 @@ * Trio host-thread Message IO for local ``popen`` + import bootstrap (coordinator and worker). Adds a hard ``trio`` dependency. Disable with ``EXECNET_TRIO_HOST=0``. Other gateway types and greenlet execmodels keep the legacy thread path. +* Removed ``execnet.script.shell``, an interactive remote prompt that injected its own + source into the pre-Trio socket server. The Trio socket server never execs an incoming + source line, so the module could no longer work against it. 2.1.2 (2025-11-11) diff --git a/src/execnet/script/shell.py b/src/execnet/script/shell.py deleted file mode 100644 index de6f8ded..00000000 --- a/src/execnet/script/shell.py +++ /dev/null @@ -1,91 +0,0 @@ -#! /usr/bin/env python -""" -a remote python shell - -for injection into startserver.py -""" - -import os -import select -import socket -import sys -from threading import Thread -from traceback import print_exc -from typing import NoReturn - - -def clientside() -> NoReturn: - print("client side starting") - host, portstr = sys.argv[1].split(":") - port = int(portstr) - with open(os.path.abspath(sys.argv[0])) as fd: - myself = fd.read() - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((host, port)) - sock.sendall((repr(myself) + "\n").encode()) - print("send boot string") - inputlist = [sock, sys.stdin] - try: - while 1: - r, _w, _e = select.select(inputlist, [], []) - if sys.stdin in r: - line = input() - sock.sendall((line + "\n").encode()) - if sock in r: - line = sock.recv(4096).decode() - sys.stdout.write(line) - sys.stdout.flush() - except BaseException: - import traceback - - traceback.print_exc() - - sys.exit(1) - - -class promptagent(Thread): - def __init__(self, clientsock) -> None: - print("server side starting") - super().__init__() # type: ignore[call-overload] - self.clientsock = clientsock - - def run(self) -> None: - print("Entering thread prompt loop") - clientfile = self.clientsock.makefile("w") - - filein = self.clientsock.makefile("r") - loc = self.clientsock.getsockname() - - while 1: - try: - clientfile.write("{} {} >>> ".format(*loc)) - clientfile.flush() - line = filein.readline() - if not line: - raise EOFError("nothing") - if line.strip(): - oldout, olderr = sys.stdout, sys.stderr - sys.stdout, sys.stderr = clientfile, clientfile - try: - try: - exec(compile(line + "\n", "", "single")) - except BaseException: - print_exc() - finally: - sys.stdout = oldout - sys.stderr = olderr - clientfile.flush() - except EOFError: - sys.stderr.write("connection close, prompt thread returns") - break - - self.clientsock.close() - - -sock = globals().get("clientsock") -if sock is not None: - prompter = promptagent(sock) - prompter.start() - print("promptagent - thread started") -else: - clientside() From 6718808a37ff099cbe5f57739f3d1aec6deac821 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 14:58:56 +0200 Subject: [PATCH 55/91] refactor!: drop the quitserver script execnet.script.quitserver stopped a socket server by connecting and sending '"raise KeyboardInterrupt"' for the server to exec. Like the shell script it depended on the pre-Trio exec-a-source-line handshake; against the Trio socket server it is just a malformed first frame. Stopping the server is now the job of whatever supervises the execnet-socketserver process. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 2 ++ src/execnet/script/quitserver.py | 17 ----------------- 2 files changed, 2 insertions(+), 17 deletions(-) delete mode 100644 src/execnet/script/quitserver.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3e99f1d5..1ac8a0ef 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,8 @@ * Removed ``execnet.script.shell``, an interactive remote prompt that injected its own source into the pre-Trio socket server. The Trio socket server never execs an incoming source line, so the module could no longer work against it. +* Removed ``execnet.script.quitserver``, which shut a socket server down by sending it + ``"raise KeyboardInterrupt"`` to exec. It relied on the same removed handshake. 2.1.2 (2025-11-11) diff --git a/src/execnet/script/quitserver.py b/src/execnet/script/quitserver.py deleted file mode 100644 index 4c94c383..00000000 --- a/src/execnet/script/quitserver.py +++ /dev/null @@ -1,17 +0,0 @@ -""" - -send a "quit" signal to a remote server - -""" - -from __future__ import annotations - -import socket -import sys - -host, port = sys.argv[1].split(":") -hostport = (host, int(port)) - -sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -sock.connect(hostport) -sock.sendall(b'"raise KeyboardInterrupt"\n') From 7124fc0cfeb51a65bc0d5454296be1486cd46a4c Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 15:00:37 +0200 Subject: [PATCH 56/91] refactor!: drop the socketserver restart loop script loop_socketserver re-Popen'd "python /socketserver.py" in a while loop, from the days when a single connection ended the server process. The Trio socket server accepts connections in a loop via trio.serve_listeners and only exits after one connection when asked with --once, so the wrapper has nothing left to restart. Its sibling-file lookup also only ever worked from a source checkout, not from an installed execnet. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 4 ++++ src/execnet/script/loop_socketserver.py | 14 -------------- 2 files changed, 4 insertions(+), 14 deletions(-) delete mode 100644 src/execnet/script/loop_socketserver.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1ac8a0ef..7bde0c0f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,10 @@ source line, so the module could no longer work against it. * Removed ``execnet.script.quitserver``, which shut a socket server down by sending it ``"raise KeyboardInterrupt"`` to exec. It relied on the same removed handshake. +* Removed ``execnet.script.loop_socketserver``, a restart loop around a sibling + ``socketserver.py`` file. The socket server serves connections in a loop itself + (``--once`` opts out), and the sibling-file path never resolved for an installed + execnet anyway. 2.1.2 (2025-11-11) diff --git a/src/execnet/script/loop_socketserver.py b/src/execnet/script/loop_socketserver.py deleted file mode 100644 index a4688a80..00000000 --- a/src/execnet/script/loop_socketserver.py +++ /dev/null @@ -1,14 +0,0 @@ -import os -import subprocess -import sys - -if __name__ == "__main__": - directory = os.path.dirname(os.path.abspath(sys.argv[0])) - script = os.path.join(directory, "socketserver.py") - while 1: - cmdlist = ["python", script] - cmdlist.extend(sys.argv[1:]) - text = "starting subcommand: " + " ".join(cmdlist) - print(text) - process = subprocess.Popen(cmdlist) - process.wait() From 47e5b78ead6be7c9d96c9649747c18254f0cb2c4 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 15:05:36 +0200 Subject: [PATCH 57/91] refactor!: drop the pywin32 windows service wrapper socketserverservice.py registered the socket server as a Windows service via pywin32. pywin32 was never a dependency, nothing imported the module, no test ever ran it, and no entry point installed it -- it was a recipe living in the package. Now that the server is a real console command, any service host can supervise it. Document that instead: an NSSM (or WinSW) wrapper on Windows, a systemd unit on Linux, plus why sc.exe alone will not start a plain console program. While rewriting that section, replace the stale instruction to download socketserver.py from raw.githubusercontent and run it standalone. The Trio server imports execnet and trio, so it is no longer a version-independent single file; "uvx --from execnet execnet-socketserver" is the install-free equivalent. Also document --once, port 0, and that the port is unauthenticated. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 3 + doc/example/test_info.rst | 65 ++++++++++++++-- src/execnet/script/socketserverservice.py | 90 ----------------------- 3 files changed, 61 insertions(+), 97 deletions(-) delete mode 100644 src/execnet/script/socketserverservice.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7bde0c0f..0b66b38a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,9 @@ ``socketserver.py`` file. The socket server serves connections in a loop itself (``--once`` opts out), and the sibling-file path never resolved for an installed execnet anyway. +* Removed ``execnet.script.socketserverservice``, the pywin32 Windows service wrapper. + Wrap the ``execnet-socketserver`` console command with a service host such as NSSM + instead; the socket gateway example documents how. 2.1.2 (2025-11-11) diff --git a/doc/example/test_info.rst b/doc/example/test_info.rst index eefe91b3..7a5ead4f 100644 --- a/doc/example/test_info.rst +++ b/doc/example/test_info.rst @@ -172,15 +172,23 @@ sends back the results. Instantiate gateways through sockets ----------------------------------------------------- -.. _`socketserver.py`: https://raw.githubusercontent.com/pytest-dev/execnet/main/src/execnet/script/socketserver.py +In cases where you do not have SSH-access to a machine you need a +bootstrapping-point that listens on a socket. execnet ships one as the +``execnet-socketserver`` console command; run it on the target machine:: -In cases where you do not have SSH-access to a machine -you need to download a small version-independent standalone -`socketserver.py`_ script to provide a remote bootstrapping-point. -You do not need to install the execnet package remotely. -Simply run the script like this:: + execnet-socketserver :8888 # bind to all IPs, port 8888 - python socketserver.py :8888 # bind to all IPs, port 8888 +If execnet is not installed there, uv_ can fetch and run it in one step +without leaving anything behind:: + + uvx --from execnet execnet-socketserver :8888 + +.. _uv: https://docs.astral.sh/uv/ + +The server accepts connections in a loop and serves each one in its own +worker subprocess; pass ``--once`` to serve a single connection and exit. +Passing port ``0`` binds an ephemeral port -- the bound address is printed +on the first line of output either way. You can then instruct execnet on your local machine to bootstrap itself into the remote socket endpoint:: @@ -191,4 +199,47 @@ itself into the remote socket endpoint:: That's it, you can now use the gateway object just like a popen- or SSH-based one. +.. warning:: + + The socket server performs no authentication and its traffic is not + encrypted -- anyone who can reach the port can execute code on that + machine. Bind it to a trusted network only, or tunnel it over SSH. + +Keeping the socket server running ++++++++++++++++++++++++++++++++++ + +``execnet-socketserver`` is an ordinary long-running process, so restarts and +boot-time startup are the job of your platform's service manager. + +On Linux, a systemd unit does it:: + + # /etc/systemd/system/execnet-socketserver.service + [Unit] + Description=execnet socket server + + [Service] + ExecStart=/usr/local/bin/execnet-socketserver :8888 + Restart=always + + [Install] + WantedBy=multi-user.target + +On Windows, wrap the console command with a service host such as NSSM_ or +WinSW_. Use ``where execnet-socketserver`` to find the installed +``execnet-socketserver.exe`` (it lives in the ``Scripts`` directory of the +environment it was installed into), then:: + + nssm install ExecNetSocketServer C:\path\to\Scripts\execnet-socketserver.exe :8888 + nssm set ExecNetSocketServer Start SERVICE_AUTO_START + net start ExecNetSocketServer + +NSSM restarts the process if it exits. Note that ``sc.exe create`` on its own +is not enough: it expects a binary that implements the Windows service control +protocol, which a plain console program does not. If you would rather not +install a service host, a Task Scheduler task with an *At startup* trigger +works too. + +.. _NSSM: https://nssm.cc/ +.. _WinSW: https://github.com/winsw/winsw + .. include:: test_ssh_fileserver.rst diff --git a/src/execnet/script/socketserverservice.py b/src/execnet/script/socketserverservice.py deleted file mode 100644 index 1c991307..00000000 --- a/src/execnet/script/socketserverservice.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -A windows service wrapper for the py.execnet socketserver. - -To use, run: - python socketserverservice.py register - net start ExecNetSocketServer -""" - -import sys -import threading - -import servicemanager -import win32event -import win32evtlogutil -import win32service -import win32serviceutil - -from . import socketserver - -appname = "ExecNetSocketServer" - - -class SocketServerService(win32serviceutil.ServiceFramework): - _svc_name_ = appname - _svc_display_name_ = "%s" % appname - _svc_deps_ = ["EventLog"] - - def __init__(self, args) -> None: - # The exe-file has messages for the Event Log Viewer. - # Register the exe-file as event source. - # - # Probably it would be better if this is done at installation time, - # so that it also could be removed if the service is uninstalled. - # Unfortunately it cannot be done in the 'if __name__ == "__main__"' - # block below, because the 'frozen' exe-file does not run this code. - # - win32evtlogutil.AddSourceToRegistry( - self._svc_display_name_, servicemanager.__file__, "Application" - ) - super().__init__(args) - self.hWaitStop = win32event.CreateEvent(None, 0, 0, None) - self.WAIT_TIME = 1000 # in milliseconds - - def SvcStop(self) -> None: - self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) - win32event.SetEvent(self.hWaitStop) - - def SvcDoRun(self) -> None: - # Redirect stdout and stderr to prevent "IOError: [Errno 9] - # Bad file descriptor". Windows services don't have functional - # output streams. - sys.stdout = sys.stderr = open("nul", "w") - - # Write a 'started' event to the event log... - win32evtlogutil.ReportEvent( - self._svc_display_name_, - servicemanager.PYS_SERVICE_STARTED, - 0, # category - servicemanager.EVENTLOG_INFORMATION_TYPE, - (self._svc_name_, ""), - ) - print("Begin: %s" % self._svc_display_name_) - - hostport = ":8888" - print("Starting py.execnet SocketServer on %s" % hostport) - thread = threading.Thread( - target=socketserver.main, args=([hostport],), daemon=True - ) - thread.start() - - # wait to be stopped or self.WAIT_TIME to pass - while True: - result = win32event.WaitForSingleObject(self.hWaitStop, self.WAIT_TIME) - if result == win32event.WAIT_OBJECT_0: - break - - # write a 'stopped' event to the event log. - win32evtlogutil.ReportEvent( - self._svc_display_name_, - servicemanager.PYS_SERVICE_STOPPED, - 0, # category - servicemanager.EVENTLOG_INFORMATION_TYPE, - (self._svc_name_, ""), - ) - print("End: %s" % appname) - - -if __name__ == "__main__": - # Note that this code will not be run in the 'frozen' exe-file!!! - win32serviceutil.HandleCommandLine(SocketServerService) From fa0d7a8b7b94f563372a0c51e393b808f8033c74 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 15:13:39 +0200 Subject: [PATCH 58/91] refactor: move the socket server to execnet._socketserver The Trio rewrite turned socketserver.py from a copy-me standalone script into ordinary package code: it imports trio and execnet._trio_host, and reaches users as the execnet-socketserver console command. Sitting in an execnet.script package still advertised the old download-and-run story. Move it next to the other private implementation modules (_trio_host, _trio_worker, _provision) and drop the execnet.script package, which is now empty. The console command and its behaviour are unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 3 +++ pyproject.toml | 2 +- src/execnet/{script/socketserver.py => _socketserver.py} | 6 ++++-- src/execnet/script/__init__.py | 1 - 4 files changed, 8 insertions(+), 4 deletions(-) rename src/execnet/{script/socketserver.py => _socketserver.py} (91%) delete mode 100644 src/execnet/script/__init__.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0b66b38a..3a1e9422 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,6 +17,9 @@ * Removed ``execnet.script.socketserverservice``, the pywin32 Windows service wrapper. Wrap the ``execnet-socketserver`` console command with a service host such as NSSM instead; the socket gateway example documents how. +* Moved the socket server to ``execnet._socketserver`` and removed the now empty + ``execnet.script`` package. The ``execnet-socketserver`` console command is unchanged + and remains the supported way to run it. 2.1.2 (2025-11-11) diff --git a/pyproject.toml b/pyproject.toml index f29b24d0..68029e90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ classifiers = [ ] [project.scripts] -execnet-socketserver = "execnet.script.socketserver:main" +execnet-socketserver = "execnet._socketserver:main" [project.optional-dependencies] gevent = [ diff --git a/src/execnet/script/socketserver.py b/src/execnet/_socketserver.py similarity index 91% rename from src/execnet/script/socketserver.py rename to src/execnet/_socketserver.py index eb2a86ae..c27609f6 100644 --- a/src/execnet/script/socketserver.py +++ b/src/execnet/_socketserver.py @@ -1,9 +1,11 @@ -#! /usr/bin/env python """Trio socket server for execnet gateways. Listens on a TCP port and hands each accepted connection (by fd) to a fresh ``python -m execnet._trio_worker`` subprocess that serves the gateway over it. -No code is executed inline. Run directly, e.g. provisioned anywhere with +No code is executed inline. + +This module implements the ``execnet-socketserver`` console command, which is +the supported entry point -- run it on the target host, or install-free with ``uvx --from execnet execnet-socketserver``. """ diff --git a/src/execnet/script/__init__.py b/src/execnet/script/__init__.py deleted file mode 100644 index 792d6005..00000000 --- a/src/execnet/script/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# From 871dfd730b3bd80407bdf271844359fed4f04a83 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 16:14:08 +0200 Subject: [PATCH 59/91] refactor!: drop the py2/py3 string coercion machinery reconfigure() configured how str/bytes cross a Python2 <-> Python3 gateway. execnet cannot have such a peer anymore, so remove it: * Gateway.reconfigure, Channel.reconfigure and AsyncChannel.reconfigure, plus the _strconfig attribute they fed. * the py2str_as_py3str / py3str_as_py2str arguments of gateway_base.loads and .load, and the strconfig argument of loads_internal. py2str_as_py3str was already unreachable on Python3 -- it only gates the PY2STRING opcode, which no Python3 serializer emits. * the retired PY2STRING and UNICODE opcodes; PY3STRING becomes STRING. Opcode bytes are unchanged for every surviving type, and M/S stay unassigned. Values dumped by execnet on Python2 no longer load. * the no-op reconfigure() call in rsync, which cost a round-trip per channel to set what was already the default. * doc/example/hybridpython.rst and py3topy2.py, along with the remaining Python2 references in basics.rst and index.rst. The RECONFIGURE message code stays reserved, but is no longer sent or handled. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 9 ++ doc/basics.rst | 8 +- doc/example/hybridpython.rst | 153 ---------------------- doc/example/py3topy2.py | 19 --- doc/examples.rst | 1 - doc/index.rst | 4 - src/execnet/_trio_gateway.py | 32 +---- src/execnet/_trio_host.py | 7 - src/execnet/gateway.py | 12 -- src/execnet/gateway_base.py | 89 ++----------- src/execnet/rsync.py | 1 - testing/test_compatibility_regressions.py | 6 +- testing/test_serializer.py | 7 +- testing/test_trio_gateway.py | 13 -- 14 files changed, 35 insertions(+), 326 deletions(-) delete mode 100644 doc/example/hybridpython.rst delete mode 100644 doc/example/py3topy2.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3a1e9422..9f2d3f38 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,15 @@ * Moved the socket server to ``execnet._socketserver`` and removed the now empty ``execnet.script`` package. The ``execnet-socketserver`` console command is unchanged and remains the supported way to run it. +* Removed ``Gateway.reconfigure`` and ``Channel.reconfigure`` along with the + ``py2str_as_py3str`` / ``py3str_as_py2str`` arguments of ``gateway_base.loads`` and + ``gateway_base.load``. They configured string coercion between Python2 and Python3 + peers, which execnet can no longer have: ``py2str_as_py3str`` was already unreachable + because only a Python2 serializer emits the opcode it gates. The ``RECONFIGURE`` + message code stays reserved but is no longer sent or handled. +* The serializer dropped the retired ``PY2STRING`` and ``UNICODE`` opcodes and renamed + ``PY3STRING`` to ``STRING``. Values dumped by execnet running on Python2 no longer + load. Opcode bytes are unchanged for every type that survives. 2.1.2 (2025-11-11) diff --git a/doc/basics.rst b/doc/basics.rst index 94f43ad8..a22ad7de 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -43,8 +43,8 @@ Examples for valid gateway specifications ``default`` via SSH through Vagrant's ``vagrant ssh`` command. It supports the same additional parameters as regular SSH connections. -* ``popen//python=python2.7//nice=20`` specification of - a python subprocess using the ``python2.7`` executable which must be +* ``popen//python=python3.13//nice=20`` specification of + a python subprocess using the ``python3.13`` executable which must be discoverable through the system ``PATH``; running with the lowest CPU priority ("nice" level). By default current dir will be the current dir of the instantiator. @@ -86,10 +86,6 @@ a channel object whose symmetric counterpart channel is available to the remotely executing source. -.. method:: Gateway.reconfigure([py2str_as_py3str=True, py3str_as_py2str=False]) - - Reconfigures the string-coercion behaviour of the gateway - .. _`Channel`: .. _`channel-api`: diff --git a/doc/example/hybridpython.rst b/doc/example/hybridpython.rst deleted file mode 100644 index f4ad465a..00000000 --- a/doc/example/hybridpython.rst +++ /dev/null @@ -1,153 +0,0 @@ -Connecting different Python interpreters -========================================== - -.. _`dumps/loads examples`: - -Dumping and loading values across interpreter versions ----------------------------------------------------------- - -.. versionadded:: 1.1 - -Execnet offers a new safe and fast :ref:`dumps/loads API` which you -can use to dump builtin python data structures and load them -later with the same or a different python interpreter (including -between Python2 and Python3). The standard library offers -the pickle and marshal modules but they do not work safely -between different interpreter versions. Using xml/json -requires a mapping of Python objects and is not easy to -get right. Moreover, execnet allows to control handling -of bytecode/strings/unicode types. Here is an example:: - - # using python2 - import execnet - with open("data.py23", "wb") as f: - f.write(execnet.dumps(["hello", "world"])) - - # using Python3 - import execnet - with open("data.py23", "rb") as f: - val = execnet.loads(f.read(), py2str_as_py3str=True) - assert val == ["hello", "world"] - -See the :ref:`dumps/loads API` for more details on string -conversion options. Please note, that you can not dump -user-level instances, only builtin python types. - -Connect to Python2/Numpy from Python3 ----------------------------------------- - -Here we run a Python3 interpreter to connect to a Python2.7 interpreter -that has numpy installed. We send items to be added to an array and -receive back the remote "repr" of the array:: - - import execnet - gw = execnet.makegateway("popen//python=python2.7") - channel = gw.remote_exec(""" - import numpy - array = numpy.array([1,2,3]) - while 1: - x = channel.receive() - if x is None: - break - array = numpy.append(array, x) - channel.send(repr(array)) - """) - for x in range(10): - channel.send(x) - channel.send(None) - print (channel.receive()) - -will print on the CPython3.1 side:: - - array([1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) - -A more refined real-life example of python3/python2 interaction -is the anyvc_ project which uses version-control bindings in -a Python2 subprocess in order to offer Python3-based library -functionality. - -.. _anyvc: http://bitbucket.org/RonnyPfannschmidt/anyvc/overview/ - - -Reconfiguring the string coercion between python2 and python3 -------------------------------------------------------------- - -Sometimes the default configuration of string coercion (2str to 3str, 3str to 2unicode) -is inconvient, thus it can be reconfigured via `gw.reconfigure` and `channel.reconfigure`. Here is an example session on a Python2 interpreter:: - - - >>> import execnet - >>> execnet.makegateway("popen//python=python3.2") - - >>> gw=execnet.makegateway("popen//python=python3.2") - >>> gw.remote_exec("channel.send('hello')").receive() - u'hello' - >>> gw.reconfigure(py3str_as_py2str=True) - >>> gw.remote_exec("channel.send('hello')").receive() - 'hello' - >>> ch = gw.remote_exec('channel.send(type(channel.receive()).__name__)') - >>> ch.send('a') - >>> ch.receive() - 'str' - >>> ch = gw.remote_exec('channel.send(type(channel.receive()).__name__)') - >>> ch.reconfigure(py2str_as_py3str=False) - >>> ch.send('a') - >>> ch.receive() - u'bytes' - - -Work with Java objects from CPython ----------------------------------------- - -Use your CPython interpreter to connect to a `Jython 2.5.1`_ interpreter -and work with Java types:: - - import execnet - gw = execnet.makegateway("popen//python=jython") - channel = gw.remote_exec(""" - from java.util import Vector - v = Vector() - v.add('aaa') - v.add('bbb') - for val in v: - channel.send(val) - """) - - for item in channel: - print (item) - -will print on the CPython side:: - - aaa - bbb - -.. _`Jython 2.5.1`: http://www.jython.org - -Work with C# objects from CPython ----------------------------------------- - -(Experimental) use your CPython interpreter to connect to a IronPython_ interpreter -which can work with C# classes. Here is an example for instantiating -a CLR Array instance and sending back its representation:: - - import execnet - gw = execnet.makegateway("popen//python=ipy") - - channel = gw.remote_exec(""" - import clr - clr.AddReference("System") - from System import Array - array = Array[float]([1,2]) - channel.send(str(array)) - """) - print (channel.receive()) - -using Mono 2.0 and IronPython-1.1 this will print on the CPython side:: - - System.Double[](1.0, 2.0) - -.. note:: - Using IronPython needs more testing, likely newer versions - will work better. please feedback if you have information. - -.. _IronPython: http://ironpython.net diff --git a/doc/example/py3topy2.py b/doc/example/py3topy2.py deleted file mode 100644 index 3dcf0437..00000000 --- a/doc/example/py3topy2.py +++ /dev/null @@ -1,19 +0,0 @@ -import execnet - -gw = execnet.makegateway("popen//python=python2") -channel = gw.remote_exec( - """ - import numpy - array = numpy.array([1,2,3]) - while 1: - x = channel.receive() - if x is None: - break - array = numpy.append(array, x) - channel.send(repr(array)) -""" -) -for x in range(10): - channel.send(x) -channel.send(None) -print(channel.receive()) diff --git a/doc/examples.rst b/doc/examples.rst index 767fa4fc..3f09197a 100644 --- a/doc/examples.rst +++ b/doc/examples.rst @@ -14,7 +14,6 @@ Note: all examples with `>>>` prompts are automatically tested. example/test_group example/test_proxy example/test_multi - example/hybridpython example/test_debug .. toctree:: diff --git a/doc/index.rst b/doc/index.rst index ce161b47..47a918cc 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -59,9 +59,6 @@ Known uses project to launch computation processes through ssh. He also compares `disco and execnet`_ in a subsequent post. -* Ronny Pfannschmidt uses it for his `anyvc`_ VCS-abstraction project - to bridge the Python2/Python3 version gap. - * Sysadmins and developers are using it for ad-hoc custom scripting .. _`quora`: http://quora.com @@ -71,7 +68,6 @@ Known uses .. _`distributed testing`: https://pypi.python.org/pypi/pytest-xdist .. _`Distributed NTLK with execnet`: http://streamhacker.com/2009/11/29/distributed-nltk-execnet/ .. _`disco and execnet`: http://streamhacker.com/2009/12/14/execnet-disco-distributed-nltk/ -.. _`anyvc`: http://bitbucket.org/RonnyPfannschmidt/anyvc/ Project status -------------------------- diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 81ea76d0..3972db4f 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -8,7 +8,7 @@ Two-level channel model: * :class:`RawChannel` (this module) -- id-routed raw byte payload streams - over the gateway: no serialization, no strconfig, no callbacks. + over the gateway: no serialization and no callbacks. ``CHANNEL_DATA`` payloads route to the channel verbatim; the layer on top decides what the bytes mean. * ``AsyncChannel`` -- the serialized object API layered on a RawChannel. @@ -44,7 +44,6 @@ from .gateway_base import Message from .gateway_base import RemoteError from .gateway_base import TimeoutError -from .gateway_base import Unserializer from .gateway_base import dumps_internal from .gateway_base import gateway_info from .gateway_base import loads_internal @@ -149,8 +148,6 @@ class RawChannel: the peer drains, hits EOF, and may keep sending to us. """ - _strconfig: tuple[bool, bool] | None = None - def __init__(self, gateway: AsyncGateway, id: int) -> None: self.gateway = gateway self.id = id @@ -405,16 +402,6 @@ async def wait_closed(self) -> None: if error is not None: raise error - async def reconfigure( - self, py2str_as_py3str: bool = True, py3str_as_py2str: bool = False - ) -> None: - """Set the string coercion for both ends of this channel.""" - strconfig = (py2str_as_py3str, py3str_as_py2str) - self._raw._strconfig = strconfig - await self.gateway._send( - Message.RECONFIGURE, self.id, dumps_internal(strconfig) - ) - def __aiter__(self) -> AsyncChannel: return self @@ -424,12 +411,8 @@ async def __anext__(self) -> Any: except EOFError: raise StopAsyncIteration from None - # Unserializer duck-type: loads_internal(data, self) reads _strconfig - # and _channelfactory off the object to resolve CHANNEL opcodes. - - @property - def _strconfig(self) -> tuple[bool, bool]: - return self._raw._strconfig or self.gateway._strconfig + # Unserializer duck-type: loads_internal(data, self) reads + # _channelfactory off the object to resolve CHANNEL opcodes. @property def _channelfactory(self) -> _AsyncChannelFactory: @@ -475,7 +458,6 @@ def __init__(self, stream: ByteStream, *, id: str, _startcount: int = 1) -> None self._async_channels: dict[int, AsyncChannel] = {} self._channelfactory = _AsyncChannelFactory(self) self._count = _startcount - self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) self._outbound_send, self._outbound = trio.open_memory_channel[ tuple[bytes, Callable[[BaseException | None], None] | None] ](math.inf) @@ -693,14 +675,6 @@ def _dispatch(self, message: Message) -> None: Message.CHANNEL_DATA, channelid, dumps_internal(gateway_info()) ) self._send_nowait(Message.CHANNEL_CLOSE, channelid) - elif code == Message.RECONFIGURE: - data = loads_internal(message.data) - assert isinstance(data, tuple) - if channelid == 0: - self._strconfig = data - else: - # picked up by the serialized channel layer - self._channel_for(channelid)._strconfig = data else: # CHANNEL_EXEC / GATEWAY_START_*: not served by the async core self._trace("rejecting unsupported message", message) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index e119ca68..1d9149dd 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -204,13 +204,6 @@ def _dispatch(self, message: Message) -> None: elif code == Message.CHANNEL_EXEC: channel = gateway._channelfactory.new(message.channelid) gateway._local_schedulexec(channel=channel, sourcetask=message.data) - elif code == Message.RECONFIGURE: - data = loads_internal(message.data, gateway) - assert isinstance(data, tuple) - if message.channelid == 0: - gateway._strconfig = data - else: - gateway._channelfactory.new(message.channelid)._strconfig = data elif code == Message.GATEWAY_START_SOCKET: handle_start_socket(gateway, message.channelid, message.data) elif code == Message.GATEWAY_START_SUB: diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index 672a0830..ac41705c 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -80,18 +80,6 @@ def exit(self) -> None: self._trace("io-error: could not send termination sequence") self._trace(" exception: %r" % exc) - def reconfigure( - self, py2str_as_py3str: bool = True, py3str_as_py2str: bool = False - ) -> None: - """Set the string coercion for this gateway. - - The default is to try to convert py2 str as py3 str, but not to try and - convert py3 str to py2 str. - """ - self._strconfig = (py2str_as_py3str, py3str_as_py2str) - data = gateway_base.dumps_internal(self._strconfig) - self._send(Message.RECONFIGURE, data=data) - def _rinfo(self, update: bool = False) -> RInfo: """Return some sys/env information from remote. diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index 75e5895a..ce19eb0f 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -186,6 +186,8 @@ class only carries the framing and the code constants. """ STATUS = 0 + #: retired: the py2/py3 string coercion switch. Nothing sends or + #: handles it anymore, the code stays reserved for reuse. RECONFIGURE = 1 GATEWAY_TERMINATE = 2 CHANNEL_EXEC = 3 @@ -393,8 +395,6 @@ def __init__(self, gateway: BaseGateway, id: int) -> None: assert isinstance(id, int) assert not isinstance(gateway, type) self.gateway = gateway - # XXX: defaults copied from Unserializer - self._strconfig = getattr(gateway, "_strconfig", (True, False)) self.id = id # serialized payloads (or ENDMARKER); None once a consumer is attached self._mailbox: Mailbox[Any] | None = Mailbox(gateway._new_wakener()) @@ -672,18 +672,6 @@ def next(self) -> Any: __next__ = next - def reconfigure( - self, py2str_as_py3str: bool = True, py3str_as_py2str: bool = False - ) -> None: - """Set the string coercion for this channel. - - The default is to try to convert py2 str as py3 str, - but not to try and convert py3 str to py2 str - """ - self._strconfig = (py2str_as_py3str, py3str_as_py2str) - data = dumps_internal(self._strconfig) - self.gateway._send(Message.RECONFIGURE, self.id, data=data) - ENDMARKER = object() INTERRUPT_TEXT = "keyboard-interrupted" @@ -838,7 +826,6 @@ def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel self._io = io self.id = id - self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) self._channelfactory = ChannelFactory(self, _startcount) # globals may be NONE at process-termination self.__trace = trace @@ -1106,35 +1093,26 @@ class opcode: NEWDICT = b"J" NEWLIST = b"K" NONE = b"L" - PY2STRING = b"M" - PY3STRING = b"N" + STRING = b"N" SET = b"O" SETITEM = b"P" STOP = b"Q" TRUE = b"R" - UNICODE = b"S" COMPLEX = b"T" class Unserializer: num2func: dict[bytes, Callable[[Unserializer], None]] = {} - py2str_as_py3str = True # True - py3str_as_py2str = False # false means py2 will get unicode def __init__( self, stream: ReadIO, channel_or_gateway: Channel | BaseGateway | None = None, - strconfig: tuple[bool, bool] | None = None, ) -> None: if isinstance(channel_or_gateway, Channel): gw: BaseGateway | None = channel_or_gateway.gateway else: gw = channel_or_gateway - if channel_or_gateway is not None: - strconfig = channel_or_gateway._strconfig - if strconfig: - self.py2str_as_py3str, self.py3str_as_py2str = strconfig self.stream = stream if gw is None: self.channelfactory = None @@ -1219,25 +1197,10 @@ def _read_byte_string(self) -> bytes: as_bytes = self.stream.read(length) return as_bytes - def load_py3string(self) -> None: - as_bytes = self._read_byte_string() - if self.py3str_as_py2str: - # XXX Should we try to decode into latin-1? - self.stack.append(as_bytes) - else: - self.stack.append(as_bytes.decode("utf-8")) - - num2func[opcode.PY3STRING] = load_py3string - - def load_py2string(self) -> None: - as_bytes = self._read_byte_string() - if self.py2str_as_py3str: - s: bytes | str = as_bytes.decode("latin-1") - else: - s = as_bytes - self.stack.append(s) + def load_string(self) -> None: + self.stack.append(self._read_byte_string().decode("utf-8")) - num2func[opcode.PY2STRING] = load_py2string + num2func[opcode.STRING] = load_string def load_bytes(self) -> None: s = self._read_byte_string() @@ -1245,11 +1208,6 @@ def load_bytes(self) -> None: num2func[opcode.BYTES] = load_bytes - def load_unicode(self) -> None: - self.stack.append(self._read_byte_string().decode("utf-8")) - - num2func[opcode.UNICODE] = load_unicode - def load_newlist(self) -> None: length = self._read_int4() self.stack.append([None] * length) @@ -1323,46 +1281,27 @@ def dump(byteio, obj: object) -> None: _Serializer(write=byteio.write).save(obj, versioned=True) -def loads( - bytestring: bytes, py2str_as_py3str: bool = False, py3str_as_py2str: bool = False -) -> Any: +def loads(bytestring: bytes) -> Any: """Deserialize the given bytestring to an object. - py2str_as_py3str: If true then string (str) objects previously - dumped on Python2 will be loaded as Python3 - strings which really are text objects. - py3str_as_py2str: If true then string (str) objects previously - dumped on Python3 will be loaded as Python2 - strings instead of unicode objects. - If the bytestring was dumped with an incompatible protocol version or if the bytestring is corrupted, the ``execnet.DataFormatError`` will be raised. """ - io = BytesIO(bytestring) - return load( - io, py2str_as_py3str=py2str_as_py3str, py3str_as_py2str=py3str_as_py2str - ) + return load(BytesIO(bytestring)) -def load( - io: ReadIO, py2str_as_py3str: bool = False, py3str_as_py2str: bool = False -) -> Any: +def load(io: ReadIO) -> Any: """Derserialize an object form the specified stream. - Behaviour and parameters are otherwise the same as with ``loads`` + Behaviour is otherwise the same as with ``loads`` """ - strconfig = (py2str_as_py3str, py3str_as_py2str) - return Unserializer(io, strconfig=strconfig).load(versioned=True) + return Unserializer(io).load(versioned=True) -def loads_internal( - bytestring: bytes, - channelfactory=None, - strconfig: tuple[bool, bool] | None = None, -) -> Any: +def loads_internal(bytestring: bytes, channelfactory=None) -> Any: io = BytesIO(bytestring) - return Unserializer(io, channelfactory, strconfig).load() + return Unserializer(io, channelfactory).load() def dumps_internal(obj: object) -> bytes: @@ -1420,7 +1359,7 @@ def save_bytes(self, bytes_: bytes) -> None: self._write_byte_sequence(bytes_) def save_str(self, s: str) -> None: - self._write(opcode.PY3STRING) + self._write(opcode.STRING) self._write_unicode_string(s) def _write_unicode_string(self, s: str) -> None: diff --git a/src/execnet/rsync.py b/src/execnet/rsync.py index f92bc37a..876e38fc 100644 --- a/src/execnet/rsync.py +++ b/src/execnet/rsync.py @@ -173,7 +173,6 @@ def itemcallback(req) -> None: self._receivequeue.put((channel, req)) channel = gateway.remote_exec(execnet.rsync_remote) - channel.reconfigure(py2str_as_py3str=False, py3str_as_py2str=False) channel.setcallback(itemcallback, endmarker=None) channel.send((str(destdir), options)) self._channels[channel] = finishedcallback diff --git a/testing/test_compatibility_regressions.py b/testing/test_compatibility_regressions.py index 5343e7d5..837c0c6d 100644 --- a/testing/test_compatibility_regressions.py +++ b/testing/test_compatibility_regressions.py @@ -18,13 +18,13 @@ def test_opcodes() -> None: "NEWDICT": b"J", "NEWLIST": b"K", "NONE": b"L", - "PY2STRING": b"M", - "PY3STRING": b"N", + # b"M" (py2 str) and b"S" (py2 unicode) are retired along with + # Python2 support -- the bytes stay unused rather than reassigned. + "STRING": b"N", "SET": b"O", "SETITEM": b"P", "STOP": b"Q", "TRUE": b"R", - "UNICODE": b"S", # added in 1.4 # causes a regression since it was ordered in # between CHANNEL and FALSE as "C" moving the other items diff --git a/testing/test_serializer.py b/testing/test_serializer.py index 49332c97..ec8e86ae 100644 --- a/testing/test_serializer.py +++ b/testing/test_serializer.py @@ -187,6 +187,7 @@ def test_tuple_nested_with_empty_in_between(dump, load) -> None: assert s == "(1, (), 3)" -def test_py2_string_loads() -> None: - """Regression test for #267.""" - assert execnet.gateway_base.loads(b"\x02M\x00\x00\x00\x01aQ") == b"a" +def test_py2_string_opcode_is_retired() -> None: + """The py2 ``str`` opcode ``M`` is gone; only Python2 ever emitted it.""" + with pytest.raises(execnet.DataFormatError, match="unknown opcode"): + execnet.gateway_base.loads(b"\x02M\x00\x00\x00\x01aQ") diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index a6f27981..ef52e232 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -403,16 +403,3 @@ async def main() -> None: await group.makegateway("id=notype") trio.run(main) - - -def test_channel_reconfigure_string_coercion() -> None: - async def main() -> None: - async with gateway_pair() as (left, right): - sender = left.open_channel() - receiver = right.open_channel(sender.id) - await sender.reconfigure(py3str_as_py2str=True) - await receiver.send("text") - # our side now loads py3 strings as bytes - assert await sender.receive() == b"text" - - trio.run(main) From 740b11c5d85d516d6a58d27e6ea4a881a018c4ac Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 21:29:26 +0200 Subject: [PATCH 60/91] refactor!: make the accidentally-public modules private execnet grew four deliberate public namespaces during the Trio port -- execnet (aliases of execnet.sync), execnet.trio, execnet.aio and execnet.portal -- and every module added since is underscore-prefixed. Six pre-Trio modules never got that treatment and stayed importable as execnet.gateway_base, .gateway, .multi, .rsync, .rsync_remote and .xspec, reachable only because `import execnet` pulled them in transitively. The docs pointed Sphinx at three of them, so they read as public API. Each is now an underscore-prefixed module plus a thin shim under the old name that warns and forwards, removable in execnet 3.0. Both `import execnet.gateway_base` and `execnet.gateway_base.X` after a plain `import execnet` keep working (pytest-xdist does the latter). gateway_base.py held six unrelated concerns in 1449 lines and is split by concern, each layer importing only from the ones above it: _trace EXECNET_DEBUG tracing _errors every exception, plus sysex and the error texts _execmodel the deprecated ExecModel presets _message IO protocols, Message, FrameDecoder _serialize the wire serializer _channel Channel, ChannelFactory, ChannelFile adapters _gateway_base BaseGateway, WorkerGateway Two cycles fell out of that. Unserializer resolved its channel argument with isinstance(Channel); it now duck-types on .gateway, which also subsumes the AsyncChannel._channelfactory property the async core needed. INTERRUPT_TEXT and MAIN_THREAD_ONLY_DEADLOCK_TEXT moved to _errors rather than _channel, since RemoteError.warn() reads the former. Surface changes: * execnet.trio no longer exports ByteStream, RawChannel, RawChannelStream or serve_gateway -- internal routing detail with no caller outside _trio_host. serve_gateway had none at all. * execnet._gateway no longer lists _find_non_builtin_globals and _source_of_function in __all__; they were there for tests only. * Added execnet.can_send(obj), the supported way to ask whether a value can cross a channel. It sits on execnet alone -- the wire contract does not vary by namespace -- so it is not mirrored onto sync/trio/aio. * Removed execnet.loads/dump/load and dropped execnet.dumps from the surface. dumps stays reachable behind a warning, out of __all__ and dir(), purely so released pytest-xdist keeps working; see the FOLLOW-UP note on execnet._XDIST_COMPAT. test_namespaces.py now enforces the boundary: every namespace __all__ resolves, no unexpected non-underscore module exists, and each shim attribute warns and forwards to the same object. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 28 + doc/basics.rst | 23 +- src/execnet/__init__.py | 50 +- src/execnet/_channel.py | 483 +++++++ src/execnet/_errors.py | 109 ++ src/execnet/_execmodel.py | 92 ++ src/execnet/_gateway.py | 158 +++ src/execnet/_gateway_base.py | 237 ++++ src/execnet/_message.py | 174 +++ src/execnet/_multi.py | 419 ++++++ src/execnet/_rsync.py | 248 ++++ src/execnet/_rsync_remote.py | 126 ++ src/execnet/_serialize.py | 439 ++++++ src/execnet/_shim.py | 47 + src/execnet/_trace.py | 47 + src/execnet/_trio_gateway.py | 25 +- src/execnet/_trio_host.py | 28 +- src/execnet/_trio_worker.py | 16 +- src/execnet/_xspec.py | 74 + src/execnet/aio.py | 17 +- src/execnet/gateway.py | 166 +-- src/execnet/gateway_base.py | 1518 +-------------------- src/execnet/multi.py | 427 +----- src/execnet/rsync.py | 246 +--- src/execnet/rsync_remote.py | 127 +- src/execnet/sync.py | 30 +- src/execnet/trio.py | 26 +- src/execnet/xspec.py | 75 +- testing/conftest.py | 6 +- testing/test_basics.py | 113 +- testing/test_channel.py | 4 +- testing/test_channel_stress.py | 2 +- testing/test_compatibility_regressions.py | 4 +- testing/test_execmodel_trio.py | 2 +- testing/test_gateway.py | 25 +- testing/test_gevent.py | 2 +- testing/test_multi.py | 10 +- testing/test_namespaces.py | 120 +- testing/test_rsync.py | 2 +- testing/test_serializer.py | 10 +- testing/test_termination.py | 4 +- testing/test_trio_gateway.py | 12 +- testing/test_xspec.py | 2 +- 43 files changed, 3148 insertions(+), 2625 deletions(-) create mode 100644 src/execnet/_channel.py create mode 100644 src/execnet/_errors.py create mode 100644 src/execnet/_execmodel.py create mode 100644 src/execnet/_gateway.py create mode 100644 src/execnet/_gateway_base.py create mode 100644 src/execnet/_message.py create mode 100644 src/execnet/_multi.py create mode 100644 src/execnet/_rsync.py create mode 100644 src/execnet/_rsync_remote.py create mode 100644 src/execnet/_serialize.py create mode 100644 src/execnet/_shim.py create mode 100644 src/execnet/_trace.py create mode 100644 src/execnet/_xspec.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9f2d3f38..0a966560 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -29,6 +29,34 @@ * The serializer dropped the retired ``PY2STRING`` and ``UNICODE`` opcodes and renamed ``PY3STRING`` to ``STRING``. Values dumped by execnet running on Python2 no longer load. Opcode bytes are unchanged for every type that survives. +* The supported API is now exactly five namespaces: ``execnet`` (aliases of + ``execnet.sync``), ``execnet.sync``, ``execnet.trio``, ``execnet.aio`` and + ``execnet.portal``. The pre-Trio modules ``execnet.gateway_base``, ``execnet.gateway``, + ``execnet.multi``, ``execnet.rsync``, ``execnet.rsync_remote`` and ``execnet.xspec`` + were only ever reachable because ``import execnet`` pulled them in transitively; they + are now deprecated forwarding shims that warn on attribute access and will be removed + in execnet 3.0. Both ``import execnet.gateway_base`` and ``execnet.gateway_base.X`` + after a plain ``import execnet`` keep working for now. Every name they exposed is + available from a public namespace, except the internals listed below. +* ``gateway_base`` was split into private modules grouped by concern: ``_trace``, + ``_errors``, ``_execmodel``, ``_message`` (IO protocols, ``Message``, + ``FrameDecoder``), ``_serialize``, ``_channel`` (``Channel``, ``ChannelFactory``, + the ``ChannelFile`` adapters) and ``_gateway_base`` (``BaseGateway``, + ``WorkerGateway``). +* Removed ``execnet.loads``, ``execnet.dump`` and ``execnet.load``, and dropped + ``execnet.dumps`` from the public surface. The standalone serializer is internal. + Added ``execnet.can_send(obj)``, which answers whether a value can cross a channel, + for callers that previously probed with ``try: execnet.dumps(x) / except DumpError``. + It lives on ``execnet`` only -- the wire contract does not vary by namespace. + + ``execnet.dumps`` itself stays *reachable* for now, warning on access, purely so + released ``pytest-xdist`` keeps working; it is absent from ``__all__`` and from + ``dir(execnet)``. It is scheduled for removal once xdist ports its probe to + ``can_send``. +* ``execnet.trio`` no longer exports ``ByteStream``, ``RawChannel``, + ``RawChannelStream`` or ``serve_gateway``; the raw-channel layer is internal routing + detail. No names were added to ``execnet.sync``, ``execnet.aio`` or + ``execnet.portal``. 2.1.2 (2025-11-11) diff --git a/doc/basics.rst b/doc/basics.rst index a22ad7de..8378dbc3 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -72,7 +72,7 @@ Examples for valid gateway specifications remote_exec: execute source code remotely =================================================== -.. currentmodule:: execnet.gateway +.. currentmodule:: execnet All gateways offer a simple method to execute source code in the instantiated subprocess-interpreter: @@ -94,7 +94,7 @@ is available to the remotely executing source. Channels: exchanging data with remote code ======================================================= -.. currentmodule:: execnet.gateway_base +.. currentmodule:: execnet A channel object allows to send and receive data between two asynchronously running programs. @@ -114,7 +114,7 @@ two asynchronously running programs. Grouped Gateways and robust termination =============================================== -.. currentmodule:: execnet.multi +.. currentmodule:: execnet All created gateway instances are part of a group. If you call ``execnet.makegateway`` it actually is forwarded to @@ -169,7 +169,7 @@ executions concurrently, they will run in non-main threads. remote_status: get low-level execution info =================================================== -.. currentmodule:: execnet.gateway +.. currentmodule:: execnet All gateways offer a simple method to obtain some status information from the remote side. @@ -232,8 +232,19 @@ Sending an unsupported value raises ``DumpError`` (a subclass of ``DataFormatError``); a corrupt or protocol-mismatched payload on receive raises ``LoadError``. These signal a **caller error to resolve** -- reduce the value to simple data before sending -- not a transport failure. The -standalone serializer itself is an internal implementation detail -(``execnet.gateway_base``) and is not part of the public API. +standalone serializer itself is an internal implementation detail and is not +part of the public API. + +To branch *before* sending rather than handling the error, ask: + +.. autofunction:: can_send + +:: + + channel.send(value if execnet.can_send(value) else repr(value)) + +It lives on ``execnet`` itself rather than on any one namespace: the wire +contract is the same whichever surface you drive a gateway from. Encode rich objects yourself ------------------------------------------------------- diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index b81aaedd..5912ae9a 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -15,11 +15,15 @@ * :mod:`execnet.portal` — cross-thread / cross-loop communication primitives shared by all of them. +``can_send`` sits here rather than on any one of them: the wire-format +contract is the same whichever surface you drive a gateway from. + (c) 2012, Holger Krekel and others """ from typing import Any +from ._serialize import can_send from ._version import version as __version__ from .sync import Channel from .sync import DataFormatError @@ -51,17 +55,57 @@ "TimeoutError", "XSpec", "__version__", + "can_send", "default_group", "makegateway", "set_execmodel", ] +#: resolved lazily so ``import execnet`` does not load the trio event loop +#: machinery, plus the deprecated pre-Trio module names -- those used to be +#: reachable here only because the import chain pulled them in, and callers +#: that still do ``execnet.gateway_base.X`` must reach the warning shim. +_LAZY_MODULES = ( + "aio", + "portal", + "trio", + "gateway", + "gateway_base", + "multi", + "rsync", + "rsync_remote", + "xspec", +) + + +#: TEMPORARY pytest-xdist compatibility. ``xdist/remote.py`` probes +#: serializability with ``try: execnet.dumps(x) / except execnet.DumpError`` +#: before shipping warning args and report attrs. The standalone serializer +#: is internal and :func:`can_send` replaces that probe, but dropping the name +#: outright breaks every released xdist, so it stays reachable -- warning, and +#: deliberately absent from ``__all__``. +#: +#: FOLLOW-UP (after the execnet release): port xdist to ``execnet.can_send``, +#: then delete this and its test. Nothing else may be added here. +_XDIST_COMPAT = ("dumps",) + + def __getattr__(name: str) -> Any: - # Lazy namespace modules: keep ``import execnet`` from loading the - # trio event loop machinery until it is actually used. - if name in ("aio", "portal", "trio"): + if name in _LAZY_MODULES: import importlib return importlib.import_module(f".{name}", __name__) + if name in _XDIST_COMPAT: + import importlib + import warnings + + warnings.warn( + f"execnet.{name} is a temporary pytest-xdist compatibility shim and" + " will be removed; the standalone serializer is not public. Use" + " execnet.can_send(obj) to test whether a value can cross a channel.", + DeprecationWarning, + stacklevel=2, + ) + return getattr(importlib.import_module("._serialize", __name__), name) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/execnet/_channel.py b/src/execnet/_channel.py new file mode 100644 index 00000000..e1f5449b --- /dev/null +++ b/src/execnet/_channel.py @@ -0,0 +1,483 @@ +"""The blocking :class:`Channel` and its registry and file adapters. + +A ``Channel`` is a facade over the async core: the gateway's Trio session +diverts inbound payloads for a channel id into a :class:`~execnet._boundary.Mailbox` +(or a registered callback), and ``receive()`` deserializes at the call site, +off the loop thread. +""" + +from __future__ import annotations + +import builtins +import threading +import weakref +from collections.abc import Callable +from collections.abc import Iterator +from contextlib import suppress +from typing import TYPE_CHECKING +from typing import Any +from typing import Literal +from typing import cast +from typing import overload + +from ._boundary import Flag +from ._boundary import Mailbox +from ._errors import RemoteError +from ._errors import TimeoutError +from ._message import Message +from ._serialize import dumps_internal +from ._serialize import loads_internal + +if TYPE_CHECKING: + from ._gateway_base import BaseGateway + +NO_ENDMARKER_WANTED = object() + + +class Channel: + """Communication channel between two Python Interpreter execution points. + + A facade over the async core: the gateway's Trio session diverts + inbound payloads for this id into a :class:`Mailbox` (or a registered + callback, invoked on the loop thread); ``receive()`` deserializes at + the call site. Sends go through ``gateway._send``. + """ + + RemoteError = RemoteError + TimeoutError = TimeoutError + _INTERNALWAKEUP = 1000 + _executing = False + #: set once a receiver callback is attached. A consumer *task* on the + #: loop drains this channel and runs the callback in a threadpool thread; + #: the task holds the channel alive, so a callback channel's lifecycle is + #: bound to consumption (and to GC once the stream closes) rather than to + #: a strong registry. + _has_consumer = False + #: loop-side hooks installed while a consumer is attached: divert one + #: payload into / close the consumer task's inbox (set by the Trio session, + #: so they encapsulate the trio memory channel; this module stays trio-free). + _consumer_feed: Callable[[bytes], None] | None = None + _consumer_close_inbox: Callable[[], None] | None = None + #: thread-safe "stop the consumer" hook -- ends the task's inbox. + _consumer_stop: Callable[[], None] | None = None + #: set by the consumer task once it has drained every item and fired the + #: endmarker; ``waitclose()`` waits on this (instead of ``_receiveclosed``) + #: so it still guarantees "all callbacks have run" before returning. + _consumer_done: Flag | None = None + + def __init__(self, gateway: BaseGateway, id: int) -> None: + """:private:""" + assert isinstance(id, int) + assert not isinstance(gateway, type) + self.gateway = gateway + self.id = id + # serialized payloads (or ENDMARKER); None once a consumer is attached + self._mailbox: Mailbox[Any] | None = Mailbox(gateway._new_wakener()) + self._closed = False + self._receiveclosed = Flag(gateway._new_wakener()) + self._remoteerrors: list[RemoteError] = [] + + def _trace(self, *msg: object) -> None: + self.gateway._trace(self.id, *msg) + + def setcallback( + self, + callback: Callable[[Any], Any], + endmarker: object = NO_ENDMARKER_WANTED, + ) -> None: + """Set a callback function for receiving items. + + A consumer task on the gateway's loop drains this channel and runs + ``callback`` for each received item in a threadpool thread (so a slow + callback never blocks the loop); items for one channel are delivered + strictly in order. Already-queued items are delivered first. After + this call ``receive()`` raises an error. + + The task keeps the channel alive for as long as it is consuming, so a + callback channel need not be referenced elsewhere. If an endmarker is + specified the callback is eventually called with it when the channel + closes, and ``waitclose()`` does not return until every callback + (including the endmarker) has run. + """ + self.gateway._start_channel_consumer(self, callback, endmarker) + + def __repr__(self) -> str: + flag = (self.isclosed() and "closed") or "open" + return "" % (self.id, flag) + + def __del__(self) -> None: + if self.gateway is None: # can be None in tests + return # type: ignore[unreachable] + + self._trace("channel.__del__") + # no multithreading issues here, because we have the last ref to 'self' + if self._closed: + # state transition "closed" --> "deleted" + for error in self._remoteerrors: + error.warn() + elif self._receiveclosed.is_set(): + # state transition "sendonly" --> "deleted" + # the remote channel is already in "deleted" state, nothing to do + pass + else: + # state transition "opened" --> "deleted" + # check if we are in the middle of interpreter shutdown + # in which case the process will go away and we probably + # don't need to try to send a closing or last message + # (and often it won't work anymore to send things out) + # A callback channel is held by its consumer task until the stream + # closes, so by the time __del__ runs it is never in the "opened" + # state -- this branch only ever fires for a plain receive channel. + if Message is not None: + with suppress(OSError, ValueError): # ignore problems with sending + # Never wait during GC: post the close best-effort. + send = getattr( + self.gateway, "_send_nonblocking", self.gateway._send + ) + send(Message.CHANNEL_CLOSE, self.id) + with suppress(Exception): + self.gateway._release_channel(self.id) + + def _getremoteerror(self): + try: + return self._remoteerrors.pop(0) + except IndexError: + try: + return self.gateway._error + except AttributeError: + pass + return None + + # + # loop-side delivery (called by the session's raw-channel consumer) + # + def _deliver_payload(self, data: bytes) -> None: + """Route one inbound serialized payload (loop thread). + + A callback channel diverts payloads into its consumer task's inbox; a + plain channel queues them for ``receive()``. + """ + if self._closed: + return # late data for a locally closed channel: drop + feed = self._consumer_feed + if feed is not None: + feed(data) + return + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(data) + # no consumer and no mailbox: closed for receiving -- drop + + def _close_from_remote(self, remoteerror=None, *, sendonly: bool = False) -> None: + """Close initiated by the peer or session shutdown (loop thread). + + For a callback channel the consumer task ends separately (its inbox is + closed) and fires the endmarker; here we only record the state. + """ + if remoteerror: + self._remoteerrors.append(remoteerror) + if self._has_consumer: + close_inbox = self._consumer_close_inbox + if close_inbox is not None: + close_inbox() # ends the consumer task (it fires the endmarker) + else: + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) + self.gateway._channelfactory._no_longer_opened(self.id) + if not sendonly: # otherwise #--> "sendonly" + self._closed = True # --> "closed" + self._receiveclosed.set() + + # + # public API for channel objects + # + def isclosed(self) -> bool: + """Return True if the channel is closed. + + A closed channel may still hold items. + """ + return self._closed + + @overload + def makefile(self, mode: Literal["r"], proxyclose: bool = ...) -> ChannelFileRead: + pass + + @overload + def makefile( + self, + mode: Literal["w"] = ..., + proxyclose: bool = ..., + ) -> ChannelFileWrite: + pass + + def makefile( + self, + mode: Literal["r", "w"] = "w", + proxyclose: bool = False, + ) -> ChannelFileWrite | ChannelFileRead: + """Return a file-like object. + + mode can be 'w' or 'r' for writeable/readable files. + If proxyclose is true, file.close() will also close the channel. + """ + if mode == "w": + return ChannelFileWrite(channel=self, proxyclose=proxyclose) + elif mode == "r": + return ChannelFileRead(channel=self, proxyclose=proxyclose) + raise ValueError(f"mode {mode!r} not available") + + def close(self, error=None) -> None: + """Close down this channel with an optional error message. + + Note that closing of a channel tied to remote_exec happens + automatically at the end of execution and cannot + be done explicitly. + """ + if self._executing: + raise OSError("cannot explicitly close channel within remote_exec") + if self._closed: + self.gateway._trace(self, "ignoring redundant call to close()") + if not self._closed: + # state transition "opened/sendonly" --> "closed" + # threads warning: the channel might be closed under our feet, + # but it's never damaging to send too many CHANNEL_CLOSE messages + # however, if the other side triggered a close already, we + # do not send back a closed message. + if not self._receiveclosed.is_set(): + put = self.gateway._send + if error is not None: + put(Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(error)) + else: + put(Message.CHANNEL_CLOSE, self.id) + self._trace("sent channel close message") + if isinstance(error, RemoteError): + self._remoteerrors.append(error) + self._closed = True # --> "closed" + self._receiveclosed.set() + if self._has_consumer: + # End the consumer task's inbox; it drains any buffered items, + # fires the endmarker, and sets _consumer_done. + stop = self._consumer_stop + if stop is not None: + stop() + else: + mailbox = self._mailbox + if mailbox is not None: + mailbox.put(ENDMARKER) + self.gateway._channelfactory._no_longer_opened(self.id) + self.gateway._release_channel(self.id) + + def waitclose(self, timeout: float | None = None) -> None: + """Wait until this channel is closed (or the remote side + otherwise signalled that no more data was being sent). + + The channel may still hold receiveable items, but not receive + any more after waitclose() has returned. + + Exceptions from executing code on the other side are reraised as local + channel.RemoteErrors. + + EOFError is raised if the reading-connection was prematurely closed, + which often indicates a dying process. + + self.TimeoutError is raised after the specified number of seconds + (default is None, i.e. wait indefinitely). + """ + # For a callback channel wait on the consumer task finishing (so every + # callback, including the endmarker, has run); otherwise wait for the + # non-"opened" state directly. + signal = ( + self._consumer_done + if self._consumer_done is not None + else (self._receiveclosed) + ) + signal.wait(timeout=timeout) + if not signal.is_set(): + raise self.TimeoutError("Timeout after %r seconds" % timeout) + error = self._getremoteerror() + if error: + raise error + + def send(self, item: object) -> None: + """Sends the given item to the other side of the channel, + possibly blocking if the sender queue is full. + + The item must be a simple Python type and will be + copied to the other side by value. + + OSError is raised if the write pipe was prematurely closed. + """ + if self.isclosed(): + raise OSError(f"cannot send to {self!r}") + self.gateway._send(Message.CHANNEL_DATA, self.id, dumps_internal(item)) + + def receive(self, timeout: float | None = None) -> Any: + """Receive a data item that was sent from the other side. + + timeout: None [default] blocked waiting. A positive number + indicates the number of seconds after which a channel.TimeoutError + exception will be raised if no item was received. + + Note that exceptions from the remotely executing code will be + reraised as channel.RemoteError exceptions containing + a textual representation of the remote traceback. + """ + mailbox = self._mailbox + if mailbox is None: + raise OSError("cannot receive(), channel has receiver callback") + try: + x = mailbox.get(timeout) + except builtins.TimeoutError: + raise self.TimeoutError("no item after %r seconds" % timeout) from None + if x is ENDMARKER: + mailbox.put(x) # for other receivers + raise self._getremoteerror() or EOFError() + else: + return loads_internal(x, self) + + def __iter__(self) -> Iterator[Any]: + return self + + def next(self) -> Any: + try: + return self.receive() + except EOFError: + raise StopIteration from None + + __next__ = next + + +ENDMARKER = object() + + +class ChannelFactory: + """Registry and id allocator for a gateway's sync channels. + + Message routing lives in the Trio session (the sync channel binds a + consumer on the session's raw channel); the factory only tracks live + channels -- weakly, so dropping the last user reference triggers + ``Channel.__del__``'s close message. A callback channel is kept alive by + its consumer task rather than by any registry here. + """ + + def __init__(self, gateway: BaseGateway, startcount: int = 1) -> None: + self._channels: weakref.WeakValueDictionary[int, Channel] = ( + weakref.WeakValueDictionary() + ) + self._writelock = threading.Lock() + self.gateway = gateway + self.count = startcount + self.finished = False + self._list = list # needed during interp-shutdown + + def new(self, id: int | None = None) -> Channel: + """Create a new Channel with 'id' (or create new id if None).""" + with self._writelock: + if self.finished: + raise OSError(f"connection already closed: {self.gateway}") + if id is None: + id = self.count + self.count += 2 + try: + channel = self._channels[id] + except KeyError: + channel = self._channels[id] = Channel(self.gateway, id) + self.gateway._bind_channel(channel) + return channel + + def allocate_id(self) -> int: + """Reserve a fresh channel id without creating a Channel object.""" + with self._writelock: + if self.finished: + raise OSError(f"connection already closed: {self.gateway}") + id = self.count + self.count += 2 + return id + + def channels(self) -> list[Channel]: + return self._list(self._channels.values()) + + # + # internal methods, called from the loop thread (or local close paths) + # + def _no_longer_opened(self, id: int) -> None: + self._channels.pop(id, None) + + def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> None: + """Close ``id`` as if the peer had closed it (no message is sent).""" + channel = self._channels.get(id) + if channel is None: + # channel already in "deleted" state + if remoteerror: + remoteerror.warn() + self._no_longer_opened(id) + else: + channel._close_from_remote(remoteerror, sendonly=sendonly) + + def _finished_receiving(self) -> None: + with self._writelock: + self.finished = True + for id in self._list(self._channels): + self._local_close(id, sendonly=True) + + +class ChannelFile: + def __init__(self, channel: Channel, proxyclose: bool = True) -> None: + self.channel = channel + self._proxyclose = proxyclose + + def isatty(self) -> bool: + return False + + def close(self) -> None: + if self._proxyclose: + self.channel.close() + + def __repr__(self) -> str: + state = (self.channel.isclosed() and "closed") or "open" + return "" % (self.channel.id, state) + + +class ChannelFileWrite(ChannelFile): + def write(self, out: bytes) -> None: + self.channel.send(out) + + def flush(self) -> None: + pass + + +class ChannelFileRead(ChannelFile): + def __init__(self, channel: Channel, proxyclose: bool = True) -> None: + super().__init__(channel, proxyclose) + self._buffer: str | None = None + + def read(self, n: int) -> str: + try: + if self._buffer is None: + self._buffer = cast(str, self.channel.receive()) + while len(self._buffer) < n: + self._buffer += cast(str, self.channel.receive()) + except EOFError: + self.close() + if self._buffer is None: + ret = "" + else: + ret = self._buffer[:n] + self._buffer = self._buffer[n:] + return ret + + def readline(self) -> str: + if self._buffer is not None: + i = self._buffer.find("\n") + if i != -1: + return self.read(i + 1) + line = self.read(len(self._buffer) + 1) + else: + line = self.read(1) + while line and line[-1] != "\n": + c = self.read(1) + if not c: + break + line += c + return line diff --git a/src/execnet/_errors.py b/src/execnet/_errors.py new file mode 100644 index 00000000..e90ee2d0 --- /dev/null +++ b/src/execnet/_errors.py @@ -0,0 +1,109 @@ +"""Exception types and the error texts that cross the wire. + +The channel-facing errors (:class:`RemoteError`, :class:`TimeoutError`, +:class:`HostNotFound`, and the :class:`DataFormatError` family) are the one +part of this module that is public: every namespace -- :mod:`execnet.sync`, +:mod:`execnet.trio`, :mod:`execnet.aio` -- re-exports them, because the same +errors are raised whichever surface you drive a gateway from. +""" + +from __future__ import annotations + +import os +import sys +import traceback + +#: exceptions that must never be swallowed by a broad ``except`` +sysex = (KeyboardInterrupt, SystemExit) + +INTERRUPT_TEXT = "keyboard-interrupted" +MAIN_THREAD_ONLY_DEADLOCK_TEXT = ( + "concurrent remote_exec would cause deadlock for main_thread_only execmodel" +) + + +class GatewayReceivedTerminate(Exception): + """Receiver got a gateway termination message.""" + + +class HostNotFound(ConnectionError): + """The remote side of a gateway could not be reached.""" + + +def geterrortext( + exc: BaseException, + format_exception=traceback.format_exception, + sysex: tuple[type[BaseException], ...] = sysex, +) -> str: + try: + # In py310, can change this to: + # l = format_exception(exc) + l = format_exception(type(exc), exc, exc.__traceback__) + errortext = "".join(l) + except sysex: + raise + except BaseException: + errortext = f"{type(exc).__name__}: {exc}" + return errortext + + +class RemoteError(Exception): + """Exception containing a stringified error from the other side.""" + + def __init__(self, formatted: str) -> None: + super().__init__() + self.formatted = formatted + + def __str__(self) -> str: + return self.formatted + + def __repr__(self) -> str: + return f"{self.__class__.__name__}: {self.formatted}" + + def warn(self) -> None: + if self.formatted != INTERRUPT_TEXT: + # XXX do this better + sys.stderr.write(f"[{os.getpid()}] Warning: unhandled {self!r}\n") + + +class TimeoutError(IOError): + """Exception indicating that a timeout was reached.""" + + +class DataFormatError(Exception): + """A value could not cross the channel in execnet's simple wire format. + + execnet only moves *simple* builtin data over a channel -- ``None``, + ``bool``, ``int``, ``float``, ``complex``, ``bytes``, ``str``, and + (arbitrarily nested) ``list``/``tuple``/``set``/``frozenset``/``dict`` of + those -- plus channel references, which pass through as channels. It does + **not** pickle: arbitrary instances, functions, ``datetime``, dataclasses, + pydantic models, numpy arrays, etc. have no wire representation. + + A ``DataFormatError`` therefore signals a caller error to resolve, not a + transport failure: reduce the value to simple data before sending (and + reconstruct it after receiving) with an encoding mechanism of your own -- + e.g. pydantic ``model_dump`` / ``model_validate`` or pytest's + ``pytest_report_to_serializable`` / ``pytest_report_from_serializable`` + hooks. See the docs, "Sending objects over a channel". + """ + + +class DumpError(DataFormatError): + """A value being **sent** is not simple wire data; convert it first. + + Raised by ``channel.send`` (and the internal serializer) when an object is + not one of execnet's simple wire types. Fix it at the call site by turning + the rich object into simple data -- e.g. ``dt.isoformat()``, + ``dataclasses.asdict(obj)``, ``model.model_dump(mode="json")`` -- rather + than expecting the channel to pickle it. Channels are the one non-builtin + that *is* sendable, so nested channel references are fine. + """ + + +class LoadError(DataFormatError): + """Received bytes could not be turned back into an object. + + Raised while **receiving** (deserializing) -- a corrupted or + protocol-incompatible payload, or data produced by a mismatched peer. + """ diff --git a/src/execnet/_execmodel.py b/src/execnet/_execmodel.py new file mode 100644 index 00000000..9cb1d75c --- /dev/null +++ b/src/execnet/_execmodel.py @@ -0,0 +1,92 @@ +"""Deprecated execution-model presets. + +The machinery behind execution models was retired during the Trio port; what +survives is the preset *name*, which maps onto the worker config axes +(``loop=`` / ``exec=`` / ``wait=``). Kept because pytest-xdist's remote worker +still builds its test queue on ``channel.gateway.execmodel.RLock()``/``Event()``. +""" + +from __future__ import annotations + +import os +import threading + + +class ExecModel: + """Deprecated preset name for an execution model. + + The machinery behind execution models was retired: protocol IO always + runs on the Trio host and blocking waits go through the boundary kit's + wakeners (``execnet._boundary``); the name maps onto the worker config + axes (``loop=`` / ``exec=`` / ``wait=``). The stdlib-delegating + members stay for API compatibility (pytest-xdist builds its test queue + on ``execmodel.RLock``/``Event``) -- every preset is thread-shaped. + """ + + def __init__(self, backend: str) -> None: + self.backend = backend + + def __repr__(self) -> str: + return "" % self.backend + + @property + def queue(self): + import queue + + return queue + + @property + def subprocess(self): + import subprocess + + return subprocess + + @property + def socket(self): + import socket + + return socket + + def get_ident(self) -> int: + import _thread + + return _thread.get_ident() + + def sleep(self, delay: float) -> None: + import time + + time.sleep(delay) + + def start(self, func, args=()) -> None: + import _thread + + _thread.start_new_thread(func, args) + + def fdopen(self, fd, mode, bufsize=1, closefd=True): + return os.fdopen(fd, mode, bufsize, encoding="utf-8", closefd=closefd) + + def Lock(self): + return threading.RLock() + + def RLock(self): + return threading.RLock() + + def Event(self) -> threading.Event: + return threading.Event() + + +#: worker profiles: where exec'd code runs relative to the protocol loop +EXECMODEL_PROFILES = ( + "thread", # hybrid: primary on the main thread, overflow on pool threads + "main_thread_only", # exec serialized on the main thread (GUI/pytest) + "trio", # pure async: loop owns the main thread, async sources as tasks + "gevent", # greenlets on a main-thread hub, one per remote_exec +) + + +def get_execmodel(backend: str | ExecModel) -> ExecModel: + if isinstance(backend, ExecModel): + return backend + if backend in EXECMODEL_PROFILES: + return ExecModel(backend) + raise ValueError(f"unknown execmodel {backend!r}") diff --git a/src/execnet/_gateway.py b/src/execnet/_gateway.py new file mode 100644 index 00000000..facbdeca --- /dev/null +++ b/src/execnet/_gateway.py @@ -0,0 +1,158 @@ +"""Gateway code for initiating popen, socket and ssh connections. + +(c) 2004-2013, Holger Krekel and others +""" + +from __future__ import annotations + +import types +from collections.abc import Callable +from typing import TYPE_CHECKING +from typing import Any + +from ._channel import Channel +from ._exec_source import normalize_exec_source +from ._gateway_base import BaseGateway +from ._message import IO +from ._message import Message +from ._multi import Group +from ._serialize import dumps_internal +from ._xspec import XSpec + +__all__ = [ + "Gateway", + "RInfo", + "RemoteStatus", +] + + +class Gateway(BaseGateway): + """Gateway to a local or remote Python Interpreter.""" + + _group: Group + + def __init__(self, io: IO, spec: XSpec) -> None: + """:private: + + The Trio session doing the Message IO is attached separately via + ``_attach_trio_session`` once the connection is established. + """ + super().__init__(io=io, id=spec.id, _startcount=1) + self.spec = spec + if spec.wait: + self._wait_backend = spec.wait + + @property + def remoteaddress(self) -> str: + # Only defined for remote IO types. + return self._io.remoteaddress # type: ignore[attr-defined,no-any-return] + + def __repr__(self) -> str: + """A string representing gateway type and status.""" + try: + r: str = (self.hasreceiver() and "receive-live") or "not-receiving" + i = str(len(self._channelfactory.channels())) + except AttributeError: + r = "uninitialized" + i = "no" + return f"<{self.__class__.__name__} id={self.id!r} {r}, {self.execmodel.backend} model, {i} active channels>" + + def exit(self) -> None: + """Trigger gateway exit. + + Defer waiting for finishing of receiver-thread and subprocess activity + to when group.terminate() is called. + """ + self._trace("gateway.exit() called") + if self not in self._group: + self._trace("gateway already unregistered with group") + return + self._group._unregister(self) + try: + self._trace("--> sending GATEWAY_TERMINATE") + self._send(Message.GATEWAY_TERMINATE) + self._trace("--> io.close_write") + self._io.close_write() + except (ValueError, EOFError, OSError) as exc: + self._trace("io-error: could not send termination sequence") + self._trace(" exception: %r" % exc) + + def _rinfo(self, update: bool = False) -> RInfo: + """Return some sys/env information from remote. + + A native protocol request (like ``remote_status``): it never + touches the exec machinery, so it cannot claim an exec slot on + main-thread-shaped workers. + """ + if update or not hasattr(self, "_cache_rinfo"): + channel = self.newchannel() + self._send(Message.GATEWAY_INFO, channel.id) + self._cache_rinfo = RInfo(channel.receive()) + # the other side didn't actually instantiate a channel + # so we just delete the internal id/channel mapping + self._channelfactory._local_close(channel.id) + return self._cache_rinfo + + def hasreceiver(self) -> bool: + """Whether gateway is able to receive data.""" + session = self._trio_session + return session is not None and bool(session.is_alive()) + + def remote_status(self) -> RemoteStatus: + """Obtain information about the remote execution status.""" + channel = self.newchannel() + self._send(Message.STATUS, channel.id) + statusdict = channel.receive() + # the other side didn't actually instantiate a channel + # so we just delete the internal id/channel mapping + self._channelfactory._local_close(channel.id) + return RemoteStatus(statusdict) + + def remote_exec( + self, + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + **kwargs: object, + ) -> Channel: + """Return channel object and connect it to a remote + execution thread where the given ``source`` executes. + + * ``source`` is a string: execute source string remotely + with a ``channel`` put into the global namespace. + * ``source`` is a pure function: serialize source and + call function with ``**kwargs``, adding a + ``channel`` object to the keyword arguments. + * ``source`` is a pure module: execute source of module + with a ``channel`` in its global namespace. + + In all cases the binding ``__name__='__channelexec__'`` + will be available in the global namespace of the remotely + executing code. + """ + source, file_name, call_name = normalize_exec_source(source, kwargs) + channel = self.newchannel() + self._send( + Message.CHANNEL_EXEC, + channel.id, + dumps_internal((source, file_name, call_name, kwargs)), + ) + return channel + + def remote_init_threads(self, num: int | None = None) -> None: + """DEPRECATED. Is currently a NO-OPERATION already.""" + print("WARNING: remote_init_threads() is a no-operation in execnet-1.2") + + +class RInfo: + def __init__(self, kwargs) -> None: + self.__dict__.update(kwargs) + + def __repr__(self) -> str: + info = ", ".join(f"{k}={v}" for k, v in sorted(self.__dict__.items())) + return "" % info + + if TYPE_CHECKING: + + def __getattr__(self, name: str) -> Any: ... + + +RemoteStatus = RInfo diff --git a/src/execnet/_gateway_base.py b/src/execnet/_gateway_base.py new file mode 100644 index 00000000..16b21602 --- /dev/null +++ b/src/execnet/_gateway_base.py @@ -0,0 +1,237 @@ +"""The gateway base classes shared by coordinator and worker. + +:class:`BaseGateway` owns the channel factory, the send path and the handle on +the Trio session that does the actual Message IO; :class:`WorkerGateway` adds +the worker-side ``remote_exec`` scheduling and shutdown. + +:copyright: 2004-2015 +:authors: + - Holger Krekel + - Armin Rigo + - Benjamin Peterson + - Ronny Pfannschmidt + - many others +""" + +from __future__ import annotations + +import os +import sys +import threading +from _thread import interrupt_main +from collections.abc import Callable +from contextlib import suppress +from typing import Any + +from ._boundary import Wakener +from ._boundary import make_wakener +from ._channel import Channel +from ._channel import ChannelFactory +from ._errors import INTERRUPT_TEXT +from ._errors import geterrortext +from ._errors import sysex +from ._message import IO +from ._message import Message +from ._trace import trace + + +class BaseGateway: + _sysex = sysex + id = "" + _trio_session: Any = None + # Set by the receiver on EOF without a prior termination message. + _error: BaseException | None = None + #: wait= axis: which wakener backend this gateway's blocking waits + #: park on (channels, write-acks, join) + _wait_backend: str = "thread" + + def __init__(self, io: IO, id, _startcount: int = 2) -> None: + self.execmodel = io.execmodel + self._io = io + self.id = id + self._channelfactory = ChannelFactory(self, _startcount) + # globals may be NONE at process-termination + self.__trace = trace + self._geterrortext = geterrortext + self._trio_session = None + + def _trace(self, *msg: object) -> None: + self.__trace(self.id, *msg) + + def _attach_trio_session(self, session: Any) -> None: + """Attach the Trio bridge session doing the Message IO.""" + self._trio_session = session + # Defensive: channels created before the session existed still + # need their inbound routing diverted to them. + for channel in self._channelfactory.channels(): + session.bind_sync_channel(channel) + + def _new_wakener(self) -> Wakener: + """A fresh wakener for one blocking-wait carrier (wait= axis).""" + return make_wakener(self._wait_backend) + + def _bind_channel(self, channel: Channel) -> None: + """Divert the session's inbound routing for ``channel.id`` to it.""" + session = self._trio_session + if session is not None: + session.bind_sync_channel(channel) + + def _release_channel(self, id: int) -> None: + """Drop the session's loop-side state for ``id`` (best-effort).""" + session = self._trio_session + if session is not None: + with suppress(Exception): + session.release_channel(id) + + def _run_on_loop(self, sync_fn: Callable[[], Any]) -> Any: + """Run ``sync_fn`` on the session's loop thread (inline without one). + + Payload dispatch happens on the loop thread, so state switches run + there to exclude interleaving with deliveries. + """ + session = self._trio_session + if session is None: + return sync_fn() + return session.run_on_loop(sync_fn) + + def _start_channel_consumer( + self, + channel: Channel, + callback: Callable[[Any], Any], + endmarker: object, + ) -> None: + """Attach a receiver callback: hand the channel to a consumer task. + + The task drains the channel on the loop and runs ``callback`` in a + threadpool thread, holding the channel alive while it consumes. + """ + session = self._trio_session + if session is None: + raise OSError(f"cannot set callback on {channel!r}: no active session") + session.attach_consumer(channel, callback, endmarker) + + def _terminate_execution(self) -> None: + pass + + def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: + message = Message(msgcode, channelid, data) + session = self._trio_session + if session is not None: + try: + session.enqueue_message(message) + self._trace("sent", message) + except (OSError, ValueError) as e: + self._trace("failed to send", message, e) + raise OSError("cannot send (already closed?)") from e + return + try: + message.to_io(self._io) + self._trace("sent", message) + except (OSError, ValueError) as e: + self._trace("failed to send", message, e) + # ValueError might be because the IO is already closed + raise OSError("cannot send (already closed?)") from e + + def _send_nonblocking(self, msgcode: int, channelid: int = 0) -> None: + """Best-effort send that never waits (used during GC). + + Safe to call from any thread, including while the interpreter or + the IO loop is shutting down. + """ + message = Message(msgcode, channelid) + session = self._trio_session + if session is not None: + session.post_message(message) + return + message.to_io(self._io) + + def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: + channel.close("execution disallowed") + + # _____________________________________________________________________ + # + # High Level Interface + # _____________________________________________________________________ + # + def newchannel(self) -> Channel: + """Return a new independent channel.""" + return self._channelfactory.new() + + def join(self, timeout: float | None = None) -> None: + """Wait for the receiver (Trio session) to terminate.""" + self._trace("waiting for receiver to finish") + session = self._trio_session + if session is not None: + session.wait_done(timeout) + + +class WorkerGateway(BaseGateway): + _trio_exec: Any = None + # The exec pool (a TrioWorkerExec duck-typed as WorkerPool for STATUS). + _execpool: Any = None + _executetask_complete: threading.Event | None = None + + def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: + trio_exec = self._trio_exec + if trio_exec is None: + channel.close("execution disallowed") + return + trio_exec.schedule(channel, sourcetask) + + def _terminate_execution(self) -> None: + # called from receiverthread + self._trace("shutting down execution pool") + self._execpool.trigger_shutdown() + if not self._execpool.waitall(5.0): + self._trace("execution ongoing after 5 secs, trying interrupt_main") + # We try hard to terminate execution based on the assumption + # that there is only one gateway object running per-process. + if sys.platform != "win32": + self._trace("sending ourselves a SIGINT") + os.kill(os.getpid(), 2) # send ourselves a SIGINT + elif interrupt_main is not None: + self._trace("calling interrupt_main()") + interrupt_main() + if not self._execpool.waitall(10.0): + self._trace( + "execution did not finish in another 10 secs, calling os._exit()" + ) + os._exit(1) + + def executetask( + self, + item: tuple[Channel, tuple[str, str | None, str | None, dict[str, object]]], + ) -> None: + try: + channel, (source, file_name, call_name, kwargs) = item + loc: dict[str, Any] = {"channel": channel, "__name__": "__channelexec__"} + self._trace(f"execution starts[{channel.id}]: {repr(source)[:50]}") + channel._executing = True + try: + co = compile(source + "\n", file_name or "", "exec") + exec(co, loc) + if call_name: + self._trace("calling %s(**%60r)" % (call_name, kwargs)) + function = loc[call_name] + function(channel, **kwargs) + finally: + channel._executing = False + self._trace("execution finished") + except KeyboardInterrupt: + channel.close(INTERRUPT_TEXT) + raise + except EOFError: + self._trace("ignoring EOFError because receiving finished") + + except BaseException as exc: + if not channel.gateway._channelfactory.finished: + self._trace(f"got exception: {exc!r}") + errortext = self._geterrortext(exc) + channel.close(errortext) + return + channel.close() + if self._executetask_complete is not None: + # Indicate that this task has finished executing, meaning + # that there is no possibility of it triggering a deadlock + # for the next spawn call. + self._executetask_complete.set() diff --git a/src/execnet/_message.py b/src/execnet/_message.py new file mode 100644 index 00000000..4d31339f --- /dev/null +++ b/src/execnet/_message.py @@ -0,0 +1,174 @@ +"""The wire protocol: IO protocols, Message framing and its decoder. + +A frame is a 9-byte header (``!bii``: message code, channel id, payload +length) followed by the payload. Nothing here dispatches -- routing lives in +``AsyncGateway._dispatch`` and the sync bridge session; this module only packs, +unpacks and buffers. +""" + +from __future__ import annotations + +import os +import struct +import sys +from collections.abc import Iterator +from typing import TYPE_CHECKING +from typing import Protocol + +if TYPE_CHECKING: + from ._execmodel import ExecModel + + +class WriteIO(Protocol): + def write(self, data: bytes, /) -> None: ... + + +class ReadIO(Protocol): + def read(self, numbytes: int, /) -> bytes: ... + + +class IO(Protocol): + execmodel: ExecModel + + def read(self, numbytes: int, /) -> bytes: ... + + def write(self, data: bytes, /) -> None: ... + + def close_read(self) -> None: ... + + def close_write(self) -> None: ... + + def wait(self) -> int | None: ... + + def kill(self) -> None: ... + + +class Message: + """Encapsulates Messages and their wire protocol. + + Dispatch lives in the async core and the sync bridge session + (``AsyncGateway._dispatch`` / ``SyncBridgeGateway._dispatch``); this + class only carries the framing and the code constants. + """ + + STATUS = 0 + #: retired: the py2/py3 string coercion switch. Nothing sends or + #: handles it anymore, the code stays reserved for reuse. + RECONFIGURE = 1 + GATEWAY_TERMINATE = 2 + CHANNEL_EXEC = 3 + CHANNEL_DATA = 4 + CHANNEL_CLOSE = 5 + CHANNEL_CLOSE_ERROR = 6 + CHANNEL_LAST_MESSAGE = 7 + GATEWAY_START_SOCKET = 8 + GATEWAY_START_SUB = 9 + GATEWAY_INFO = 10 + + # message code -> name + _types: dict[int, str] = { + STATUS: "STATUS", + RECONFIGURE: "RECONFIGURE", + GATEWAY_TERMINATE: "GATEWAY_TERMINATE", + CHANNEL_EXEC: "CHANNEL_EXEC", + CHANNEL_DATA: "CHANNEL_DATA", + CHANNEL_CLOSE: "CHANNEL_CLOSE", + CHANNEL_CLOSE_ERROR: "CHANNEL_CLOSE_ERROR", + CHANNEL_LAST_MESSAGE: "CHANNEL_LAST_MESSAGE", + GATEWAY_START_SOCKET: "GATEWAY_START_SOCKET", + GATEWAY_START_SUB: "GATEWAY_START_SUB", + GATEWAY_INFO: "GATEWAY_INFO", + } + + def __init__(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: + self.msgcode = msgcode + self.channelid = channelid + self.data = data + + def pack(self) -> bytes: + """Return the full wire frame (9-byte header + payload).""" + header = struct.pack("!bii", self.msgcode, self.channelid, len(self.data)) + return header + self.data + + @staticmethod + def from_header(header: bytes) -> tuple[int, int, int]: + """Unpack a 9-byte header into (msgtype, channelid, payload_len).""" + if len(header) != 9: + raise EOFError("couldn't load message header, short read") + msgtype, channel, payload = struct.unpack("!bii", header) + return msgtype, channel, payload + + @staticmethod + def from_parts(msgtype: int, channel: int, data: bytes) -> Message: + return Message(msgtype, channel, data) + + @staticmethod + def from_io(io: ReadIO) -> Message: + try: + header = io.read(9) # type 1, channel 4, payload 4 + if not header: + raise EOFError("empty read") + except EOFError as e: + raise EOFError("couldn't load message header, " + e.args[0]) from None + msgtype, channel, payload = Message.from_header(header) + return Message(msgtype, channel, io.read(payload)) + + def to_io(self, io: WriteIO) -> None: + io.write(self.pack()) + + def __repr__(self) -> str: + name = self._types[self.msgcode] + return f"" + + +def gateway_info() -> dict[str, object]: + """Payload for ``Message.GATEWAY_INFO``: sys/env facts about this side. + + Answered natively by the dispatch loop -- an info request never + touches the exec machinery, so it cannot claim an exec slot (with + main-thread profiles, an info call stealing the primary slot used to + push the real workload onto a worker thread). + """ + return { + "executable": sys.executable, + "version_info": tuple(sys.version_info[:5]), + "platform": sys.platform, + "cwd": os.getcwd(), + "pid": os.getpid(), + } + + +class FrameDecoder: + """Incremental decoder for the 9-byte-header Message framing. + + ``feed(data)`` accepts arbitrary byte chunks and yields every complete + Message; partial frames buffer internally until more bytes arrive. + Pure computation — no IO, no awaits, no knowledge of streams — so + receivers only ever stream bytes in (``receive_some`` loops) and the + decoder owns framing. + """ + + def __init__(self) -> None: + self._buffer = bytearray() + + def feed(self, data: bytes) -> Iterator[Message]: + self._buffer += data + return self._parse() + + def _parse(self) -> Iterator[Message]: + while len(self._buffer) >= 9: + msgtype, channelid, payload_len = Message.from_header( + bytes(self._buffer[:9]) + ) + if len(self._buffer) < 9 + payload_len: + return + payload = bytes(self._buffer[9 : 9 + payload_len]) + del self._buffer[: 9 + payload_len] + yield Message(msgtype, channelid, payload) + + def close(self) -> None: + """Signal EOF; raises EOFError if the stream ended mid-frame.""" + if self._buffer: + raise EOFError( + "connection closed mid-frame (%d buffered bytes)" % len(self._buffer) + ) diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py new file mode 100644 index 00000000..429103f7 --- /dev/null +++ b/src/execnet/_multi.py @@ -0,0 +1,419 @@ +""" +Managing Gateway Groups and interactions with multiple channels. + +(c) 2008-2014, Holger Krekel and others +""" + +from __future__ import annotations + +import atexit +import queue +import threading +import time +import types +from collections.abc import Callable +from collections.abc import Iterable +from collections.abc import Iterator +from collections.abc import Sequence +from contextlib import suppress +from threading import Lock +from typing import TYPE_CHECKING +from typing import Any +from typing import Literal +from typing import TypeAlias +from typing import overload + +from ._boundary import wakener_names +from ._channel import Channel +from ._execmodel import EXECMODEL_PROFILES +from ._execmodel import ExecModel +from ._execmodel import get_execmodel +from ._trace import trace +from ._xspec import XSpec + +if TYPE_CHECKING: + from ._gateway import Gateway + + +NO_ENDMARKER_WANTED = object() + + +class Group: + """Gateway Group.""" + + defaultspec = "popen" + + def __init__( + self, xspecs: Iterable[XSpec | str | None] = (), execmodel: str = "thread" + ) -> None: + """Initialize a group and make gateways as specified. + + execmodel can be one of the supported execution models. + """ + self._gateways: list[Gateway] = [] + self._autoidcounter = 0 + self._autoidlock = Lock() + self._gateways_to_join: list[Gateway] = [] + self._trio_host: Any = None + self._async_group: Any = None + # we use the same execmodel for all of the Gateway objects + # we spawn on our side. Probably we should not allow different + # execmodels between different groups but not clear. + # Note that "other side" execmodels may differ and is typically + # specified by the spec passed to makegateway. + self.set_execmodel(execmodel) + for xspec in xspecs: + self.makegateway(xspec) + atexit.register(self._cleanup_atexit) + + def _ensure_trio_host(self) -> Any: + if self._trio_host is None: + from . import _trio_host + + self._trio_host = _trio_host.TrioHost(name="execnet-trio-group") + self._trio_host.start() + return self._trio_host + + def _ensure_async_group(self) -> Any: + """The FacadeAsyncGroup owning the async side, running on the host.""" + if self._async_group is None: + from . import _trio_host + + host = self._ensure_trio_host() + + async def _start() -> Any: + async_group = _trio_host.FacadeAsyncGroup(self, host) + return await host._nursery.start(async_group.run) + + self._async_group = host.call(_start) + return self._async_group + + @property + def execmodel(self) -> ExecModel: + return self._execmodel + + @property + def remote_execmodel(self) -> ExecModel: + return self._remote_execmodel + + def set_execmodel( + self, execmodel: str, remote_execmodel: str | None = None + ) -> None: + """Set the execution model for local and remote site. + + execmodel can be one of the supported execution models. + It determines the execution model for any newly created gateway. + If remote_execmodel is not specified it takes on the value of execmodel. + + NOTE: Execution models can only be set before any gateway is created. + """ + if self._gateways: + raise ValueError( + "can not set execution models if gateways have been created already" + ) + if remote_execmodel is None: + remote_execmodel = execmodel + self._execmodel = get_execmodel(execmodel) + self._remote_execmodel = get_execmodel(remote_execmodel) + + def __repr__(self) -> str: + idgateways = [gw.id for gw in self] + return "" % idgateways + + def __getitem__(self, key: int | str | Gateway) -> Gateway: + if isinstance(key, int): + return self._gateways[key] + for gw in self._gateways: + if gw == key or gw.id == key: + return gw + raise KeyError(key) + + def __contains__(self, key: str) -> bool: + try: + self[key] + return True + except KeyError: + return False + + def __len__(self) -> int: + return len(self._gateways) + + def __iter__(self) -> Iterator[Gateway]: + return iter(list(self._gateways)) + + def makegateway(self, spec: XSpec | str | None = None) -> Gateway: + """Create and configure a gateway to a Python interpreter. + + The ``spec`` string encodes the target gateway type + and configuration information. The general format is:: + + key1=value1//key2=value2//... + + If you leave out the ``=value`` part a True value is assumed. + Valid types: ``popen``, ``ssh=hostname``, ``socket=host:port``. + Valid configuration:: + + id= specifies the gateway id + python= specifies which python interpreter to execute + execmodel=name worker profile: where exec'd code runs relative + to the worker's protocol loop. 'thread' (pool + threads) or 'main_thread_only' (serialized on + the worker main thread, GUI/signal-safe). + wait=backend wakener for blocking waits ('thread' default) + chdir= specifies to which directory to change + nice= specifies process priority of new process + env:NAME=value specifies a remote environment variable setting. + + If no spec is given, self.defaultspec is used. + """ + if not spec: + spec = self.defaultspec + if not isinstance(spec, XSpec): + spec = XSpec(spec) + self.allocate_id(spec) + if spec.execmodel is None: + spec.execmodel = self.remote_execmodel.backend + elif spec.execmodel not in EXECMODEL_PROFILES: + raise ValueError( + f"unknown execmodel {spec.execmodel!r}" + f" (known profiles: {list(EXECMODEL_PROFILES)})" + ) + if spec.wait is not None and spec.wait not in wakener_names(): + raise ValueError( + f"unknown wait backend {spec.wait!r} (known: {wakener_names()})" + ) + from . import _trio_host + + if not (spec.socket or spec.via or spec.ssh or spec.vagrant_ssh or spec.popen): + raise ValueError(f"no gateway type found for {spec._spec!r}") + gw = _trio_host.makegateway_trio(self, spec) + gw.spec = spec + self._register(gw) + # chdir/nice/env travel in the worker config and are applied at + # worker startup -- no remote_exec, so no exec slot is claimed. + return gw + + def allocate_id(self, spec: XSpec) -> None: + """(re-entrant) allocate id for the given xspec object.""" + if spec.id is None: + with self._autoidlock: + id = "gw" + str(self._autoidcounter) + self._autoidcounter += 1 + if id in self: + raise ValueError(f"already have gateway with id {id!r}") + spec.id = id + + def _register(self, gateway: Gateway) -> None: + assert not hasattr(gateway, "_group") + assert gateway.id + assert gateway.id not in self + self._gateways.append(gateway) + gateway._group = self + + def _unregister(self, gateway: Gateway) -> None: + self._gateways.remove(gateway) + self._gateways_to_join.append(gateway) + + def _cleanup_atexit(self) -> None: + trace(f"=== atexit cleanup {self!r} ===") + self.terminate(timeout=1.0) + if self._async_group is not None: + with suppress(Exception): + self._trio_host.call_sync(self._async_group.shutdown.set) + self._async_group = None + if self._trio_host is not None: + self._trio_host.stop(timeout=1.0) + self._trio_host = None + + def terminate(self, timeout: float | None = None) -> None: + """Trigger exit of member gateways and wait for termination + of member gateways and associated subprocesses. + + After waiting timeout seconds try to to kill local sub processes of + popen- and ssh-gateways. + + Timeout defaults to None meaning open-ended waiting and no kill + attempts. + """ + while self or self._gateways_to_join: + vias: set[str] = set() + for gw in self: + if gw.spec.via: + vias.add(gw.spec.via) + for gw in self: + if gw.id not in vias: + gw.exit() + if self._async_group is not None: + # Tunneled (via) gateways terminate before their masters, + # each with a GATEWAY_TERMINATE + timeout grace, then kill; + # bounded at roughly twice the timeout (issues #43 / #221). + try: + self._host_terminate(timeout) + except Exception as exc: + trace("group terminate error:", exc) + for gw in self._gateways_to_join: + gw.join() + self._gateways_to_join[:] = [] + + def _host_terminate(self, timeout: float | None) -> None: + """Terminate the async group, parking correctly for wait backends. + + A member gateway created with a non-thread ``wait=`` implies the + caller may be a greenlet: wait on a OneShot instead of blocking + the OS thread (which would stall the hub for the whole grace). + """ + backends = {gw._wait_backend for gw in self._gateways_to_join} | { + gw._wait_backend for gw in self + } + backends.discard("thread") + if backends: + from ._boundary import make_wakener + + self._trio_host.call_pending( + self._async_group.terminate, + timeout, + wakener=make_wakener(backends.pop()), + ).wait() + else: + self._trio_host.call(self._async_group.terminate, timeout) + + def remote_exec( + self, + source: str | types.FunctionType | Callable[..., object] | types.ModuleType, + **kwargs, + ) -> MultiChannel: + """remote_exec source on all member gateways and return + a MultiChannel connecting to all sub processes.""" + channels = [] + for gw in self: + channels.append(gw.remote_exec(source, **kwargs)) + return MultiChannel(channels) + + +class MultiChannel: + def __init__(self, channels: Sequence[Channel]) -> None: + self._channels = channels + + def __len__(self) -> int: + return len(self._channels) + + def __iter__(self) -> Iterator[Channel]: + return iter(self._channels) + + def __getitem__(self, key: int) -> Channel: + return self._channels[key] + + def __contains__(self, chan: Channel) -> bool: + return chan in self._channels + + def send_each(self, item: object) -> None: + for ch in self._channels: + ch.send(item) + + @overload + def receive_each(self, withchannel: Literal[False] = ...) -> list[Any]: + pass + + @overload + def receive_each(self, withchannel: Literal[True]) -> list[tuple[Channel, Any]]: + pass + + def receive_each( + self, withchannel: bool = False + ) -> list[tuple[Channel, Any]] | list[Any]: + assert not hasattr(self, "_queue") + l: list[object] = [] + for ch in self._channels: + obj = ch.receive() + if withchannel: + l.append((ch, obj)) + else: + l.append(obj) + return l + + def make_receive_queue(self, endmarker: object = NO_ENDMARKER_WANTED): + try: + return self._queue # type: ignore[has-type] + except AttributeError: + self._queue: queue.Queue[tuple[Channel, Any]] | None = None + for ch in self._channels: + if self._queue is None: + self._queue = queue.Queue() + + def putreceived(obj, channel: Channel = ch) -> None: + self._queue.put((channel, obj)) # type: ignore[union-attr] + + if endmarker is NO_ENDMARKER_WANTED: + ch.setcallback(putreceived) + else: + ch.setcallback(putreceived, endmarker=endmarker) + return self._queue + + def waitclose(self) -> None: + first = None + for ch in self._channels: + try: + ch.waitclose() + except ch.RemoteError as exc: + if first is None: + first = exc + if first: + raise first + + +TermKillFunc: TypeAlias = Callable[[], object] +TermKillPair: TypeAlias = tuple[TermKillFunc, TermKillFunc] + + +def safe_terminate( + execmodel: ExecModel, + timeout: float | None, + list_of_paired_functions: Sequence[TermKillPair], +) -> None: + """Run terminate/kill pairs in parallel with a hard wait bound. + + Each termfunc is given ``timeout``. If it does not finish, killfunc runs. + The final wait is also bounded so a stuck kill cannot hang the caller + forever (see issues #43 / #221). ``execmodel`` is accepted for + backward compatibility and unused (daemon threads do the waiting). + """ + errors: list[BaseException] = [] + + def termkill(termfunc: TermKillFunc, killfunc: TermKillFunc) -> None: + term_done = threading.Event() + term_errors: list[BaseException] = [] + + def run_term() -> None: + try: + termfunc() + except BaseException as exc: + term_errors.append(exc) + finally: + term_done.set() + + threading.Thread(target=run_term, daemon=True).start() + if not term_done.wait(timeout): + killfunc() + return + if term_errors: + errors.append(term_errors[0]) + + threads = [ + threading.Thread(target=termkill, args=pair, daemon=True) + for pair in list_of_paired_functions + ] + for thread in threads: + thread.start() + # Allow term timeout plus a kill attempt; never block indefinitely. + wait_timeout = None if timeout is None else timeout * 2 + deadline = None if wait_timeout is None else time.monotonic() + wait_timeout + for thread in threads: + remaining = None if deadline is None else max(0, deadline - time.monotonic()) + thread.join(remaining) + if errors: + raise errors[0] + + +default_group = Group() +makegateway = default_group.makegateway +set_execmodel = default_group.set_execmodel diff --git a/src/execnet/_rsync.py b/src/execnet/_rsync.py new file mode 100644 index 00000000..d14825f6 --- /dev/null +++ b/src/execnet/_rsync.py @@ -0,0 +1,248 @@ +""" +1:N rsync implementation on top of execnet. + +(c) 2006-2009, Armin Rigo, Holger Krekel, Maciej Fijalkowski +""" + +from __future__ import annotations + +import os +import stat +from collections.abc import Callable +from hashlib import md5 +from queue import Queue +from typing import Literal + +from execnet import _rsync_remote +from execnet._channel import Channel +from execnet._gateway import Gateway +from execnet._gateway_base import BaseGateway + + +class RSync: + """This class allows to send a directory structure (recursively) + to one or multiple remote filesystems. + + There is limited support for symlinks, which means that symlinks + pointing to the sourcetree will be send "as is" while external + symlinks will be just copied (regardless of existence of such + a path on remote side). + """ + + def __init__(self, sourcedir, callback=None, verbose: bool = True) -> None: + self._sourcedir = str(sourcedir) + self._verbose = verbose + assert callback is None or callable(callback) + self._callback = callback + self._channels: dict[Channel, Callable[[], None] | None] = {} + self._receivequeue: Queue[ + tuple[ + Channel, + ( + None + | tuple[Literal["send"], tuple[list[str], bytes]] + | tuple[Literal["list_done"], None] + | tuple[Literal["ack"], str] + | tuple[Literal["links"], None] + | tuple[Literal["done"], None] + ), + ] + ] = Queue() + self._links: list[tuple[Literal["linkbase", "link"], str, str]] = [] + + def filter(self, path: str) -> bool: + return True + + def _end_of_channel(self, channel: Channel) -> None: + if channel in self._channels: + # too early! we must have got an error + channel.waitclose() + # or else we raise one + raise OSError(f"connection unexpectedly closed: {channel.gateway} ") + + def _process_link(self, channel: Channel) -> None: + for link in self._links: + channel.send(link) + # completion marker, this host is done + channel.send(42) + + def _done(self, channel: Channel) -> None: + """Call all callbacks.""" + finishedcallback = self._channels.pop(channel) + if finishedcallback: + finishedcallback() + channel.waitclose() + + def _list_done(self, channel: Channel) -> None: + # sum up all to send + if self._callback: + s = sum([self._paths[i] for i in self._to_send[channel]]) + self._callback("list", s, channel) + + def _send_item( + self, + channel: Channel, + modified_rel_path_components: list[str], + checksum: bytes, + ) -> None: + """Send one item.""" + modified_path = os.path.join(self._sourcedir, *modified_rel_path_components) + try: + with open(modified_path, "rb") as fp: + data = fp.read() + except OSError: + data = None + + # provide info to progress callback function + modified_rel_path = "/".join(modified_rel_path_components) + if data is not None: + self._paths[modified_rel_path] = len(data) + else: + self._paths[modified_rel_path] = 0 + if channel not in self._to_send: + self._to_send[channel] = [] + self._to_send[channel].append(modified_rel_path) + # print "sending", modified_rel_path, data and len(data) or 0, checksum + + if data is not None: + if checksum is not None and checksum == md5(data).digest(): + data = None # not really modified + else: + self._report_send_file(channel.gateway, modified_rel_path) + channel.send(data) + + def _report_send_file(self, gateway: BaseGateway, modified_rel_path: str) -> None: + if self._verbose: + print(f"{gateway} <= {modified_rel_path}") + + def send(self, raises: bool = True) -> None: + """Sends a sourcedir to all added targets. + + raises indicates whether to raise an error or return in case of lack of + targets. + """ + if not self._channels: + if raises: + raise OSError( + "no targets available, maybe you are trying call send() twice?" + ) + return + # normalize a trailing '/' away + self._sourcedir = os.path.dirname(os.path.join(self._sourcedir, "x")) + # send directory structure and file timestamps/sizes + self._send_directory_structure(self._sourcedir) + + # paths and to_send are only used for doing + # progress-related callbacks + self._paths: dict[str, int] = {} + self._to_send: dict[Channel, list[str]] = {} + + # send modified file to clients + while self._channels: + channel, req = self._receivequeue.get() + if req is None: + self._end_of_channel(channel) + else: + if req[0] == "links": + self._process_link(channel) + elif req[0] == "done": + self._done(channel) + elif req[0] == "ack": + if self._callback: + self._callback("ack", self._paths[req[1]], channel) + elif req[0] == "list_done": + self._list_done(channel) + elif req[0] == "send": + self._send_item(channel, req[1][0], req[1][1]) + else: + assert "Unknown command %s" % req[0] # type: ignore[unreachable] + + def add_target( + self, + gateway: Gateway, + destdir: str | os.PathLike[str], + finishedcallback: Callable[[], None] | None = None, + **options, + ) -> None: + """Add a remote target specified via a gateway and a remote destination + directory.""" + for name in options: + assert name in ("delete",) + + def itemcallback(req) -> None: + self._receivequeue.put((channel, req)) + + channel = gateway.remote_exec(_rsync_remote) + channel.setcallback(itemcallback, endmarker=None) + channel.send((str(destdir), options)) + self._channels[channel] = finishedcallback + + def _broadcast(self, msg: object) -> None: + for channel in self._channels: + channel.send(msg) + + def _send_link( + self, + linktype: Literal["linkbase", "link"], + basename: str, + linkpoint: str, + ) -> None: + self._links.append((linktype, basename, linkpoint)) + + def _send_directory(self, path: str) -> None: + # dir: send a list of entries + names = [] + subpaths = [] + for name in os.listdir(path): + p = os.path.join(path, name) + if self.filter(p): + names.append(name) + subpaths.append(p) + mode = os.lstat(path).st_mode + self._broadcast([mode, *names]) + for p in subpaths: + self._send_directory_structure(p) + + def _send_link_structure(self, path: str) -> None: + sourcedir = self._sourcedir + basename = path[len(self._sourcedir) + 1 :] + linkpoint = os.readlink(path) + # On Windows, readlink returns an extended path (//?/) for + # absolute links, but relpath doesn't like mixing extended + # and non-extended paths. So fix it up ourselves. + if ( + os.path.__name__ == "ntpath" + and linkpoint.startswith("\\\\?\\") + and not self._sourcedir.startswith("\\\\?\\") + ): + sourcedir = "\\\\?\\" + self._sourcedir + try: + relpath = os.path.relpath(linkpoint, sourcedir) + except ValueError: + relpath = None + if ( + relpath is not None + and relpath not in (os.curdir, os.pardir) + and not relpath.startswith(os.pardir + os.sep) + ): + self._send_link("linkbase", basename, relpath) + else: + # relative or absolute link, just send it + self._send_link("link", basename, linkpoint) + self._broadcast(None) + + def _send_directory_structure(self, path: str) -> None: + try: + st = os.lstat(path) + except OSError: + self._broadcast((None, 0, 0)) + return + if stat.S_ISREG(st.st_mode): + # regular file: send a mode/timestamp/size pair + self._broadcast((st.st_mode, st.st_mtime, st.st_size)) + elif stat.S_ISDIR(st.st_mode): + self._send_directory(path) + elif stat.S_ISLNK(st.st_mode): + self._send_link_structure(path) + else: + raise ValueError(f"cannot sync {path!r}") diff --git a/src/execnet/_rsync_remote.py b/src/execnet/_rsync_remote.py new file mode 100644 index 00000000..bc2e4f41 --- /dev/null +++ b/src/execnet/_rsync_remote.py @@ -0,0 +1,126 @@ +""" +(c) 2006-2013, Armin Rigo, Holger Krekel, Maciej Fijalkowski +""" + +from __future__ import annotations + +from contextlib import suppress +from typing import TYPE_CHECKING +from typing import Literal +from typing import cast + +if TYPE_CHECKING: + from execnet._channel import Channel + + +def serve_rsync(channel: Channel) -> None: + import os + import shutil + import stat + from hashlib import md5 + + destdir, options = cast("tuple[str, dict[str, object]]", channel.receive()) + modifiedfiles = [] + + def remove(path: str) -> None: + assert path.startswith(destdir) + try: + os.unlink(path) + except OSError: + # assume it's a dir + shutil.rmtree(path, True) + + def receive_directory_structure(path: str, relcomponents: list[str]) -> None: + try: + st = os.lstat(path) + except OSError: + st = None + msg = channel.receive() + if isinstance(msg, list): + if st and not stat.S_ISDIR(st.st_mode): + os.unlink(path) + st = None + if not st: + os.makedirs(path) + mode = msg.pop(0) + if mode: + # Ensure directories are writable, otherwise a + # permission denied error (EACCES) would be raised + # when attempting to receive read-only directory + # structures. + os.chmod(path, mode | 0o700) + entrynames = {} + for entryname in msg: + destpath = os.path.join(path, entryname) + receive_directory_structure(destpath, [*relcomponents, entryname]) + entrynames[entryname] = True + if options.get("delete"): + for othername in os.listdir(path): + if othername not in entrynames: + otherpath = os.path.join(path, othername) + remove(otherpath) + elif msg is not None: + assert isinstance(msg, tuple) + checksum = None + if st: + if stat.S_ISREG(st.st_mode): + msg_mode, msg_mtime, msg_size = msg + if msg_size != st.st_size: + pass + elif msg_mtime != st.st_mtime: + with open(path, "rb") as fp: + checksum = md5(fp.read()).digest() + elif msg_mode and msg_mode != st.st_mode: + os.chmod(path, msg_mode | 0o700) + return + else: + return # already fine + else: + remove(path) + channel.send(("send", (relcomponents, checksum))) + modifiedfiles.append((path, msg)) + + receive_directory_structure(destdir, []) + + STRICT_CHECK = False # seems most useful this way for py.test + channel.send(("list_done", None)) + + for path, (mode, time, size) in modifiedfiles: + data = cast(bytes, channel.receive()) + channel.send(("ack", path[len(destdir) + 1 :])) + if data is not None: + if STRICT_CHECK and len(data) != size: + raise OSError(f"file modified during rsync: {path!r}") + with open(path, "wb") as fp: + fp.write(data) + try: + if mode: + os.chmod(path, mode) + os.utime(path, (time, time)) + except OSError: + pass + del data + channel.send(("links", None)) + + msg = channel.receive() + while msg != 42: + # we get symlink + _type, relpath, linkpoint = cast( + "tuple[Literal['linkbase', 'link'], str, str]", msg + ) + path = os.path.join(destdir, relpath) + with suppress(OSError): + remove(path) + + if _type == "linkbase": + src = os.path.join(destdir, linkpoint) + else: + assert _type == "link", _type + src = linkpoint + os.symlink(src, path) + msg = channel.receive() + channel.send(("done", None)) + + +if __name__ == "__channelexec__": + serve_rsync(channel) # type: ignore[name-defined] # noqa:F821 diff --git a/src/execnet/_serialize.py b/src/execnet/_serialize.py new file mode 100644 index 00000000..3151efe3 --- /dev/null +++ b/src/execnet/_serialize.py @@ -0,0 +1,439 @@ +"""The wire serializer for execnet's simple builtin data format. + +Internal: no public namespace re-exports ``dumps``/``loads``. Callers that +need to know whether a value can cross a channel use +:func:`execnet.can_send`, which is exported from every public namespace. + +The channel layer is *not* imported here -- ``Unserializer`` resolves a +channel or gateway argument by duck-typing -- so ``_channel`` may depend on +this module and not the other way round. +""" + +from __future__ import annotations + +import struct +from collections.abc import Callable +from io import BytesIO +from typing import TYPE_CHECKING +from typing import Any + +from ._errors import DumpError +from ._errors import LoadError + +if TYPE_CHECKING: + from ._channel import Channel + from ._gateway_base import BaseGateway + from ._message import ReadIO + + +def bchr(n: int) -> bytes: + return bytes([n]) + + +DUMPFORMAT_VERSION = bchr(2) + +FOUR_BYTE_INT_MAX = 2147483647 +FOUR_BYTE_INT_MIN = -2147483648 + +FLOAT_FORMAT = "!d" +FLOAT_FORMAT_SIZE = struct.calcsize(FLOAT_FORMAT) +COMPLEX_FORMAT = "!dd" +COMPLEX_FORMAT_SIZE = struct.calcsize(COMPLEX_FORMAT) + + +class _Stop(Exception): + pass + + +class opcode: + """Container for name -> num mappings.""" + + BUILDTUPLE = b"@" + BYTES = b"A" + CHANNEL = b"B" + FALSE = b"C" + FLOAT = b"D" + FROZENSET = b"E" + INT = b"F" + LONG = b"G" + LONGINT = b"H" + LONGLONG = b"I" + NEWDICT = b"J" + NEWLIST = b"K" + NONE = b"L" + STRING = b"N" + SET = b"O" + SETITEM = b"P" + STOP = b"Q" + TRUE = b"R" + COMPLEX = b"T" + + +class Unserializer: + num2func: dict[bytes, Callable[[Unserializer], None]] = {} + + def __init__( + self, + stream: ReadIO, + channel_or_gateway: Channel | BaseGateway | None = None, + ) -> None: + # A channel -- sync or trio-native -- resolves through its gateway; a + # gateway is already the right object. Duck-typed so the serializer + # stays independent of the channel layer. + gw: Any = getattr(channel_or_gateway, "gateway", channel_or_gateway) + self.stream = stream + if gw is None: + self.channelfactory = None + else: + self.channelfactory = gw._channelfactory + + def load(self, versioned: bool = False) -> Any: + if versioned: + ver = self.stream.read(1) + if ver != DUMPFORMAT_VERSION: + raise LoadError("wrong dumpformat version %r" % ver) + self.stack: list[object] = [] + try: + while True: + opcode = self.stream.read(1) + if not opcode: + raise EOFError + try: + loader = self.num2func[opcode] + except KeyError: + raise LoadError( + f"unknown opcode {opcode!r} - wire protocol corruption?" + ) from None + loader(self) + except _Stop: + if len(self.stack) != 1: + raise LoadError("internal unserialization error") from None + return self.stack.pop(0) + else: + raise LoadError("didn't get STOP") + + def load_none(self) -> None: + self.stack.append(None) + + num2func[opcode.NONE] = load_none + + def load_true(self) -> None: + self.stack.append(True) + + num2func[opcode.TRUE] = load_true + + def load_false(self) -> None: + self.stack.append(False) + + num2func[opcode.FALSE] = load_false + + def load_int(self) -> None: + i = self._read_int4() + self.stack.append(i) + + num2func[opcode.INT] = load_int + + def load_longint(self) -> None: + s = self._read_byte_string() + self.stack.append(int(s)) + + num2func[opcode.LONGINT] = load_longint + + load_long = load_int + num2func[opcode.LONG] = load_long + load_longlong = load_longint + num2func[opcode.LONGLONG] = load_longlong + + def load_float(self) -> None: + binary = self.stream.read(FLOAT_FORMAT_SIZE) + self.stack.append(struct.unpack(FLOAT_FORMAT, binary)[0]) + + num2func[opcode.FLOAT] = load_float + + def load_complex(self) -> None: + binary = self.stream.read(COMPLEX_FORMAT_SIZE) + self.stack.append(complex(*struct.unpack(COMPLEX_FORMAT, binary))) + + num2func[opcode.COMPLEX] = load_complex + + def _read_int4(self) -> int: + value: int = struct.unpack("!i", self.stream.read(4))[0] + return value + + def _read_byte_string(self) -> bytes: + length = self._read_int4() + as_bytes = self.stream.read(length) + return as_bytes + + def load_string(self) -> None: + self.stack.append(self._read_byte_string().decode("utf-8")) + + num2func[opcode.STRING] = load_string + + def load_bytes(self) -> None: + s = self._read_byte_string() + self.stack.append(s) + + num2func[opcode.BYTES] = load_bytes + + def load_newlist(self) -> None: + length = self._read_int4() + self.stack.append([None] * length) + + num2func[opcode.NEWLIST] = load_newlist + + def load_setitem(self) -> None: + if len(self.stack) < 3: + raise LoadError("not enough items for setitem") + value = self.stack.pop() + key = self.stack.pop() + self.stack[-1][key] = value # type: ignore[index] + + num2func[opcode.SETITEM] = load_setitem + + def load_newdict(self) -> None: + self.stack.append({}) + + num2func[opcode.NEWDICT] = load_newdict + + def _load_collection(self, type_: type) -> None: + length = self._read_int4() + if length: + res = type_(self.stack[-length:]) + del self.stack[-length:] + self.stack.append(res) + else: + self.stack.append(type_()) + + def load_buildtuple(self) -> None: + self._load_collection(tuple) + + num2func[opcode.BUILDTUPLE] = load_buildtuple + + def load_set(self) -> None: + self._load_collection(set) + + num2func[opcode.SET] = load_set + + def load_frozenset(self) -> None: + self._load_collection(frozenset) + + num2func[opcode.FROZENSET] = load_frozenset + + def load_stop(self) -> None: + raise _Stop + + num2func[opcode.STOP] = load_stop + + def load_channel(self) -> None: + id = self._read_int4() + assert self.channelfactory is not None + newchannel = self.channelfactory.new(id) + self.stack.append(newchannel) + + num2func[opcode.CHANNEL] = load_channel + + +def dumps(obj: object) -> bytes: + """Serialize the given obj to a bytestring. + + The obj and all contained objects must be of a builtin + Python type (so nested dicts, sets, etc. are all OK but + not user-level instances). + """ + return _Serializer().save(obj, versioned=True) # type: ignore[return-value] + + +def dump(byteio, obj: object) -> None: + """write a serialized bytestring of the given obj to the given stream.""" + _Serializer(write=byteio.write).save(obj, versioned=True) + + +def loads(bytestring: bytes) -> Any: + """Deserialize the given bytestring to an object. + + If the bytestring was dumped with an incompatible protocol + version or if the bytestring is corrupted, the + ``execnet.DataFormatError`` will be raised. + """ + return load(BytesIO(bytestring)) + + +def load(io: ReadIO) -> Any: + """Derserialize an object form the specified stream. + + Behaviour is otherwise the same as with ``loads`` + """ + return Unserializer(io).load(versioned=True) + + +def loads_internal(bytestring: bytes, channelfactory=None) -> Any: + io = BytesIO(bytestring) + return Unserializer(io, channelfactory).load() + + +def dumps_internal(obj: object) -> bytes: + return _Serializer().save(obj) # type: ignore[return-value] + + +def can_send(obj: object) -> bool: + """Whether ``obj`` can cross a channel as-is. + + True for execnet's simple builtin wire data -- ``None``, ``bool``, + ``int``, ``float``, ``complex``, ``bytes``, ``str`` and arbitrarily + nested ``list``/``tuple``/``set``/``frozenset``/``dict`` of those -- + and for channel references. False for anything execnet has no wire + representation for, which ``channel.send`` would reject with + :class:`~execnet.DumpError`. + + Use it to branch *before* sending, instead of sending and handling the + error:: + + channel.send(value if execnet.can_send(value) else repr(value)) + """ + try: + _Serializer().save(obj) + except DumpError: + return False + return True + + +class _Serializer: + _dispatch: dict[type, Callable[[_Serializer, object], None]] = {} + + def __init__(self, write: Callable[[bytes], None] | None = None) -> None: + if write is None: + self._streamlist: list[bytes] = [] + write = self._streamlist.append + self._write = write + + def save(self, obj: object, versioned: bool = False) -> bytes | None: + # calling here is not re-entrant but multiple instances + # may write to the same stream because of the common platform + # atomic-write guarantee (concurrent writes each happen atomically) + if versioned: + self._write(DUMPFORMAT_VERSION) + self._save(obj) + self._write(opcode.STOP) + try: + streamlist = self._streamlist + except AttributeError: + return None + return b"".join(streamlist) + + def _save(self, obj: object) -> None: + tp = type(obj) + try: + dispatch = self._dispatch[tp] + except KeyError: + methodname = "save_" + tp.__name__ + meth: Callable[[_Serializer, object], None] | None = getattr( + self.__class__, methodname, None + ) + if meth is None: + raise DumpError(f"can't serialize {tp}") from None + dispatch = self._dispatch[tp] = meth + dispatch(self, obj) + + def save_NoneType(self, non: None) -> None: + self._write(opcode.NONE) + + def save_bool(self, boolean: bool) -> None: + if boolean: + self._write(opcode.TRUE) + else: + self._write(opcode.FALSE) + + def save_bytes(self, bytes_: bytes) -> None: + self._write(opcode.BYTES) + self._write_byte_sequence(bytes_) + + def save_str(self, s: str) -> None: + self._write(opcode.STRING) + self._write_unicode_string(s) + + def _write_unicode_string(self, s: str) -> None: + try: + as_bytes = s.encode("utf-8") + except UnicodeEncodeError as e: + raise DumpError("strings must be utf-8 encodable") from e + self._write_byte_sequence(as_bytes) + + def _write_byte_sequence(self, bytes_: bytes) -> None: + self._write_int4(len(bytes_), "string is too long") + self._write(bytes_) + + def _save_integral(self, i: int, short_op: bytes, long_op: bytes) -> None: + # The short op packs a signed 4-byte int; anything outside that range + # (in either direction) goes through the arbitrary-precision long op. + if FOUR_BYTE_INT_MIN <= i <= FOUR_BYTE_INT_MAX: + self._write(short_op) + self._write_int4(i) + else: + self._write(long_op) + self._write_byte_sequence(str(i).rstrip("L").encode("ascii")) + + def save_int(self, i: int) -> None: + self._save_integral(i, opcode.INT, opcode.LONGINT) + + def save_long(self, l: int) -> None: + self._save_integral(l, opcode.LONG, opcode.LONGLONG) + + def save_float(self, flt: float) -> None: + self._write(opcode.FLOAT) + self._write(struct.pack(FLOAT_FORMAT, flt)) + + def save_complex(self, cpx: complex) -> None: + self._write(opcode.COMPLEX) + self._write(struct.pack(COMPLEX_FORMAT, cpx.real, cpx.imag)) + + def _write_int4( + self, i: int, error: str = "int must be less than %i" % (FOUR_BYTE_INT_MAX,) + ) -> None: + if i > FOUR_BYTE_INT_MAX: + raise DumpError(error) + self._write(struct.pack("!i", i)) + + def save_list(self, L: list[object]) -> None: + self._write(opcode.NEWLIST) + self._write_int4(len(L), "list is too long") + for i, item in enumerate(L): + self._write_setitem(i, item) + + def _write_setitem(self, key: object, value: object) -> None: + self._save(key) + self._save(value) + self._write(opcode.SETITEM) + + def save_dict(self, d: dict[object, object]) -> None: + self._write(opcode.NEWDICT) + for key, value in d.items(): + self._write_setitem(key, value) + + def save_tuple(self, tup: tuple[object, ...]) -> None: + for item in tup: + self._save(item) + self._write(opcode.BUILDTUPLE) + self._write_int4(len(tup), "tuple is too long") + + def _write_set(self, s: set[object] | frozenset[object], op: bytes) -> None: + for item in s: + self._save(item) + self._write(op) + self._write_int4(len(s), "set is too long") + + def save_set(self, s: set[object]) -> None: + self._write_set(s, opcode.SET) + + def save_frozenset(self, s: frozenset[object]) -> None: + self._write_set(s, opcode.FROZENSET) + + def save_Channel(self, channel: Channel) -> None: + self._write(opcode.CHANNEL) + self._write_int4(channel.id) + + def save_AsyncChannel(self, channel: Any) -> None: + # trio-native channel (execnet._trio_gateway); same wire opcode, + # duck-typed here to avoid importing the async core. + self._write(opcode.CHANNEL) + self._write_int4(channel.id) diff --git a/src/execnet/_shim.py b/src/execnet/_shim.py new file mode 100644 index 00000000..35e7682c --- /dev/null +++ b/src/execnet/_shim.py @@ -0,0 +1,47 @@ +"""Machinery for the deprecated pre-Trio module names. + +``execnet.gateway_base``, ``execnet.gateway``, ``execnet.multi``, +``execnet.rsync``, ``execnet.rsync_remote`` and ``execnet.xspec`` were never +part of a documented public API -- they were importable only because +``import execnet`` pulled them in transitively. Their contents now live in +private modules grouped by concern; each old name survives as a thin module +whose ``__getattr__`` warns and forwards. + +The supported surfaces are :mod:`execnet` / :mod:`execnet.sync`, +:mod:`execnet.trio`, :mod:`execnet.aio` and :mod:`execnet.portal`. +""" + +from __future__ import annotations + +import importlib +import warnings +from typing import Any + +#: shims are scheduled for removal in this release +REMOVED_IN = "execnet 3.0" + + +def forwarder(shim: str, moved: dict[str, str]) -> Any: + """Build the ``__getattr__`` for a deprecated module. + + ``moved`` maps each previously-reachable name to the private module that + now defines it (a relative name such as ``"._channel"``). + """ + + def __getattr__(name: str) -> Any: + try: + module = moved[name] + except KeyError: + raise AttributeError( + f"module 'execnet.{shim}' has no attribute {name!r}" + ) from None + warnings.warn( + f"execnet.{shim} is private and will be removed in {REMOVED_IN}; " + f"{name} now lives in execnet{module}. The supported surfaces are " + f"execnet, execnet.sync, execnet.trio, execnet.aio and execnet.portal.", + DeprecationWarning, + stacklevel=2, + ) + return getattr(importlib.import_module(module, __package__), name) + + return __getattr__ diff --git a/src/execnet/_trace.py b/src/execnet/_trace.py new file mode 100644 index 00000000..edb650d8 --- /dev/null +++ b/src/execnet/_trace.py @@ -0,0 +1,47 @@ +"""Debug tracing, configured once from ``EXECNET_DEBUG``. + +:EXECNET_DEBUG=1: write per-process trace files to ``execnet-debug-PID`` +:EXECNET_DEBUG=2: trace to stderr (popen workers forward this to their + instantiator) + +Unset, ``trace`` is a no-op lambda so tracing costs a call and nothing else. +""" + +from __future__ import annotations + +import os +import sys + +DEBUG = os.environ.get("EXECNET_DEBUG") +pid = os.getpid() + +if DEBUG == "2": + + def trace(*msg: object) -> None: + try: + line = " ".join(map(str, msg)) + sys.stderr.write(f"[{pid}] {line}\n") + sys.stderr.flush() + except Exception: + pass # nothing we can do, likely interpreter-shutdown + +elif DEBUG: + import tempfile + + fn = os.path.join(tempfile.gettempdir(), "execnet-debug-%d" % pid) + # sys.stderr.write("execnet-debug at %r" % (fn,)) + debugfile = open(fn, "w") + + def trace(*msg: object) -> None: + try: + line = " ".join(map(str, msg)) + debugfile.write(line + "\n") + debugfile.flush() + except Exception as exc: + try: + sys.stderr.write(f"[{pid}] exception during tracing: {exc!r}\n") + except Exception: + pass # nothing we can do, likely interpreter-shutdown + +else: + notrace = trace = lambda *msg: None diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 3972db4f..90246fb7 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -1,6 +1,7 @@ """Trio-native gateway core: async dispatch loop and low-level raw channels. -Async-first counterpart of the sync machinery in ``gateway_base``: an +Async-first counterpart of the sync machinery in ``_channel`` / +``_gateway_base``: an :class:`AsyncGateway` owns a :class:`ByteStream` and runs a single dispatch task (stream -> ``FrameDecoder`` -> route). Message handlers execute inline on that task, so there is no receiver thread and no receive lock. @@ -37,17 +38,17 @@ if TYPE_CHECKING: from typing_extensions import Self +from ._errors import GatewayReceivedTerminate +from ._errors import HostNotFound +from ._errors import RemoteError +from ._errors import TimeoutError from ._exec_source import normalize_exec_source -from .gateway_base import FrameDecoder -from .gateway_base import GatewayReceivedTerminate -from .gateway_base import HostNotFound -from .gateway_base import Message -from .gateway_base import RemoteError -from .gateway_base import TimeoutError -from .gateway_base import dumps_internal -from .gateway_base import gateway_info -from .gateway_base import loads_internal -from .gateway_base import trace +from ._message import FrameDecoder +from ._message import Message +from ._message import gateway_info +from ._serialize import dumps_internal +from ._serialize import loads_internal +from ._trace import trace RECEIVE_CHUNK = 65536 @@ -898,7 +899,7 @@ async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: ``installvia=``), and ``via=`` sub-gateways relayed through a group member. """ - from .xspec import XSpec + from ._xspec import XSpec if self._nursery is None: raise RuntimeError(f"{self!r} is not entered") diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 1d9149dd..a1ea4acb 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -26,6 +26,17 @@ import trio from ._boundary import Flag +from ._channel import ENDMARKER +from ._channel import NO_ENDMARKER_WANTED +from ._errors import GatewayReceivedTerminate +from ._errors import RemoteError +from ._execmodel import ExecModel +from ._message import FrameDecoder +from ._message import Message +from ._message import gateway_info +from ._serialize import dumps_internal +from ._serialize import loads_internal +from ._trace import trace from ._trio_gateway import RECEIVE_CHUNK from ._trio_gateway import AsyncGateway from ._trio_gateway import AsyncGroup @@ -34,17 +45,6 @@ from ._trio_gateway import open_popen_process from ._trio_gateway import read_handshake_ack from ._trio_gateway import ssh_transport_args -from .gateway_base import ENDMARKER -from .gateway_base import NO_ENDMARKER_WANTED -from .gateway_base import ExecModel -from .gateway_base import FrameDecoder -from .gateway_base import GatewayReceivedTerminate -from .gateway_base import Message -from .gateway_base import RemoteError -from .gateway_base import dumps_internal -from .gateway_base import gateway_info -from .gateway_base import loads_internal -from .gateway_base import trace from .portal import LoopPortal from .portal import OneShot @@ -60,9 +60,9 @@ def _run_callback(callback: Callable[[Any], Any], data: bytes, channel: Any) -> if TYPE_CHECKING: - from .gateway import Gateway - from .gateway_base import BaseGateway - from .multi import Group + from ._gateway import Gateway + from ._gateway_base import BaseGateway + from ._multi import Group T = TypeVar("T") diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 4f3a65f2..633b7d94 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -12,18 +12,18 @@ import trio -from .gateway_base import MAIN_THREAD_ONLY_DEADLOCK_TEXT -from .gateway_base import WorkerGateway -from .gateway_base import get_execmodel -from .gateway_base import geterrortext -from .gateway_base import loads_internal -from .gateway_base import trace +from ._errors import MAIN_THREAD_ONLY_DEADLOCK_TEXT +from ._errors import geterrortext +from ._execmodel import get_execmodel +from ._gateway_base import WorkerGateway +from ._serialize import loads_internal +from ._trace import trace from .portal import Mailbox if TYPE_CHECKING: from . import _trio_host - from .gateway_base import Channel - from .gateway_base import ExecModel + from ._channel import Channel + from ._execmodel import ExecModel ExecItem = tuple[Any, ...] diff --git a/src/execnet/_xspec.py b/src/execnet/_xspec.py new file mode 100644 index 00000000..afc58f4c --- /dev/null +++ b/src/execnet/_xspec.py @@ -0,0 +1,74 @@ +""" +(c) 2008-2013, holger krekel +""" + +from __future__ import annotations + + +class XSpec: + """Execution Specification: key1=value1//key2=value2 ... + + * Keys need to be unique within the specification scope + * Neither key nor value are allowed to contain "//" + * Keys are not allowed to contain "=" + * Keys are not allowed to start with underscore + * If no "=value" is given, assume a boolean True value + """ + + # XXX allow customization, for only allow specific key names + chdir: str | None = None + dont_write_bytecode: bool | None = None + execmodel: str | None = None + id: str | None = None + installvia: str | None = None + nice: str | None = None + popen: bool | None = None + python: str | None = None + socket: str | None = None + ssh: str | None = None + ssh_config: str | None = None + vagrant_ssh: str | None = None + via: str | None = None + wait: str | None = None + + def __init__(self, string: str) -> None: + self._spec = string + self.env = {} + for keyvalue in string.split("//"): + i = keyvalue.find("=") + value: str | bool + if i == -1: + key, value = keyvalue, True + else: + key, value = keyvalue[:i], keyvalue[i + 1 :] + if key[0] == "_": + raise AttributeError("%r not a valid XSpec key" % key) + if key in self.__dict__: + raise ValueError(f"duplicate key: {key!r} in {string!r}") + if key.startswith("env:"): + self.env[key[4:]] = value + else: + setattr(self, key, value) + + def __getattr__(self, name: str) -> None | bool | str: + if name[0] == "_": + raise AttributeError(name) + return None + + def __repr__(self) -> str: + return f"" + + def __str__(self) -> str: + return self._spec + + def __hash__(self) -> int: + return hash(self._spec) + + def __eq__(self, other: object) -> bool: + return self._spec == getattr(other, "_spec", None) + + def __ne__(self, other: object) -> bool: + return self._spec != getattr(other, "_spec", None) + + def _samefilesystem(self) -> bool: + return self.popen is not None and self.chdir is None diff --git a/src/execnet/aio.py b/src/execnet/aio.py index 7151ab5d..36bd8751 100644 --- a/src/execnet/aio.py +++ b/src/execnet/aio.py @@ -26,7 +26,8 @@ async def main(): The error types are shared with :mod:`execnet.sync` and :mod:`execnet.trio`. Items you send must already be simple builtin data (plus channels); the standalone serializer is intentionally not part of -the public API -- see ``DumpError``. +the public API -- ``execnet.can_send`` checks a value before you send it; +see ``DumpError``. """ from __future__ import annotations @@ -45,17 +46,17 @@ async def main(): import trio +from ._errors import DataFormatError +from ._errors import DumpError +from ._errors import HostNotFound +from ._errors import LoadError +from ._errors import RemoteError +from ._errors import TimeoutError from ._trio_gateway import AsyncChannel from ._trio_gateway import AsyncGateway from ._trio_gateway import AsyncGroup from ._trio_host import TrioHost -from .gateway_base import DataFormatError -from .gateway_base import DumpError -from .gateway_base import HostNotFound -from .gateway_base import LoadError -from .gateway_base import RemoteError -from .gateway_base import TimeoutError -from .xspec import XSpec +from ._xspec import XSpec if TYPE_CHECKING: from typing_extensions import Self diff --git a/src/execnet/gateway.py b/src/execnet/gateway.py index ac41705c..d15263ca 100644 --- a/src/execnet/gateway.py +++ b/src/execnet/gateway.py @@ -1,161 +1,19 @@ -"""Gateway code for initiating popen, socket and ssh connections. +"""Deprecated alias for :mod:`execnet._gateway`. -(c) 2004-2013, Holger Krekel and others +``Gateway`` is exported from :mod:`execnet` and :mod:`execnet.sync`; use those. """ from __future__ import annotations -import types -from collections.abc import Callable -from typing import TYPE_CHECKING -from typing import Any +from ._shim import forwarder -from . import gateway_base -from ._exec_source import _find_non_builtin_globals -from ._exec_source import _source_of_function -from ._exec_source import normalize_exec_source -from .gateway_base import IO -from .gateway_base import Channel -from .gateway_base import Message -from .multi import Group -from .xspec import XSpec +_MOVED = { + "Gateway": "._gateway", + "RInfo": "._gateway", + "RemoteStatus": "._gateway", + "normalize_exec_source": "._exec_source", + "_find_non_builtin_globals": "._exec_source", + "_source_of_function": "._exec_source", +} -__all__ = [ - "Gateway", - "RInfo", - "RemoteStatus", - "_find_non_builtin_globals", - "_source_of_function", -] - - -class Gateway(gateway_base.BaseGateway): - """Gateway to a local or remote Python Interpreter.""" - - _group: Group - - def __init__(self, io: IO, spec: XSpec) -> None: - """:private: - - The Trio session doing the Message IO is attached separately via - ``_attach_trio_session`` once the connection is established. - """ - super().__init__(io=io, id=spec.id, _startcount=1) - self.spec = spec - if spec.wait: - self._wait_backend = spec.wait - - @property - def remoteaddress(self) -> str: - # Only defined for remote IO types. - return self._io.remoteaddress # type: ignore[attr-defined,no-any-return] - - def __repr__(self) -> str: - """A string representing gateway type and status.""" - try: - r: str = (self.hasreceiver() and "receive-live") or "not-receiving" - i = str(len(self._channelfactory.channels())) - except AttributeError: - r = "uninitialized" - i = "no" - return f"<{self.__class__.__name__} id={self.id!r} {r}, {self.execmodel.backend} model, {i} active channels>" - - def exit(self) -> None: - """Trigger gateway exit. - - Defer waiting for finishing of receiver-thread and subprocess activity - to when group.terminate() is called. - """ - self._trace("gateway.exit() called") - if self not in self._group: - self._trace("gateway already unregistered with group") - return - self._group._unregister(self) - try: - self._trace("--> sending GATEWAY_TERMINATE") - self._send(Message.GATEWAY_TERMINATE) - self._trace("--> io.close_write") - self._io.close_write() - except (ValueError, EOFError, OSError) as exc: - self._trace("io-error: could not send termination sequence") - self._trace(" exception: %r" % exc) - - def _rinfo(self, update: bool = False) -> RInfo: - """Return some sys/env information from remote. - - A native protocol request (like ``remote_status``): it never - touches the exec machinery, so it cannot claim an exec slot on - main-thread-shaped workers. - """ - if update or not hasattr(self, "_cache_rinfo"): - channel = self.newchannel() - self._send(Message.GATEWAY_INFO, channel.id) - self._cache_rinfo = RInfo(channel.receive()) - # the other side didn't actually instantiate a channel - # so we just delete the internal id/channel mapping - self._channelfactory._local_close(channel.id) - return self._cache_rinfo - - def hasreceiver(self) -> bool: - """Whether gateway is able to receive data.""" - session = self._trio_session - return session is not None and bool(session.is_alive()) - - def remote_status(self) -> RemoteStatus: - """Obtain information about the remote execution status.""" - channel = self.newchannel() - self._send(Message.STATUS, channel.id) - statusdict = channel.receive() - # the other side didn't actually instantiate a channel - # so we just delete the internal id/channel mapping - self._channelfactory._local_close(channel.id) - return RemoteStatus(statusdict) - - def remote_exec( - self, - source: str | types.FunctionType | Callable[..., object] | types.ModuleType, - **kwargs: object, - ) -> Channel: - """Return channel object and connect it to a remote - execution thread where the given ``source`` executes. - - * ``source`` is a string: execute source string remotely - with a ``channel`` put into the global namespace. - * ``source`` is a pure function: serialize source and - call function with ``**kwargs``, adding a - ``channel`` object to the keyword arguments. - * ``source`` is a pure module: execute source of module - with a ``channel`` in its global namespace. - - In all cases the binding ``__name__='__channelexec__'`` - will be available in the global namespace of the remotely - executing code. - """ - source, file_name, call_name = normalize_exec_source(source, kwargs) - channel = self.newchannel() - self._send( - Message.CHANNEL_EXEC, - channel.id, - gateway_base.dumps_internal((source, file_name, call_name, kwargs)), - ) - return channel - - def remote_init_threads(self, num: int | None = None) -> None: - """DEPRECATED. Is currently a NO-OPERATION already.""" - print("WARNING: remote_init_threads() is a no-operation in execnet-1.2") - - -class RInfo: - def __init__(self, kwargs) -> None: - self.__dict__.update(kwargs) - - def __repr__(self) -> str: - info = ", ".join(f"{k}={v}" for k, v in sorted(self.__dict__.items())) - return "" % info - - if TYPE_CHECKING: - - def __getattr__(self, name: str) -> Any: ... - - -RemoteStatus = RInfo +__getattr__ = forwarder("gateway", _MOVED) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index ce19eb0f..b54c68bf 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -1,1449 +1,79 @@ -"""Core gateway, channel and serialization code shared by coordinator and worker. +"""Deprecated: the pre-Trio core module, now split by concern. -:copyright: 2004-2015 -:authors: - - Holger Krekel - - Armin Rigo - - Benjamin Peterson - - Ronny Pfannschmidt - - many others +``execnet.gateway_base`` was never a supported public API. Its contents live +in private modules -- ``_trace``, ``_errors``, ``_execmodel``, ``_message``, +``_serialize``, ``_channel`` and ``_gateway_base`` -- and this shim forwards +to them with a :class:`DeprecationWarning`. + +Use :mod:`execnet` / :mod:`execnet.sync`, :mod:`execnet.trio`, +:mod:`execnet.aio` or :mod:`execnet.portal` instead. The standalone +serializer stays internal; :func:`execnet.can_send` answers "can this value +cross a channel?" without it. """ from __future__ import annotations -import builtins -import os -import struct -import sys -import threading -import traceback -import weakref -from _thread import interrupt_main -from collections.abc import Callable -from collections.abc import Iterator -from contextlib import suppress -from io import BytesIO -from typing import Any -from typing import Literal -from typing import Protocol -from typing import cast -from typing import overload - -from ._boundary import Flag -from ._boundary import Mailbox -from ._boundary import Wakener -from ._boundary import make_wakener - - -class WriteIO(Protocol): - def write(self, data: bytes, /) -> None: ... - - -class ReadIO(Protocol): - def read(self, numbytes: int, /) -> bytes: ... - - -class IO(Protocol): - execmodel: ExecModel - - def read(self, numbytes: int, /) -> bytes: ... - - def write(self, data: bytes, /) -> None: ... - - def close_read(self) -> None: ... - - def close_write(self) -> None: ... - - def wait(self) -> int | None: ... - - def kill(self) -> None: ... - - -class ExecModel: - """Deprecated preset name for an execution model. - - The machinery behind execution models was retired: protocol IO always - runs on the Trio host and blocking waits go through the boundary kit's - wakeners (``execnet._boundary``); the name maps onto the worker config - axes (``loop=`` / ``exec=`` / ``wait=``). The stdlib-delegating - members stay for API compatibility (pytest-xdist builds its test queue - on ``execmodel.RLock``/``Event``) -- every preset is thread-shaped. - """ - - def __init__(self, backend: str) -> None: - self.backend = backend - - def __repr__(self) -> str: - return "" % self.backend - - @property - def queue(self): - import queue - - return queue - - @property - def subprocess(self): - import subprocess - - return subprocess - - @property - def socket(self): - import socket - - return socket - - def get_ident(self) -> int: - import _thread - - return _thread.get_ident() - - def sleep(self, delay: float) -> None: - import time - - time.sleep(delay) - - def start(self, func, args=()) -> None: - import _thread - - _thread.start_new_thread(func, args) - - def fdopen(self, fd, mode, bufsize=1, closefd=True): - return os.fdopen(fd, mode, bufsize, encoding="utf-8", closefd=closefd) - - def Lock(self): - return threading.RLock() - - def RLock(self): - return threading.RLock() - - def Event(self) -> threading.Event: - return threading.Event() - - -#: worker profiles: where exec'd code runs relative to the protocol loop -EXECMODEL_PROFILES = ( - "thread", # hybrid: primary on the main thread, overflow on pool threads - "main_thread_only", # exec serialized on the main thread (GUI/pytest) - "trio", # pure async: loop owns the main thread, async sources as tasks - "gevent", # greenlets on a main-thread hub, one per remote_exec -) - - -def get_execmodel(backend: str | ExecModel) -> ExecModel: - if isinstance(backend, ExecModel): - return backend - if backend in EXECMODEL_PROFILES: - return ExecModel(backend) - raise ValueError(f"unknown execmodel {backend!r}") - - -sysex = (KeyboardInterrupt, SystemExit) - - -DEBUG = os.environ.get("EXECNET_DEBUG") -pid = os.getpid() -if DEBUG == "2": - - def trace(*msg: object) -> None: - try: - line = " ".join(map(str, msg)) - sys.stderr.write(f"[{pid}] {line}\n") - sys.stderr.flush() - except Exception: - pass # nothing we can do, likely interpreter-shutdown - -elif DEBUG: - import os - import tempfile - - fn = os.path.join(tempfile.gettempdir(), "execnet-debug-%d" % pid) - # sys.stderr.write("execnet-debug at %r" % (fn,)) - debugfile = open(fn, "w") - - def trace(*msg: object) -> None: - try: - line = " ".join(map(str, msg)) - debugfile.write(line + "\n") - debugfile.flush() - except Exception as exc: - try: - sys.stderr.write(f"[{pid}] exception during tracing: {exc!r}\n") - except Exception: - pass # nothing we can do, likely interpreter-shutdown - -else: - notrace = trace = lambda *msg: None - - -class Message: - """Encapsulates Messages and their wire protocol. - - Dispatch lives in the async core and the sync bridge session - (``AsyncGateway._dispatch`` / ``SyncBridgeGateway._dispatch``); this - class only carries the framing and the code constants. - """ - - STATUS = 0 - #: retired: the py2/py3 string coercion switch. Nothing sends or - #: handles it anymore, the code stays reserved for reuse. - RECONFIGURE = 1 - GATEWAY_TERMINATE = 2 - CHANNEL_EXEC = 3 - CHANNEL_DATA = 4 - CHANNEL_CLOSE = 5 - CHANNEL_CLOSE_ERROR = 6 - CHANNEL_LAST_MESSAGE = 7 - GATEWAY_START_SOCKET = 8 - GATEWAY_START_SUB = 9 - GATEWAY_INFO = 10 - - # message code -> name - _types: dict[int, str] = { - STATUS: "STATUS", - RECONFIGURE: "RECONFIGURE", - GATEWAY_TERMINATE: "GATEWAY_TERMINATE", - CHANNEL_EXEC: "CHANNEL_EXEC", - CHANNEL_DATA: "CHANNEL_DATA", - CHANNEL_CLOSE: "CHANNEL_CLOSE", - CHANNEL_CLOSE_ERROR: "CHANNEL_CLOSE_ERROR", - CHANNEL_LAST_MESSAGE: "CHANNEL_LAST_MESSAGE", - GATEWAY_START_SOCKET: "GATEWAY_START_SOCKET", - GATEWAY_START_SUB: "GATEWAY_START_SUB", - GATEWAY_INFO: "GATEWAY_INFO", - } - - def __init__(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: - self.msgcode = msgcode - self.channelid = channelid - self.data = data - - def pack(self) -> bytes: - """Return the full wire frame (9-byte header + payload).""" - header = struct.pack("!bii", self.msgcode, self.channelid, len(self.data)) - return header + self.data - - @staticmethod - def from_header(header: bytes) -> tuple[int, int, int]: - """Unpack a 9-byte header into (msgtype, channelid, payload_len).""" - if len(header) != 9: - raise EOFError("couldn't load message header, short read") - msgtype, channel, payload = struct.unpack("!bii", header) - return msgtype, channel, payload - - @staticmethod - def from_parts(msgtype: int, channel: int, data: bytes) -> Message: - return Message(msgtype, channel, data) - - @staticmethod - def from_io(io: ReadIO) -> Message: - try: - header = io.read(9) # type 1, channel 4, payload 4 - if not header: - raise EOFError("empty read") - except EOFError as e: - raise EOFError("couldn't load message header, " + e.args[0]) from None - msgtype, channel, payload = Message.from_header(header) - return Message(msgtype, channel, io.read(payload)) - - def to_io(self, io: WriteIO) -> None: - io.write(self.pack()) - - def __repr__(self) -> str: - name = self._types[self.msgcode] - return f"" - - -def gateway_info() -> dict[str, object]: - """Payload for ``Message.GATEWAY_INFO``: sys/env facts about this side. - - Answered natively by the dispatch loop -- an info request never - touches the exec machinery, so it cannot claim an exec slot (with - main-thread profiles, an info call stealing the primary slot used to - push the real workload onto a worker thread). - """ - return { - "executable": sys.executable, - "version_info": tuple(sys.version_info[:5]), - "platform": sys.platform, - "cwd": os.getcwd(), - "pid": os.getpid(), - } - - -class FrameDecoder: - """Incremental decoder for the 9-byte-header Message framing. - - ``feed(data)`` accepts arbitrary byte chunks and yields every complete - Message; partial frames buffer internally until more bytes arrive. - Pure computation — no IO, no awaits, no knowledge of streams — so - receivers only ever stream bytes in (``receive_some`` loops) and the - decoder owns framing. - """ - - def __init__(self) -> None: - self._buffer = bytearray() - - def feed(self, data: bytes) -> Iterator[Message]: - self._buffer += data - return self._parse() - - def _parse(self) -> Iterator[Message]: - while len(self._buffer) >= 9: - msgtype, channelid, payload_len = Message.from_header( - bytes(self._buffer[:9]) - ) - if len(self._buffer) < 9 + payload_len: - return - payload = bytes(self._buffer[9 : 9 + payload_len]) - del self._buffer[: 9 + payload_len] - yield Message(msgtype, channelid, payload) - - def close(self) -> None: - """Signal EOF; raises EOFError if the stream ended mid-frame.""" - if self._buffer: - raise EOFError( - "connection closed mid-frame (%d buffered bytes)" % len(self._buffer) - ) - - -class GatewayReceivedTerminate(Exception): - """Receiver got a gateway termination message.""" - - -class HostNotFound(ConnectionError): - """The remote side of a gateway could not be reached.""" - - -def geterrortext( - exc: BaseException, - format_exception=traceback.format_exception, - sysex: tuple[type[BaseException], ...] = sysex, -) -> str: - try: - # In py310, can change this to: - # l = format_exception(exc) - l = format_exception(type(exc), exc, exc.__traceback__) - errortext = "".join(l) - except sysex: - raise - except BaseException: - errortext = f"{type(exc).__name__}: {exc}" - return errortext - - -class RemoteError(Exception): - """Exception containing a stringified error from the other side.""" - - def __init__(self, formatted: str) -> None: - super().__init__() - self.formatted = formatted - - def __str__(self) -> str: - return self.formatted - - def __repr__(self) -> str: - return f"{self.__class__.__name__}: {self.formatted}" - - def warn(self) -> None: - if self.formatted != INTERRUPT_TEXT: - # XXX do this better - sys.stderr.write(f"[{os.getpid()}] Warning: unhandled {self!r}\n") - - -class TimeoutError(IOError): - """Exception indicating that a timeout was reached.""" - - -NO_ENDMARKER_WANTED = object() - - -class Channel: - """Communication channel between two Python Interpreter execution points. - - A facade over the async core: the gateway's Trio session diverts - inbound payloads for this id into a :class:`Mailbox` (or a registered - callback, invoked on the loop thread); ``receive()`` deserializes at - the call site. Sends go through ``gateway._send``. - """ - - RemoteError = RemoteError - TimeoutError = TimeoutError - _INTERNALWAKEUP = 1000 - _executing = False - #: set once a receiver callback is attached. A consumer *task* on the - #: loop drains this channel and runs the callback in a threadpool thread; - #: the task holds the channel alive, so a callback channel's lifecycle is - #: bound to consumption (and to GC once the stream closes) rather than to - #: a strong registry. - _has_consumer = False - #: loop-side hooks installed while a consumer is attached: divert one - #: payload into / close the consumer task's inbox (set by the Trio session, - #: so they encapsulate the trio memory channel; gateway_base stays trio-free). - _consumer_feed: Callable[[bytes], None] | None = None - _consumer_close_inbox: Callable[[], None] | None = None - #: thread-safe "stop the consumer" hook -- ends the task's inbox. - _consumer_stop: Callable[[], None] | None = None - #: set by the consumer task once it has drained every item and fired the - #: endmarker; ``waitclose()`` waits on this (instead of ``_receiveclosed``) - #: so it still guarantees "all callbacks have run" before returning. - _consumer_done: Flag | None = None - - def __init__(self, gateway: BaseGateway, id: int) -> None: - """:private:""" - assert isinstance(id, int) - assert not isinstance(gateway, type) - self.gateway = gateway - self.id = id - # serialized payloads (or ENDMARKER); None once a consumer is attached - self._mailbox: Mailbox[Any] | None = Mailbox(gateway._new_wakener()) - self._closed = False - self._receiveclosed = Flag(gateway._new_wakener()) - self._remoteerrors: list[RemoteError] = [] - - def _trace(self, *msg: object) -> None: - self.gateway._trace(self.id, *msg) - - def setcallback( - self, - callback: Callable[[Any], Any], - endmarker: object = NO_ENDMARKER_WANTED, - ) -> None: - """Set a callback function for receiving items. - - A consumer task on the gateway's loop drains this channel and runs - ``callback`` for each received item in a threadpool thread (so a slow - callback never blocks the loop); items for one channel are delivered - strictly in order. Already-queued items are delivered first. After - this call ``receive()`` raises an error. - - The task keeps the channel alive for as long as it is consuming, so a - callback channel need not be referenced elsewhere. If an endmarker is - specified the callback is eventually called with it when the channel - closes, and ``waitclose()`` does not return until every callback - (including the endmarker) has run. - """ - self.gateway._start_channel_consumer(self, callback, endmarker) - - def __repr__(self) -> str: - flag = (self.isclosed() and "closed") or "open" - return "" % (self.id, flag) - - def __del__(self) -> None: - if self.gateway is None: # can be None in tests - return # type: ignore[unreachable] - - self._trace("channel.__del__") - # no multithreading issues here, because we have the last ref to 'self' - if self._closed: - # state transition "closed" --> "deleted" - for error in self._remoteerrors: - error.warn() - elif self._receiveclosed.is_set(): - # state transition "sendonly" --> "deleted" - # the remote channel is already in "deleted" state, nothing to do - pass - else: - # state transition "opened" --> "deleted" - # check if we are in the middle of interpreter shutdown - # in which case the process will go away and we probably - # don't need to try to send a closing or last message - # (and often it won't work anymore to send things out) - # A callback channel is held by its consumer task until the stream - # closes, so by the time __del__ runs it is never in the "opened" - # state -- this branch only ever fires for a plain receive channel. - if Message is not None: - with suppress(OSError, ValueError): # ignore problems with sending - # Never wait during GC: post the close best-effort. - send = getattr( - self.gateway, "_send_nonblocking", self.gateway._send - ) - send(Message.CHANNEL_CLOSE, self.id) - with suppress(Exception): - self.gateway._release_channel(self.id) - - def _getremoteerror(self): - try: - return self._remoteerrors.pop(0) - except IndexError: - try: - return self.gateway._error - except AttributeError: - pass - return None - - # - # loop-side delivery (called by the session's raw-channel consumer) - # - def _deliver_payload(self, data: bytes) -> None: - """Route one inbound serialized payload (loop thread). - - A callback channel diverts payloads into its consumer task's inbox; a - plain channel queues them for ``receive()``. - """ - if self._closed: - return # late data for a locally closed channel: drop - feed = self._consumer_feed - if feed is not None: - feed(data) - return - mailbox = self._mailbox - if mailbox is not None: - mailbox.put(data) - # no consumer and no mailbox: closed for receiving -- drop - - def _close_from_remote(self, remoteerror=None, *, sendonly: bool = False) -> None: - """Close initiated by the peer or session shutdown (loop thread). - - For a callback channel the consumer task ends separately (its inbox is - closed) and fires the endmarker; here we only record the state. - """ - if remoteerror: - self._remoteerrors.append(remoteerror) - if self._has_consumer: - close_inbox = self._consumer_close_inbox - if close_inbox is not None: - close_inbox() # ends the consumer task (it fires the endmarker) - else: - mailbox = self._mailbox - if mailbox is not None: - mailbox.put(ENDMARKER) - self.gateway._channelfactory._no_longer_opened(self.id) - if not sendonly: # otherwise #--> "sendonly" - self._closed = True # --> "closed" - self._receiveclosed.set() - - # - # public API for channel objects - # - def isclosed(self) -> bool: - """Return True if the channel is closed. - - A closed channel may still hold items. - """ - return self._closed - - @overload - def makefile(self, mode: Literal["r"], proxyclose: bool = ...) -> ChannelFileRead: - pass - - @overload - def makefile( - self, - mode: Literal["w"] = ..., - proxyclose: bool = ..., - ) -> ChannelFileWrite: - pass - - def makefile( - self, - mode: Literal["r", "w"] = "w", - proxyclose: bool = False, - ) -> ChannelFileWrite | ChannelFileRead: - """Return a file-like object. - - mode can be 'w' or 'r' for writeable/readable files. - If proxyclose is true, file.close() will also close the channel. - """ - if mode == "w": - return ChannelFileWrite(channel=self, proxyclose=proxyclose) - elif mode == "r": - return ChannelFileRead(channel=self, proxyclose=proxyclose) - raise ValueError(f"mode {mode!r} not available") - - def close(self, error=None) -> None: - """Close down this channel with an optional error message. - - Note that closing of a channel tied to remote_exec happens - automatically at the end of execution and cannot - be done explicitly. - """ - if self._executing: - raise OSError("cannot explicitly close channel within remote_exec") - if self._closed: - self.gateway._trace(self, "ignoring redundant call to close()") - if not self._closed: - # state transition "opened/sendonly" --> "closed" - # threads warning: the channel might be closed under our feet, - # but it's never damaging to send too many CHANNEL_CLOSE messages - # however, if the other side triggered a close already, we - # do not send back a closed message. - if not self._receiveclosed.is_set(): - put = self.gateway._send - if error is not None: - put(Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(error)) - else: - put(Message.CHANNEL_CLOSE, self.id) - self._trace("sent channel close message") - if isinstance(error, RemoteError): - self._remoteerrors.append(error) - self._closed = True # --> "closed" - self._receiveclosed.set() - if self._has_consumer: - # End the consumer task's inbox; it drains any buffered items, - # fires the endmarker, and sets _consumer_done. - stop = self._consumer_stop - if stop is not None: - stop() - else: - mailbox = self._mailbox - if mailbox is not None: - mailbox.put(ENDMARKER) - self.gateway._channelfactory._no_longer_opened(self.id) - self.gateway._release_channel(self.id) - - def waitclose(self, timeout: float | None = None) -> None: - """Wait until this channel is closed (or the remote side - otherwise signalled that no more data was being sent). - - The channel may still hold receiveable items, but not receive - any more after waitclose() has returned. - - Exceptions from executing code on the other side are reraised as local - channel.RemoteErrors. - - EOFError is raised if the reading-connection was prematurely closed, - which often indicates a dying process. - - self.TimeoutError is raised after the specified number of seconds - (default is None, i.e. wait indefinitely). - """ - # For a callback channel wait on the consumer task finishing (so every - # callback, including the endmarker, has run); otherwise wait for the - # non-"opened" state directly. - signal = ( - self._consumer_done - if self._consumer_done is not None - else (self._receiveclosed) - ) - signal.wait(timeout=timeout) - if not signal.is_set(): - raise self.TimeoutError("Timeout after %r seconds" % timeout) - error = self._getremoteerror() - if error: - raise error - - def send(self, item: object) -> None: - """Sends the given item to the other side of the channel, - possibly blocking if the sender queue is full. - - The item must be a simple Python type and will be - copied to the other side by value. - - OSError is raised if the write pipe was prematurely closed. - """ - if self.isclosed(): - raise OSError(f"cannot send to {self!r}") - self.gateway._send(Message.CHANNEL_DATA, self.id, dumps_internal(item)) - - def receive(self, timeout: float | None = None) -> Any: - """Receive a data item that was sent from the other side. - - timeout: None [default] blocked waiting. A positive number - indicates the number of seconds after which a channel.TimeoutError - exception will be raised if no item was received. - - Note that exceptions from the remotely executing code will be - reraised as channel.RemoteError exceptions containing - a textual representation of the remote traceback. - """ - mailbox = self._mailbox - if mailbox is None: - raise OSError("cannot receive(), channel has receiver callback") - try: - x = mailbox.get(timeout) - except builtins.TimeoutError: - raise self.TimeoutError("no item after %r seconds" % timeout) from None - if x is ENDMARKER: - mailbox.put(x) # for other receivers - raise self._getremoteerror() or EOFError() - else: - return loads_internal(x, self) - - def __iter__(self) -> Iterator[Any]: - return self - - def next(self) -> Any: - try: - return self.receive() - except EOFError: - raise StopIteration from None - - __next__ = next - - -ENDMARKER = object() -INTERRUPT_TEXT = "keyboard-interrupted" -MAIN_THREAD_ONLY_DEADLOCK_TEXT = ( - "concurrent remote_exec would cause deadlock for main_thread_only execmodel" -) - - -class ChannelFactory: - """Registry and id allocator for a gateway's sync channels. - - Message routing lives in the Trio session (the sync channel binds a - consumer on the session's raw channel); the factory only tracks live - channels -- weakly, so dropping the last user reference triggers - ``Channel.__del__``'s close message. A callback channel is kept alive by - its consumer task rather than by any registry here. - """ - - def __init__(self, gateway: BaseGateway, startcount: int = 1) -> None: - self._channels: weakref.WeakValueDictionary[int, Channel] = ( - weakref.WeakValueDictionary() - ) - self._writelock = threading.Lock() - self.gateway = gateway - self.count = startcount - self.finished = False - self._list = list # needed during interp-shutdown - - def new(self, id: int | None = None) -> Channel: - """Create a new Channel with 'id' (or create new id if None).""" - with self._writelock: - if self.finished: - raise OSError(f"connection already closed: {self.gateway}") - if id is None: - id = self.count - self.count += 2 - try: - channel = self._channels[id] - except KeyError: - channel = self._channels[id] = Channel(self.gateway, id) - self.gateway._bind_channel(channel) - return channel - - def allocate_id(self) -> int: - """Reserve a fresh channel id without creating a Channel object.""" - with self._writelock: - if self.finished: - raise OSError(f"connection already closed: {self.gateway}") - id = self.count - self.count += 2 - return id - - def channels(self) -> list[Channel]: - return self._list(self._channels.values()) - - # - # internal methods, called from the loop thread (or local close paths) - # - def _no_longer_opened(self, id: int) -> None: - self._channels.pop(id, None) - - def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> None: - """Close ``id`` as if the peer had closed it (no message is sent).""" - channel = self._channels.get(id) - if channel is None: - # channel already in "deleted" state - if remoteerror: - remoteerror.warn() - self._no_longer_opened(id) - else: - channel._close_from_remote(remoteerror, sendonly=sendonly) - - def _finished_receiving(self) -> None: - with self._writelock: - self.finished = True - for id in self._list(self._channels): - self._local_close(id, sendonly=True) - - -class ChannelFile: - def __init__(self, channel: Channel, proxyclose: bool = True) -> None: - self.channel = channel - self._proxyclose = proxyclose - - def isatty(self) -> bool: - return False - - def close(self) -> None: - if self._proxyclose: - self.channel.close() - - def __repr__(self) -> str: - state = (self.channel.isclosed() and "closed") or "open" - return "" % (self.channel.id, state) - - -class ChannelFileWrite(ChannelFile): - def write(self, out: bytes) -> None: - self.channel.send(out) - - def flush(self) -> None: - pass - - -class ChannelFileRead(ChannelFile): - def __init__(self, channel: Channel, proxyclose: bool = True) -> None: - super().__init__(channel, proxyclose) - self._buffer: str | None = None - - def read(self, n: int) -> str: - try: - if self._buffer is None: - self._buffer = cast(str, self.channel.receive()) - while len(self._buffer) < n: - self._buffer += cast(str, self.channel.receive()) - except EOFError: - self.close() - if self._buffer is None: - ret = "" - else: - ret = self._buffer[:n] - self._buffer = self._buffer[n:] - return ret - - def readline(self) -> str: - if self._buffer is not None: - i = self._buffer.find("\n") - if i != -1: - return self.read(i + 1) - line = self.read(len(self._buffer) + 1) - else: - line = self.read(1) - while line and line[-1] != "\n": - c = self.read(1) - if not c: - break - line += c - return line - - -class BaseGateway: - _sysex = sysex - id = "" - _trio_session: Any = None - # Set by the receiver on EOF without a prior termination message. - _error: BaseException | None = None - #: wait= axis: which wakener backend this gateway's blocking waits - #: park on (channels, write-acks, join) - _wait_backend: str = "thread" - - def __init__(self, io: IO, id, _startcount: int = 2) -> None: - self.execmodel = io.execmodel - self._io = io - self.id = id - self._channelfactory = ChannelFactory(self, _startcount) - # globals may be NONE at process-termination - self.__trace = trace - self._geterrortext = geterrortext - self._trio_session = None - - def _trace(self, *msg: object) -> None: - self.__trace(self.id, *msg) - - def _attach_trio_session(self, session: Any) -> None: - """Attach the Trio bridge session doing the Message IO.""" - self._trio_session = session - # Defensive: channels created before the session existed still - # need their inbound routing diverted to them. - for channel in self._channelfactory.channels(): - session.bind_sync_channel(channel) - - def _new_wakener(self) -> Wakener: - """A fresh wakener for one blocking-wait carrier (wait= axis).""" - return make_wakener(self._wait_backend) - - def _bind_channel(self, channel: Channel) -> None: - """Divert the session's inbound routing for ``channel.id`` to it.""" - session = self._trio_session - if session is not None: - session.bind_sync_channel(channel) - - def _release_channel(self, id: int) -> None: - """Drop the session's loop-side state for ``id`` (best-effort).""" - session = self._trio_session - if session is not None: - with suppress(Exception): - session.release_channel(id) - - def _run_on_loop(self, sync_fn: Callable[[], Any]) -> Any: - """Run ``sync_fn`` on the session's loop thread (inline without one). - - Payload dispatch happens on the loop thread, so state switches run - there to exclude interleaving with deliveries. - """ - session = self._trio_session - if session is None: - return sync_fn() - return session.run_on_loop(sync_fn) - - def _start_channel_consumer( - self, - channel: Channel, - callback: Callable[[Any], Any], - endmarker: object, - ) -> None: - """Attach a receiver callback: hand the channel to a consumer task. - - The task drains the channel on the loop and runs ``callback`` in a - threadpool thread, holding the channel alive while it consumes. - """ - session = self._trio_session - if session is None: - raise OSError(f"cannot set callback on {channel!r}: no active session") - session.attach_consumer(channel, callback, endmarker) - - def _terminate_execution(self) -> None: - pass - - def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: - message = Message(msgcode, channelid, data) - session = self._trio_session - if session is not None: - try: - session.enqueue_message(message) - self._trace("sent", message) - except (OSError, ValueError) as e: - self._trace("failed to send", message, e) - raise OSError("cannot send (already closed?)") from e - return - try: - message.to_io(self._io) - self._trace("sent", message) - except (OSError, ValueError) as e: - self._trace("failed to send", message, e) - # ValueError might be because the IO is already closed - raise OSError("cannot send (already closed?)") from e - - def _send_nonblocking(self, msgcode: int, channelid: int = 0) -> None: - """Best-effort send that never waits (used during GC). - - Safe to call from any thread, including while the interpreter or - the IO loop is shutting down. - """ - message = Message(msgcode, channelid) - session = self._trio_session - if session is not None: - session.post_message(message) - return - message.to_io(self._io) - - def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: - channel.close("execution disallowed") - - # _____________________________________________________________________ - # - # High Level Interface - # _____________________________________________________________________ - # - def newchannel(self) -> Channel: - """Return a new independent channel.""" - return self._channelfactory.new() - - def join(self, timeout: float | None = None) -> None: - """Wait for the receiver (Trio session) to terminate.""" - self._trace("waiting for receiver to finish") - session = self._trio_session - if session is not None: - session.wait_done(timeout) - - -class WorkerGateway(BaseGateway): - _trio_exec: Any = None - # The exec pool (a TrioWorkerExec duck-typed as WorkerPool for STATUS). - _execpool: Any = None - _executetask_complete: threading.Event | None = None - - def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: - trio_exec = self._trio_exec - if trio_exec is None: - channel.close("execution disallowed") - return - trio_exec.schedule(channel, sourcetask) - - def _terminate_execution(self) -> None: - # called from receiverthread - self._trace("shutting down execution pool") - self._execpool.trigger_shutdown() - if not self._execpool.waitall(5.0): - self._trace("execution ongoing after 5 secs, trying interrupt_main") - # We try hard to terminate execution based on the assumption - # that there is only one gateway object running per-process. - if sys.platform != "win32": - self._trace("sending ourselves a SIGINT") - os.kill(os.getpid(), 2) # send ourselves a SIGINT - elif interrupt_main is not None: - self._trace("calling interrupt_main()") - interrupt_main() - if not self._execpool.waitall(10.0): - self._trace( - "execution did not finish in another 10 secs, calling os._exit()" - ) - os._exit(1) - - def executetask( - self, - item: tuple[Channel, tuple[str, str | None, str | None, dict[str, object]]], - ) -> None: - try: - channel, (source, file_name, call_name, kwargs) = item - loc: dict[str, Any] = {"channel": channel, "__name__": "__channelexec__"} - self._trace(f"execution starts[{channel.id}]: {repr(source)[:50]}") - channel._executing = True - try: - co = compile(source + "\n", file_name or "", "exec") - exec(co, loc) - if call_name: - self._trace("calling %s(**%60r)" % (call_name, kwargs)) - function = loc[call_name] - function(channel, **kwargs) - finally: - channel._executing = False - self._trace("execution finished") - except KeyboardInterrupt: - channel.close(INTERRUPT_TEXT) - raise - except EOFError: - self._trace("ignoring EOFError because receiving finished") - - except BaseException as exc: - if not channel.gateway._channelfactory.finished: - self._trace(f"got exception: {exc!r}") - errortext = self._geterrortext(exc) - channel.close(errortext) - return - channel.close() - if self._executetask_complete is not None: - # Indicate that this task has finished executing, meaning - # that there is no possibility of it triggering a deadlock - # for the next spawn call. - self._executetask_complete.set() - - -# -# Cross-Python pickling code, tested from test_serializer.py -# - - -class DataFormatError(Exception): - """A value could not cross the channel in execnet's simple wire format. - - execnet only moves *simple* builtin data over a channel -- ``None``, - ``bool``, ``int``, ``float``, ``complex``, ``bytes``, ``str``, and - (arbitrarily nested) ``list``/``tuple``/``set``/``frozenset``/``dict`` of - those -- plus channel references, which pass through as channels. It does - **not** pickle: arbitrary instances, functions, ``datetime``, dataclasses, - pydantic models, numpy arrays, etc. have no wire representation. - - A ``DataFormatError`` therefore signals a caller error to resolve, not a - transport failure: reduce the value to simple data before sending (and - reconstruct it after receiving) with an encoding mechanism of your own -- - e.g. pydantic ``model_dump`` / ``model_validate`` or pytest's - ``pytest_report_to_serializable`` / ``pytest_report_from_serializable`` - hooks. See the docs, "Sending objects over a channel". - """ - - -class DumpError(DataFormatError): - """A value being **sent** is not simple wire data; convert it first. - - Raised by ``channel.send`` (and the internal serializer) when an object is - not one of execnet's simple wire types. Fix it at the call site by turning - the rich object into simple data -- e.g. ``dt.isoformat()``, - ``dataclasses.asdict(obj)``, ``model.model_dump(mode="json")`` -- rather - than expecting the channel to pickle it. Channels are the one non-builtin - that *is* sendable, so nested channel references are fine. - """ - - -class LoadError(DataFormatError): - """Received bytes could not be turned back into an object. - - Raised while **receiving** (deserializing) -- a corrupted or - protocol-incompatible payload, or data produced by a mismatched peer. - """ - - -def bchr(n: int) -> bytes: - return bytes([n]) - - -DUMPFORMAT_VERSION = bchr(2) - -FOUR_BYTE_INT_MAX = 2147483647 -FOUR_BYTE_INT_MIN = -2147483648 - -FLOAT_FORMAT = "!d" -FLOAT_FORMAT_SIZE = struct.calcsize(FLOAT_FORMAT) -COMPLEX_FORMAT = "!dd" -COMPLEX_FORMAT_SIZE = struct.calcsize(COMPLEX_FORMAT) - - -class _Stop(Exception): - pass - - -class opcode: - """Container for name -> num mappings.""" - - BUILDTUPLE = b"@" - BYTES = b"A" - CHANNEL = b"B" - FALSE = b"C" - FLOAT = b"D" - FROZENSET = b"E" - INT = b"F" - LONG = b"G" - LONGINT = b"H" - LONGLONG = b"I" - NEWDICT = b"J" - NEWLIST = b"K" - NONE = b"L" - STRING = b"N" - SET = b"O" - SETITEM = b"P" - STOP = b"Q" - TRUE = b"R" - COMPLEX = b"T" - - -class Unserializer: - num2func: dict[bytes, Callable[[Unserializer], None]] = {} - - def __init__( - self, - stream: ReadIO, - channel_or_gateway: Channel | BaseGateway | None = None, - ) -> None: - if isinstance(channel_or_gateway, Channel): - gw: BaseGateway | None = channel_or_gateway.gateway - else: - gw = channel_or_gateway - self.stream = stream - if gw is None: - self.channelfactory = None - else: - self.channelfactory = gw._channelfactory - - def load(self, versioned: bool = False) -> Any: - if versioned: - ver = self.stream.read(1) - if ver != DUMPFORMAT_VERSION: - raise LoadError("wrong dumpformat version %r" % ver) - self.stack: list[object] = [] - try: - while True: - opcode = self.stream.read(1) - if not opcode: - raise EOFError - try: - loader = self.num2func[opcode] - except KeyError: - raise LoadError( - f"unknown opcode {opcode!r} - wire protocol corruption?" - ) from None - loader(self) - except _Stop: - if len(self.stack) != 1: - raise LoadError("internal unserialization error") from None - return self.stack.pop(0) - else: - raise LoadError("didn't get STOP") - - def load_none(self) -> None: - self.stack.append(None) - - num2func[opcode.NONE] = load_none - - def load_true(self) -> None: - self.stack.append(True) - - num2func[opcode.TRUE] = load_true - - def load_false(self) -> None: - self.stack.append(False) - - num2func[opcode.FALSE] = load_false - - def load_int(self) -> None: - i = self._read_int4() - self.stack.append(i) - - num2func[opcode.INT] = load_int - - def load_longint(self) -> None: - s = self._read_byte_string() - self.stack.append(int(s)) - - num2func[opcode.LONGINT] = load_longint - - load_long = load_int - num2func[opcode.LONG] = load_long - load_longlong = load_longint - num2func[opcode.LONGLONG] = load_longlong - - def load_float(self) -> None: - binary = self.stream.read(FLOAT_FORMAT_SIZE) - self.stack.append(struct.unpack(FLOAT_FORMAT, binary)[0]) - - num2func[opcode.FLOAT] = load_float - - def load_complex(self) -> None: - binary = self.stream.read(COMPLEX_FORMAT_SIZE) - self.stack.append(complex(*struct.unpack(COMPLEX_FORMAT, binary))) - - num2func[opcode.COMPLEX] = load_complex - - def _read_int4(self) -> int: - value: int = struct.unpack("!i", self.stream.read(4))[0] - return value - - def _read_byte_string(self) -> bytes: - length = self._read_int4() - as_bytes = self.stream.read(length) - return as_bytes - - def load_string(self) -> None: - self.stack.append(self._read_byte_string().decode("utf-8")) - - num2func[opcode.STRING] = load_string - - def load_bytes(self) -> None: - s = self._read_byte_string() - self.stack.append(s) - - num2func[opcode.BYTES] = load_bytes - - def load_newlist(self) -> None: - length = self._read_int4() - self.stack.append([None] * length) - - num2func[opcode.NEWLIST] = load_newlist - - def load_setitem(self) -> None: - if len(self.stack) < 3: - raise LoadError("not enough items for setitem") - value = self.stack.pop() - key = self.stack.pop() - self.stack[-1][key] = value # type: ignore[index] - - num2func[opcode.SETITEM] = load_setitem - - def load_newdict(self) -> None: - self.stack.append({}) - - num2func[opcode.NEWDICT] = load_newdict - - def _load_collection(self, type_: type) -> None: - length = self._read_int4() - if length: - res = type_(self.stack[-length:]) - del self.stack[-length:] - self.stack.append(res) - else: - self.stack.append(type_()) - - def load_buildtuple(self) -> None: - self._load_collection(tuple) - - num2func[opcode.BUILDTUPLE] = load_buildtuple - - def load_set(self) -> None: - self._load_collection(set) - - num2func[opcode.SET] = load_set - - def load_frozenset(self) -> None: - self._load_collection(frozenset) - - num2func[opcode.FROZENSET] = load_frozenset - - def load_stop(self) -> None: - raise _Stop - - num2func[opcode.STOP] = load_stop - - def load_channel(self) -> None: - id = self._read_int4() - assert self.channelfactory is not None - newchannel = self.channelfactory.new(id) - self.stack.append(newchannel) - - num2func[opcode.CHANNEL] = load_channel - - -def dumps(obj: object) -> bytes: - """Serialize the given obj to a bytestring. - - The obj and all contained objects must be of a builtin - Python type (so nested dicts, sets, etc. are all OK but - not user-level instances). - """ - return _Serializer().save(obj, versioned=True) # type: ignore[return-value] - - -def dump(byteio, obj: object) -> None: - """write a serialized bytestring of the given obj to the given stream.""" - _Serializer(write=byteio.write).save(obj, versioned=True) - - -def loads(bytestring: bytes) -> Any: - """Deserialize the given bytestring to an object. - - If the bytestring was dumped with an incompatible protocol - version or if the bytestring is corrupted, the - ``execnet.DataFormatError`` will be raised. - """ - return load(BytesIO(bytestring)) - - -def load(io: ReadIO) -> Any: - """Derserialize an object form the specified stream. - - Behaviour is otherwise the same as with ``loads`` - """ - return Unserializer(io).load(versioned=True) - - -def loads_internal(bytestring: bytes, channelfactory=None) -> Any: - io = BytesIO(bytestring) - return Unserializer(io, channelfactory).load() - - -def dumps_internal(obj: object) -> bytes: - return _Serializer().save(obj) # type: ignore[return-value] - - -class _Serializer: - _dispatch: dict[type, Callable[[_Serializer, object], None]] = {} - - def __init__(self, write: Callable[[bytes], None] | None = None) -> None: - if write is None: - self._streamlist: list[bytes] = [] - write = self._streamlist.append - self._write = write - - def save(self, obj: object, versioned: bool = False) -> bytes | None: - # calling here is not re-entrant but multiple instances - # may write to the same stream because of the common platform - # atomic-write guarantee (concurrent writes each happen atomically) - if versioned: - self._write(DUMPFORMAT_VERSION) - self._save(obj) - self._write(opcode.STOP) - try: - streamlist = self._streamlist - except AttributeError: - return None - return b"".join(streamlist) - - def _save(self, obj: object) -> None: - tp = type(obj) - try: - dispatch = self._dispatch[tp] - except KeyError: - methodname = "save_" + tp.__name__ - meth: Callable[[_Serializer, object], None] | None = getattr( - self.__class__, methodname, None - ) - if meth is None: - raise DumpError(f"can't serialize {tp}") from None - dispatch = self._dispatch[tp] = meth - dispatch(self, obj) - - def save_NoneType(self, non: None) -> None: - self._write(opcode.NONE) - - def save_bool(self, boolean: bool) -> None: - if boolean: - self._write(opcode.TRUE) - else: - self._write(opcode.FALSE) - - def save_bytes(self, bytes_: bytes) -> None: - self._write(opcode.BYTES) - self._write_byte_sequence(bytes_) - - def save_str(self, s: str) -> None: - self._write(opcode.STRING) - self._write_unicode_string(s) - - def _write_unicode_string(self, s: str) -> None: - try: - as_bytes = s.encode("utf-8") - except UnicodeEncodeError as e: - raise DumpError("strings must be utf-8 encodable") from e - self._write_byte_sequence(as_bytes) - - def _write_byte_sequence(self, bytes_: bytes) -> None: - self._write_int4(len(bytes_), "string is too long") - self._write(bytes_) - - def _save_integral(self, i: int, short_op: bytes, long_op: bytes) -> None: - # The short op packs a signed 4-byte int; anything outside that range - # (in either direction) goes through the arbitrary-precision long op. - if FOUR_BYTE_INT_MIN <= i <= FOUR_BYTE_INT_MAX: - self._write(short_op) - self._write_int4(i) - else: - self._write(long_op) - self._write_byte_sequence(str(i).rstrip("L").encode("ascii")) - - def save_int(self, i: int) -> None: - self._save_integral(i, opcode.INT, opcode.LONGINT) - - def save_long(self, l: int) -> None: - self._save_integral(l, opcode.LONG, opcode.LONGLONG) - - def save_float(self, flt: float) -> None: - self._write(opcode.FLOAT) - self._write(struct.pack(FLOAT_FORMAT, flt)) - - def save_complex(self, cpx: complex) -> None: - self._write(opcode.COMPLEX) - self._write(struct.pack(COMPLEX_FORMAT, cpx.real, cpx.imag)) - - def _write_int4( - self, i: int, error: str = "int must be less than %i" % (FOUR_BYTE_INT_MAX,) - ) -> None: - if i > FOUR_BYTE_INT_MAX: - raise DumpError(error) - self._write(struct.pack("!i", i)) - - def save_list(self, L: list[object]) -> None: - self._write(opcode.NEWLIST) - self._write_int4(len(L), "list is too long") - for i, item in enumerate(L): - self._write_setitem(i, item) - - def _write_setitem(self, key: object, value: object) -> None: - self._save(key) - self._save(value) - self._write(opcode.SETITEM) - - def save_dict(self, d: dict[object, object]) -> None: - self._write(opcode.NEWDICT) - for key, value in d.items(): - self._write_setitem(key, value) - - def save_tuple(self, tup: tuple[object, ...]) -> None: - for item in tup: - self._save(item) - self._write(opcode.BUILDTUPLE) - self._write_int4(len(tup), "tuple is too long") - - def _write_set(self, s: set[object] | frozenset[object], op: bytes) -> None: - for item in s: - self._save(item) - self._write(op) - self._write_int4(len(s), "set is too long") - - def save_set(self, s: set[object]) -> None: - self._write_set(s, opcode.SET) - - def save_frozenset(self, s: frozenset[object]) -> None: - self._write_set(s, opcode.FROZENSET) - - def save_Channel(self, channel: Channel) -> None: - self._write(opcode.CHANNEL) - self._write_int4(channel.id) - - def save_AsyncChannel(self, channel: Any) -> None: - # trio-native channel (execnet._trio_gateway); same wire opcode, - # duck-typed here to avoid importing the async core. - self._write(opcode.CHANNEL) - self._write_int4(channel.id) +from ._shim import forwarder + +_MOVED = { + # tracing + "DEBUG": "._trace", + "pid": "._trace", + "trace": "._trace", + "notrace": "._trace", + # errors and error texts + "sysex": "._errors", + "INTERRUPT_TEXT": "._errors", + "MAIN_THREAD_ONLY_DEADLOCK_TEXT": "._errors", + "GatewayReceivedTerminate": "._errors", + "HostNotFound": "._errors", + "geterrortext": "._errors", + "RemoteError": "._errors", + "TimeoutError": "._errors", + "DataFormatError": "._errors", + "DumpError": "._errors", + "LoadError": "._errors", + # execution model presets + "ExecModel": "._execmodel", + "EXECMODEL_PROFILES": "._execmodel", + "get_execmodel": "._execmodel", + # wire protocol + "WriteIO": "._message", + "ReadIO": "._message", + "IO": "._message", + "Message": "._message", + "gateway_info": "._message", + "FrameDecoder": "._message", + # serializer + "bchr": "._serialize", + "DUMPFORMAT_VERSION": "._serialize", + "FOUR_BYTE_INT_MAX": "._serialize", + "FOUR_BYTE_INT_MIN": "._serialize", + "FLOAT_FORMAT": "._serialize", + "FLOAT_FORMAT_SIZE": "._serialize", + "COMPLEX_FORMAT": "._serialize", + "COMPLEX_FORMAT_SIZE": "._serialize", + "opcode": "._serialize", + "Unserializer": "._serialize", + "_Serializer": "._serialize", + "_Stop": "._serialize", + "dumps": "._serialize", + "dump": "._serialize", + "loads": "._serialize", + "load": "._serialize", + "dumps_internal": "._serialize", + "loads_internal": "._serialize", + # channels + "Channel": "._channel", + "ChannelFactory": "._channel", + "ChannelFile": "._channel", + "ChannelFileWrite": "._channel", + "ChannelFileRead": "._channel", + "ENDMARKER": "._channel", + "NO_ENDMARKER_WANTED": "._channel", + # gateways + "BaseGateway": "._gateway_base", + "WorkerGateway": "._gateway_base", +} + +__getattr__ = forwarder("gateway_base", _MOVED) diff --git a/src/execnet/multi.py b/src/execnet/multi.py index 7c0810a7..06683457 100644 --- a/src/execnet/multi.py +++ b/src/execnet/multi.py @@ -1,419 +1,22 @@ -""" -Managing Gateway Groups and interactions with multiple channels. +"""Deprecated alias for :mod:`execnet._multi`. -(c) 2008-2014, Holger Krekel and others +``Group``, ``MultiChannel``, ``default_group``, ``makegateway`` and +``set_execmodel`` are exported from :mod:`execnet` and :mod:`execnet.sync`; +use those. """ from __future__ import annotations -import atexit -import queue -import threading -import time -import types -from collections.abc import Callable -from collections.abc import Iterable -from collections.abc import Iterator -from collections.abc import Sequence -from contextlib import suppress -from threading import Lock -from typing import TYPE_CHECKING -from typing import Any -from typing import Literal -from typing import TypeAlias -from typing import overload - -from ._boundary import wakener_names -from .gateway_base import EXECMODEL_PROFILES -from .gateway_base import Channel -from .gateway_base import ExecModel -from .gateway_base import get_execmodel -from .gateway_base import trace -from .xspec import XSpec - -if TYPE_CHECKING: - from .gateway import Gateway - - -NO_ENDMARKER_WANTED = object() - - -class Group: - """Gateway Group.""" - - defaultspec = "popen" - - def __init__( - self, xspecs: Iterable[XSpec | str | None] = (), execmodel: str = "thread" - ) -> None: - """Initialize a group and make gateways as specified. - - execmodel can be one of the supported execution models. - """ - self._gateways: list[Gateway] = [] - self._autoidcounter = 0 - self._autoidlock = Lock() - self._gateways_to_join: list[Gateway] = [] - self._trio_host: Any = None - self._async_group: Any = None - # we use the same execmodel for all of the Gateway objects - # we spawn on our side. Probably we should not allow different - # execmodels between different groups but not clear. - # Note that "other side" execmodels may differ and is typically - # specified by the spec passed to makegateway. - self.set_execmodel(execmodel) - for xspec in xspecs: - self.makegateway(xspec) - atexit.register(self._cleanup_atexit) - - def _ensure_trio_host(self) -> Any: - if self._trio_host is None: - from . import _trio_host - - self._trio_host = _trio_host.TrioHost(name="execnet-trio-group") - self._trio_host.start() - return self._trio_host - - def _ensure_async_group(self) -> Any: - """The FacadeAsyncGroup owning the async side, running on the host.""" - if self._async_group is None: - from . import _trio_host - - host = self._ensure_trio_host() - - async def _start() -> Any: - async_group = _trio_host.FacadeAsyncGroup(self, host) - return await host._nursery.start(async_group.run) - - self._async_group = host.call(_start) - return self._async_group - - @property - def execmodel(self) -> ExecModel: - return self._execmodel - - @property - def remote_execmodel(self) -> ExecModel: - return self._remote_execmodel - - def set_execmodel( - self, execmodel: str, remote_execmodel: str | None = None - ) -> None: - """Set the execution model for local and remote site. - - execmodel can be one of the supported execution models. - It determines the execution model for any newly created gateway. - If remote_execmodel is not specified it takes on the value of execmodel. - - NOTE: Execution models can only be set before any gateway is created. - """ - if self._gateways: - raise ValueError( - "can not set execution models if gateways have been created already" - ) - if remote_execmodel is None: - remote_execmodel = execmodel - self._execmodel = get_execmodel(execmodel) - self._remote_execmodel = get_execmodel(remote_execmodel) - - def __repr__(self) -> str: - idgateways = [gw.id for gw in self] - return "" % idgateways - - def __getitem__(self, key: int | str | Gateway) -> Gateway: - if isinstance(key, int): - return self._gateways[key] - for gw in self._gateways: - if gw == key or gw.id == key: - return gw - raise KeyError(key) - - def __contains__(self, key: str) -> bool: - try: - self[key] - return True - except KeyError: - return False - - def __len__(self) -> int: - return len(self._gateways) - - def __iter__(self) -> Iterator[Gateway]: - return iter(list(self._gateways)) - - def makegateway(self, spec: XSpec | str | None = None) -> Gateway: - """Create and configure a gateway to a Python interpreter. - - The ``spec`` string encodes the target gateway type - and configuration information. The general format is:: - - key1=value1//key2=value2//... - - If you leave out the ``=value`` part a True value is assumed. - Valid types: ``popen``, ``ssh=hostname``, ``socket=host:port``. - Valid configuration:: - - id= specifies the gateway id - python= specifies which python interpreter to execute - execmodel=name worker profile: where exec'd code runs relative - to the worker's protocol loop. 'thread' (pool - threads) or 'main_thread_only' (serialized on - the worker main thread, GUI/signal-safe). - wait=backend wakener for blocking waits ('thread' default) - chdir= specifies to which directory to change - nice= specifies process priority of new process - env:NAME=value specifies a remote environment variable setting. - - If no spec is given, self.defaultspec is used. - """ - if not spec: - spec = self.defaultspec - if not isinstance(spec, XSpec): - spec = XSpec(spec) - self.allocate_id(spec) - if spec.execmodel is None: - spec.execmodel = self.remote_execmodel.backend - elif spec.execmodel not in EXECMODEL_PROFILES: - raise ValueError( - f"unknown execmodel {spec.execmodel!r}" - f" (known profiles: {list(EXECMODEL_PROFILES)})" - ) - if spec.wait is not None and spec.wait not in wakener_names(): - raise ValueError( - f"unknown wait backend {spec.wait!r} (known: {wakener_names()})" - ) - from . import _trio_host - - if not (spec.socket or spec.via or spec.ssh or spec.vagrant_ssh or spec.popen): - raise ValueError(f"no gateway type found for {spec._spec!r}") - gw = _trio_host.makegateway_trio(self, spec) - gw.spec = spec - self._register(gw) - # chdir/nice/env travel in the worker config and are applied at - # worker startup -- no remote_exec, so no exec slot is claimed. - return gw - - def allocate_id(self, spec: XSpec) -> None: - """(re-entrant) allocate id for the given xspec object.""" - if spec.id is None: - with self._autoidlock: - id = "gw" + str(self._autoidcounter) - self._autoidcounter += 1 - if id in self: - raise ValueError(f"already have gateway with id {id!r}") - spec.id = id - - def _register(self, gateway: Gateway) -> None: - assert not hasattr(gateway, "_group") - assert gateway.id - assert gateway.id not in self - self._gateways.append(gateway) - gateway._group = self - - def _unregister(self, gateway: Gateway) -> None: - self._gateways.remove(gateway) - self._gateways_to_join.append(gateway) - - def _cleanup_atexit(self) -> None: - trace(f"=== atexit cleanup {self!r} ===") - self.terminate(timeout=1.0) - if self._async_group is not None: - with suppress(Exception): - self._trio_host.call_sync(self._async_group.shutdown.set) - self._async_group = None - if self._trio_host is not None: - self._trio_host.stop(timeout=1.0) - self._trio_host = None - - def terminate(self, timeout: float | None = None) -> None: - """Trigger exit of member gateways and wait for termination - of member gateways and associated subprocesses. - - After waiting timeout seconds try to to kill local sub processes of - popen- and ssh-gateways. - - Timeout defaults to None meaning open-ended waiting and no kill - attempts. - """ - while self or self._gateways_to_join: - vias: set[str] = set() - for gw in self: - if gw.spec.via: - vias.add(gw.spec.via) - for gw in self: - if gw.id not in vias: - gw.exit() - if self._async_group is not None: - # Tunneled (via) gateways terminate before their masters, - # each with a GATEWAY_TERMINATE + timeout grace, then kill; - # bounded at roughly twice the timeout (issues #43 / #221). - try: - self._host_terminate(timeout) - except Exception as exc: - trace("group terminate error:", exc) - for gw in self._gateways_to_join: - gw.join() - self._gateways_to_join[:] = [] - - def _host_terminate(self, timeout: float | None) -> None: - """Terminate the async group, parking correctly for wait backends. - - A member gateway created with a non-thread ``wait=`` implies the - caller may be a greenlet: wait on a OneShot instead of blocking - the OS thread (which would stall the hub for the whole grace). - """ - backends = {gw._wait_backend for gw in self._gateways_to_join} | { - gw._wait_backend for gw in self - } - backends.discard("thread") - if backends: - from ._boundary import make_wakener - - self._trio_host.call_pending( - self._async_group.terminate, - timeout, - wakener=make_wakener(backends.pop()), - ).wait() - else: - self._trio_host.call(self._async_group.terminate, timeout) - - def remote_exec( - self, - source: str | types.FunctionType | Callable[..., object] | types.ModuleType, - **kwargs, - ) -> MultiChannel: - """remote_exec source on all member gateways and return - a MultiChannel connecting to all sub processes.""" - channels = [] - for gw in self: - channels.append(gw.remote_exec(source, **kwargs)) - return MultiChannel(channels) - - -class MultiChannel: - def __init__(self, channels: Sequence[Channel]) -> None: - self._channels = channels - - def __len__(self) -> int: - return len(self._channels) - - def __iter__(self) -> Iterator[Channel]: - return iter(self._channels) - - def __getitem__(self, key: int) -> Channel: - return self._channels[key] - - def __contains__(self, chan: Channel) -> bool: - return chan in self._channels - - def send_each(self, item: object) -> None: - for ch in self._channels: - ch.send(item) - - @overload - def receive_each(self, withchannel: Literal[False] = ...) -> list[Any]: - pass - - @overload - def receive_each(self, withchannel: Literal[True]) -> list[tuple[Channel, Any]]: - pass - - def receive_each( - self, withchannel: bool = False - ) -> list[tuple[Channel, Any]] | list[Any]: - assert not hasattr(self, "_queue") - l: list[object] = [] - for ch in self._channels: - obj = ch.receive() - if withchannel: - l.append((ch, obj)) - else: - l.append(obj) - return l - - def make_receive_queue(self, endmarker: object = NO_ENDMARKER_WANTED): - try: - return self._queue # type: ignore[has-type] - except AttributeError: - self._queue: queue.Queue[tuple[Channel, Any]] | None = None - for ch in self._channels: - if self._queue is None: - self._queue = queue.Queue() - - def putreceived(obj, channel: Channel = ch) -> None: - self._queue.put((channel, obj)) # type: ignore[union-attr] - - if endmarker is NO_ENDMARKER_WANTED: - ch.setcallback(putreceived) - else: - ch.setcallback(putreceived, endmarker=endmarker) - return self._queue - - def waitclose(self) -> None: - first = None - for ch in self._channels: - try: - ch.waitclose() - except ch.RemoteError as exc: - if first is None: - first = exc - if first: - raise first - - -TermKillFunc: TypeAlias = Callable[[], object] -TermKillPair: TypeAlias = tuple[TermKillFunc, TermKillFunc] - - -def safe_terminate( - execmodel: ExecModel, - timeout: float | None, - list_of_paired_functions: Sequence[TermKillPair], -) -> None: - """Run terminate/kill pairs in parallel with a hard wait bound. - - Each termfunc is given ``timeout``. If it does not finish, killfunc runs. - The final wait is also bounded so a stuck kill cannot hang the caller - forever (see issues #43 / #221). ``execmodel`` is accepted for - backward compatibility and unused (daemon threads do the waiting). - """ - errors: list[BaseException] = [] - - def termkill(termfunc: TermKillFunc, killfunc: TermKillFunc) -> None: - term_done = threading.Event() - term_errors: list[BaseException] = [] - - def run_term() -> None: - try: - termfunc() - except BaseException as exc: - term_errors.append(exc) - finally: - term_done.set() - - threading.Thread(target=run_term, daemon=True).start() - if not term_done.wait(timeout): - killfunc() - return - if term_errors: - errors.append(term_errors[0]) - - threads = [ - threading.Thread(target=termkill, args=pair, daemon=True) - for pair in list_of_paired_functions - ] - for thread in threads: - thread.start() - # Allow term timeout plus a kill attempt; never block indefinitely. - wait_timeout = None if timeout is None else timeout * 2 - deadline = None if wait_timeout is None else time.monotonic() + wait_timeout - for thread in threads: - remaining = None if deadline is None else max(0, deadline - time.monotonic()) - thread.join(remaining) - if errors: - raise errors[0] +from ._shim import forwarder +_MOVED = { + "Group": "._multi", + "MultiChannel": "._multi", + "default_group": "._multi", + "makegateway": "._multi", + "set_execmodel": "._multi", + "safe_terminate": "._multi", + "NO_ENDMARKER_WANTED": "._multi", +} -default_group = Group() -makegateway = default_group.makegateway -set_execmodel = default_group.set_execmodel +__getattr__ = forwarder("multi", _MOVED) diff --git a/src/execnet/rsync.py b/src/execnet/rsync.py index 876e38fc..786ed42b 100644 --- a/src/execnet/rsync.py +++ b/src/execnet/rsync.py @@ -1,248 +1,12 @@ -""" -1:N rsync implementation on top of execnet. +"""Deprecated alias for :mod:`execnet._rsync`. -(c) 2006-2009, Armin Rigo, Holger Krekel, Maciej Fijalkowski +``RSync`` is exported from :mod:`execnet` and :mod:`execnet.sync`; use those. """ from __future__ import annotations -import os -import stat -from collections.abc import Callable -from hashlib import md5 -from queue import Queue -from typing import Literal - -import execnet.rsync_remote -from execnet.gateway import Gateway -from execnet.gateway_base import BaseGateway -from execnet.gateway_base import Channel - - -class RSync: - """This class allows to send a directory structure (recursively) - to one or multiple remote filesystems. - - There is limited support for symlinks, which means that symlinks - pointing to the sourcetree will be send "as is" while external - symlinks will be just copied (regardless of existence of such - a path on remote side). - """ - - def __init__(self, sourcedir, callback=None, verbose: bool = True) -> None: - self._sourcedir = str(sourcedir) - self._verbose = verbose - assert callback is None or callable(callback) - self._callback = callback - self._channels: dict[Channel, Callable[[], None] | None] = {} - self._receivequeue: Queue[ - tuple[ - Channel, - ( - None - | tuple[Literal["send"], tuple[list[str], bytes]] - | tuple[Literal["list_done"], None] - | tuple[Literal["ack"], str] - | tuple[Literal["links"], None] - | tuple[Literal["done"], None] - ), - ] - ] = Queue() - self._links: list[tuple[Literal["linkbase", "link"], str, str]] = [] - - def filter(self, path: str) -> bool: - return True - - def _end_of_channel(self, channel: Channel) -> None: - if channel in self._channels: - # too early! we must have got an error - channel.waitclose() - # or else we raise one - raise OSError(f"connection unexpectedly closed: {channel.gateway} ") - - def _process_link(self, channel: Channel) -> None: - for link in self._links: - channel.send(link) - # completion marker, this host is done - channel.send(42) - - def _done(self, channel: Channel) -> None: - """Call all callbacks.""" - finishedcallback = self._channels.pop(channel) - if finishedcallback: - finishedcallback() - channel.waitclose() - - def _list_done(self, channel: Channel) -> None: - # sum up all to send - if self._callback: - s = sum([self._paths[i] for i in self._to_send[channel]]) - self._callback("list", s, channel) - - def _send_item( - self, - channel: Channel, - modified_rel_path_components: list[str], - checksum: bytes, - ) -> None: - """Send one item.""" - modified_path = os.path.join(self._sourcedir, *modified_rel_path_components) - try: - with open(modified_path, "rb") as fp: - data = fp.read() - except OSError: - data = None - - # provide info to progress callback function - modified_rel_path = "/".join(modified_rel_path_components) - if data is not None: - self._paths[modified_rel_path] = len(data) - else: - self._paths[modified_rel_path] = 0 - if channel not in self._to_send: - self._to_send[channel] = [] - self._to_send[channel].append(modified_rel_path) - # print "sending", modified_rel_path, data and len(data) or 0, checksum - - if data is not None: - if checksum is not None and checksum == md5(data).digest(): - data = None # not really modified - else: - self._report_send_file(channel.gateway, modified_rel_path) - channel.send(data) - - def _report_send_file(self, gateway: BaseGateway, modified_rel_path: str) -> None: - if self._verbose: - print(f"{gateway} <= {modified_rel_path}") - - def send(self, raises: bool = True) -> None: - """Sends a sourcedir to all added targets. - - raises indicates whether to raise an error or return in case of lack of - targets. - """ - if not self._channels: - if raises: - raise OSError( - "no targets available, maybe you are trying call send() twice?" - ) - return - # normalize a trailing '/' away - self._sourcedir = os.path.dirname(os.path.join(self._sourcedir, "x")) - # send directory structure and file timestamps/sizes - self._send_directory_structure(self._sourcedir) - - # paths and to_send are only used for doing - # progress-related callbacks - self._paths: dict[str, int] = {} - self._to_send: dict[Channel, list[str]] = {} - - # send modified file to clients - while self._channels: - channel, req = self._receivequeue.get() - if req is None: - self._end_of_channel(channel) - else: - if req[0] == "links": - self._process_link(channel) - elif req[0] == "done": - self._done(channel) - elif req[0] == "ack": - if self._callback: - self._callback("ack", self._paths[req[1]], channel) - elif req[0] == "list_done": - self._list_done(channel) - elif req[0] == "send": - self._send_item(channel, req[1][0], req[1][1]) - else: - assert "Unknown command %s" % req[0] # type: ignore[unreachable] - - def add_target( - self, - gateway: Gateway, - destdir: str | os.PathLike[str], - finishedcallback: Callable[[], None] | None = None, - **options, - ) -> None: - """Add a remote target specified via a gateway and a remote destination - directory.""" - for name in options: - assert name in ("delete",) - - def itemcallback(req) -> None: - self._receivequeue.put((channel, req)) - - channel = gateway.remote_exec(execnet.rsync_remote) - channel.setcallback(itemcallback, endmarker=None) - channel.send((str(destdir), options)) - self._channels[channel] = finishedcallback - - def _broadcast(self, msg: object) -> None: - for channel in self._channels: - channel.send(msg) - - def _send_link( - self, - linktype: Literal["linkbase", "link"], - basename: str, - linkpoint: str, - ) -> None: - self._links.append((linktype, basename, linkpoint)) - - def _send_directory(self, path: str) -> None: - # dir: send a list of entries - names = [] - subpaths = [] - for name in os.listdir(path): - p = os.path.join(path, name) - if self.filter(p): - names.append(name) - subpaths.append(p) - mode = os.lstat(path).st_mode - self._broadcast([mode, *names]) - for p in subpaths: - self._send_directory_structure(p) +from ._shim import forwarder - def _send_link_structure(self, path: str) -> None: - sourcedir = self._sourcedir - basename = path[len(self._sourcedir) + 1 :] - linkpoint = os.readlink(path) - # On Windows, readlink returns an extended path (//?/) for - # absolute links, but relpath doesn't like mixing extended - # and non-extended paths. So fix it up ourselves. - if ( - os.path.__name__ == "ntpath" - and linkpoint.startswith("\\\\?\\") - and not self._sourcedir.startswith("\\\\?\\") - ): - sourcedir = "\\\\?\\" + self._sourcedir - try: - relpath = os.path.relpath(linkpoint, sourcedir) - except ValueError: - relpath = None - if ( - relpath is not None - and relpath not in (os.curdir, os.pardir) - and not relpath.startswith(os.pardir + os.sep) - ): - self._send_link("linkbase", basename, relpath) - else: - # relative or absolute link, just send it - self._send_link("link", basename, linkpoint) - self._broadcast(None) +_MOVED = {"RSync": "._rsync"} - def _send_directory_structure(self, path: str) -> None: - try: - st = os.lstat(path) - except OSError: - self._broadcast((None, 0, 0)) - return - if stat.S_ISREG(st.st_mode): - # regular file: send a mode/timestamp/size pair - self._broadcast((st.st_mode, st.st_mtime, st.st_size)) - elif stat.S_ISDIR(st.st_mode): - self._send_directory(path) - elif stat.S_ISLNK(st.st_mode): - self._send_link_structure(path) - else: - raise ValueError(f"cannot sync {path!r}") +__getattr__ = forwarder("rsync", _MOVED) diff --git a/src/execnet/rsync_remote.py b/src/execnet/rsync_remote.py index a8467b76..4770918a 100644 --- a/src/execnet/rsync_remote.py +++ b/src/execnet/rsync_remote.py @@ -1,126 +1,13 @@ -""" -(c) 2006-2013, Armin Rigo, Holger Krekel, Maciej Fijalkowski +"""Deprecated alias for :mod:`execnet._rsync_remote`. + +The worker half of the rsync protocol; :class:`execnet.RSync` ships it to the +remote side itself, so there is no reason to reference this module. """ from __future__ import annotations -from contextlib import suppress -from typing import TYPE_CHECKING -from typing import Literal -from typing import cast - -if TYPE_CHECKING: - from execnet.gateway_base import Channel - - -def serve_rsync(channel: Channel) -> None: - import os - import shutil - import stat - from hashlib import md5 - - destdir, options = cast("tuple[str, dict[str, object]]", channel.receive()) - modifiedfiles = [] - - def remove(path: str) -> None: - assert path.startswith(destdir) - try: - os.unlink(path) - except OSError: - # assume it's a dir - shutil.rmtree(path, True) - - def receive_directory_structure(path: str, relcomponents: list[str]) -> None: - try: - st = os.lstat(path) - except OSError: - st = None - msg = channel.receive() - if isinstance(msg, list): - if st and not stat.S_ISDIR(st.st_mode): - os.unlink(path) - st = None - if not st: - os.makedirs(path) - mode = msg.pop(0) - if mode: - # Ensure directories are writable, otherwise a - # permission denied error (EACCES) would be raised - # when attempting to receive read-only directory - # structures. - os.chmod(path, mode | 0o700) - entrynames = {} - for entryname in msg: - destpath = os.path.join(path, entryname) - receive_directory_structure(destpath, [*relcomponents, entryname]) - entrynames[entryname] = True - if options.get("delete"): - for othername in os.listdir(path): - if othername not in entrynames: - otherpath = os.path.join(path, othername) - remove(otherpath) - elif msg is not None: - assert isinstance(msg, tuple) - checksum = None - if st: - if stat.S_ISREG(st.st_mode): - msg_mode, msg_mtime, msg_size = msg - if msg_size != st.st_size: - pass - elif msg_mtime != st.st_mtime: - with open(path, "rb") as fp: - checksum = md5(fp.read()).digest() - elif msg_mode and msg_mode != st.st_mode: - os.chmod(path, msg_mode | 0o700) - return - else: - return # already fine - else: - remove(path) - channel.send(("send", (relcomponents, checksum))) - modifiedfiles.append((path, msg)) - - receive_directory_structure(destdir, []) - - STRICT_CHECK = False # seems most useful this way for py.test - channel.send(("list_done", None)) - - for path, (mode, time, size) in modifiedfiles: - data = cast(bytes, channel.receive()) - channel.send(("ack", path[len(destdir) + 1 :])) - if data is not None: - if STRICT_CHECK and len(data) != size: - raise OSError(f"file modified during rsync: {path!r}") - with open(path, "wb") as fp: - fp.write(data) - try: - if mode: - os.chmod(path, mode) - os.utime(path, (time, time)) - except OSError: - pass - del data - channel.send(("links", None)) - - msg = channel.receive() - while msg != 42: - # we get symlink - _type, relpath, linkpoint = cast( - "tuple[Literal['linkbase', 'link'], str, str]", msg - ) - path = os.path.join(destdir, relpath) - with suppress(OSError): - remove(path) - - if _type == "linkbase": - src = os.path.join(destdir, linkpoint) - else: - assert _type == "link", _type - src = linkpoint - os.symlink(src, path) - msg = channel.receive() - channel.send(("done", None)) +from ._shim import forwarder +_MOVED = {"serve_rsync": "._rsync_remote"} -if __name__ == "__channelexec__": - serve_rsync(channel) # type: ignore[name-defined] # noqa:F821 +__getattr__ = forwarder("rsync_remote", _MOVED) diff --git a/src/execnet/sync.py b/src/execnet/sync.py index e5e026c7..37259bb7 100644 --- a/src/execnet/sync.py +++ b/src/execnet/sync.py @@ -6,21 +6,21 @@ into this module. """ -from .gateway import Gateway -from .gateway_base import Channel -from .gateway_base import DataFormatError -from .gateway_base import DumpError -from .gateway_base import HostNotFound -from .gateway_base import LoadError -from .gateway_base import RemoteError -from .gateway_base import TimeoutError -from .multi import Group -from .multi import MultiChannel -from .multi import default_group -from .multi import makegateway -from .multi import set_execmodel -from .rsync import RSync -from .xspec import XSpec +from ._channel import Channel +from ._errors import DataFormatError +from ._errors import DumpError +from ._errors import HostNotFound +from ._errors import LoadError +from ._errors import RemoteError +from ._errors import TimeoutError +from ._gateway import Gateway +from ._multi import Group +from ._multi import MultiChannel +from ._multi import default_group +from ._multi import makegateway +from ._multi import set_execmodel +from ._rsync import RSync +from ._xspec import XSpec __all__ = [ "Channel", diff --git a/src/execnet/trio.py b/src/execnet/trio.py index 4b712c0a..9704b6ef 100644 --- a/src/execnet/trio.py +++ b/src/execnet/trio.py @@ -16,40 +16,32 @@ async def main(): The error types are shared with the blocking API in :mod:`execnet.sync`. Items you send must already be simple builtin data (plus channels); the -standalone serializer is intentionally not part of the public API -- see -``DumpError``. +standalone serializer is intentionally not part of the public API -- +``execnet.can_send`` checks a value before you send it; see ``DumpError``. """ +from ._errors import DataFormatError +from ._errors import DumpError +from ._errors import HostNotFound +from ._errors import LoadError +from ._errors import RemoteError +from ._errors import TimeoutError from ._trio_gateway import AsyncChannel from ._trio_gateway import AsyncGateway from ._trio_gateway import AsyncGroup -from ._trio_gateway import ByteStream -from ._trio_gateway import RawChannel -from ._trio_gateway import RawChannelStream from ._trio_gateway import open_popen_gateway -from ._trio_gateway import serve_gateway -from .gateway_base import DataFormatError -from .gateway_base import DumpError -from .gateway_base import HostNotFound -from .gateway_base import LoadError -from .gateway_base import RemoteError -from .gateway_base import TimeoutError -from .xspec import XSpec +from ._xspec import XSpec __all__ = [ "AsyncChannel", "AsyncGateway", "AsyncGroup", - "ByteStream", "DataFormatError", "DumpError", "HostNotFound", "LoadError", - "RawChannel", - "RawChannelStream", "RemoteError", "TimeoutError", "XSpec", "open_popen_gateway", - "serve_gateway", ] diff --git a/src/execnet/xspec.py b/src/execnet/xspec.py index afc58f4c..9f5a2634 100644 --- a/src/execnet/xspec.py +++ b/src/execnet/xspec.py @@ -1,74 +1,13 @@ -""" -(c) 2008-2013, holger krekel +"""Deprecated alias for :mod:`execnet._xspec`. + +``XSpec`` is exported from :mod:`execnet`, :mod:`execnet.sync`, +:mod:`execnet.trio` and :mod:`execnet.aio`; use those. """ from __future__ import annotations +from ._shim import forwarder -class XSpec: - """Execution Specification: key1=value1//key2=value2 ... - - * Keys need to be unique within the specification scope - * Neither key nor value are allowed to contain "//" - * Keys are not allowed to contain "=" - * Keys are not allowed to start with underscore - * If no "=value" is given, assume a boolean True value - """ - - # XXX allow customization, for only allow specific key names - chdir: str | None = None - dont_write_bytecode: bool | None = None - execmodel: str | None = None - id: str | None = None - installvia: str | None = None - nice: str | None = None - popen: bool | None = None - python: str | None = None - socket: str | None = None - ssh: str | None = None - ssh_config: str | None = None - vagrant_ssh: str | None = None - via: str | None = None - wait: str | None = None - - def __init__(self, string: str) -> None: - self._spec = string - self.env = {} - for keyvalue in string.split("//"): - i = keyvalue.find("=") - value: str | bool - if i == -1: - key, value = keyvalue, True - else: - key, value = keyvalue[:i], keyvalue[i + 1 :] - if key[0] == "_": - raise AttributeError("%r not a valid XSpec key" % key) - if key in self.__dict__: - raise ValueError(f"duplicate key: {key!r} in {string!r}") - if key.startswith("env:"): - self.env[key[4:]] = value - else: - setattr(self, key, value) - - def __getattr__(self, name: str) -> None | bool | str: - if name[0] == "_": - raise AttributeError(name) - return None - - def __repr__(self) -> str: - return f"" - - def __str__(self) -> str: - return self._spec - - def __hash__(self) -> int: - return hash(self._spec) - - def __eq__(self, other: object) -> bool: - return self._spec == getattr(other, "_spec", None) - - def __ne__(self, other: object) -> bool: - return self._spec != getattr(other, "_spec", None) +_MOVED = {"XSpec": "._xspec"} - def _samefilesystem(self) -> bool: - return self.popen is not None and self.chdir is None +__getattr__ = forwarder("xspec", _MOVED) diff --git a/testing/conftest.py b/testing/conftest.py index 04dbe631..de1e2866 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -11,9 +11,9 @@ import pytest import execnet -from execnet.gateway import Gateway -from execnet.gateway_base import ExecModel -from execnet.gateway_base import get_execmodel +from execnet import Gateway +from execnet._execmodel import ExecModel +from execnet._execmodel import get_execmodel collect_ignore = ["build", "doc/_build"] diff --git a/testing/test_basics.py b/testing/test_basics.py index d5606610..a81ed1f6 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -16,11 +16,14 @@ import execnet from execnet import _boundary -from execnet import gateway -from execnet import gateway_base -from execnet.gateway_base import ChannelFactory -from execnet.gateway_base import ExecModel -from execnet.gateway_base import Message +from execnet import _errors +from execnet import _exec_source +from execnet import _gateway_base +from execnet import _message +from execnet import _serialize +from execnet._channel import ChannelFactory +from execnet._execmodel import ExecModel +from execnet._message import Message skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), @@ -28,45 +31,49 @@ ) -# The standalone serializer is an internal detail (execnet.gateway_base), +# The standalone serializer is an internal detail (execnet._serialize), # not part of the public API -- see the docs, "Sending objects over a channel". @pytest.mark.parametrize("val", ["123", 42, [1, 2, 3], ["23", 25]]) class TestSerializeAPI: def test_serializer_api(self, val: object) -> None: - dumped = gateway_base.dumps(val) - val2 = gateway_base.loads(dumped) + dumped = _serialize.dumps(val) + val2 = _serialize.loads(dumped) assert val == val2 def test_mmap(self, tmp_path: Path, val: object) -> None: mmap = pytest.importorskip("mmap").mmap p = tmp_path / "data.bin" - p.write_bytes(gateway_base.dumps(val)) + p.write_bytes(_serialize.dumps(val)) with p.open("r+b") as f: m = mmap(f.fileno(), 0) - val2 = gateway_base.load(m) + val2 = _serialize.load(m) assert val == val2 def test_bytesio(self, val: object) -> None: f = BytesIO() - gateway_base.dump(f, val) + _serialize.dump(f, val) read = BytesIO(f.getvalue()) - val2 = gateway_base.load(read) + val2 = _serialize.load(read) assert val == val2 def test_serializer_not_public() -> None: - # dumps/loads/dump/load are internal to gateway_base only. - for name in ("dumps", "loads", "dump", "load"): + # dumps/loads/dump/load stay internal to execnet._serialize. ``dumps`` + # is still *reachable* as a temporary pytest-xdist shim, but it is not + # part of the surface -- see test_namespaces.py. + for name in ("loads", "dump", "load"): assert not hasattr(execnet, name), name + for name in ("dumps", "loads", "dump", "load"): + assert name not in execnet.__all__, name def test_serializer_api_version_error(monkeypatch: pytest.MonkeyPatch) -> None: - bchr = gateway_base.bchr - monkeypatch.setattr(gateway_base, "DUMPFORMAT_VERSION", bchr(1)) - dumped = gateway_base.dumps(42) - monkeypatch.setattr(gateway_base, "DUMPFORMAT_VERSION", bchr(2)) - pytest.raises(execnet.DataFormatError, lambda: gateway_base.loads(dumped)) + bchr = _serialize.bchr + monkeypatch.setattr(_serialize, "DUMPFORMAT_VERSION", bchr(1)) + dumped = _serialize.dumps(42) + monkeypatch.setattr(_serialize, "DUMPFORMAT_VERSION", bchr(2)) + pytest.raises(execnet.DataFormatError, lambda: _serialize.loads(dumped)) def test_errors_on_execnet() -> None: @@ -75,26 +82,22 @@ def test_errors_on_execnet() -> None: assert hasattr(execnet, "DataFormatError") -def standalone_gateway_base_source() -> str: - """gateway_base's source as a self-contained script. +def standalone_protocol_source() -> str: + """The wire-protocol core as a self-contained script. - The module imports the trio-free boundary kit relatively; for the - run-on-any-python checks the kit source is inlined in place of those - imports (minus its own future import, which must stay file-leading). + ``_errors``, ``_message`` and ``_serialize`` are the layers a peer needs + to frame and serialize; between them the only runtime execnet imports are + of each other, so concatenating their sources -- dropping those imports + and keeping a single file-leading future import -- yields a script that + runs on any interpreter, which is what these checks verify. The + ``if TYPE_CHECKING:`` imports are left in place; they never execute. """ - boundary_source = inspect.getsource(_boundary).replace( - "from __future__ import annotations\n", "" - ) - lines = [] - inlined = False - for line in inspect.getsource(gateway_base).splitlines(keepends=True): - if line.startswith("from ._boundary import"): - if not inlined: - inlined = True - lines.append(boundary_source) - else: + lines = ["from __future__ import annotations\n"] + for module in (_errors, _message, _serialize): + for line in inspect.getsource(module).splitlines(keepends=True): + if line.startswith(("from __future__ import annotations", "from ._")): + continue lines.append(line) - assert inlined return "".join(lines) @@ -156,14 +159,14 @@ def checker(anypython: str, tmp_path: Path) -> Checker: def test_io_message(checker: Checker) -> None: - out = checker.run_check(standalone_gateway_base_source() + IO_MESSAGE_EXTRA_SOURCE) + out = checker.run_check(standalone_protocol_source() + IO_MESSAGE_EXTRA_SOURCE) print(out.stdout) assert "all passed" in out.stdout def test_geterrortext(checker: Checker) -> None: out = checker.run_check( - standalone_gateway_base_source() + standalone_protocol_source() + """ class Arg(Exception): pass @@ -230,7 +233,7 @@ def close(self, errortext: str | None = None) -> None: def test_exectask(execmodel: ExecModel) -> None: io = BytesIO() io.execmodel = execmodel # type: ignore[attr-defined] - gw = gateway_base.WorkerGateway(io, id="something") # type: ignore[arg-type] + gw = _gateway_base.WorkerGateway(io, id="something") # type: ignore[arg-type] ch = PseudoChannel() gw.executetask((ch, ("raise ValueError()", None, {}))) # type: ignore[arg-type] assert "ValueError" in str(ch._closed[0]) @@ -261,7 +264,7 @@ def _messages(self) -> list[Message]: ] def test_single_feed_yields_all(self) -> None: - decoder = gateway_base.FrameDecoder() + decoder = _message.FrameDecoder() blob = b"".join(m.pack() for m in self._messages()) got = list(decoder.feed(blob)) assert [(m.msgcode, m.channelid, m.data) for m in got] == [ @@ -271,7 +274,7 @@ def test_single_feed_yields_all(self) -> None: @pytest.mark.parametrize("chunksize", [1, 2, 3, 8, 9, 10, 13]) def test_adversarial_chunk_splits(self, chunksize: int) -> None: - decoder = gateway_base.FrameDecoder() + decoder = _message.FrameDecoder() blob = b"".join(m.pack() for m in self._messages()) got: list[Message] = [] for start in range(0, len(blob), chunksize): @@ -282,14 +285,14 @@ def test_adversarial_chunk_splits(self, chunksize: int) -> None: decoder.close() def test_close_mid_frame_raises(self) -> None: - decoder = gateway_base.FrameDecoder() + decoder = _message.FrameDecoder() blob = Message(Message.CHANNEL_DATA, 1, b"hello").pack() assert list(decoder.feed(blob[:-2])) == [] with pytest.raises(EOFError, match="mid-frame"): decoder.close() def test_feed_buffers_even_when_not_iterated(self) -> None: - decoder = gateway_base.FrameDecoder() + decoder = _message.FrameDecoder() blob = Message(Message.CHANNEL_DATA, 5, b"data").pack() decoder.feed(blob[:4]) # result deliberately not iterated (msg,) = decoder.feed(blob[4:]) @@ -317,7 +320,7 @@ async def sender() -> None: await theirs.send_eof() async def receiver() -> None: - decoder = gateway_base.FrameDecoder() + decoder = _message.FrameDecoder() while True: data = await ours.receive_some(4096) if not data: @@ -382,13 +385,13 @@ def test_channel_makefile_incompatmode(self, fac) -> None: class TestSourceOfFunction: def test_lambda_unsupported(self) -> None: - pytest.raises(ValueError, gateway._source_of_function, lambda: 1) + pytest.raises(ValueError, _exec_source._source_of_function, lambda: 1) def test_wrong_prototype_fails(self) -> None: def prototype(wrong) -> None: pass - pytest.raises(ValueError, gateway._source_of_function, prototype) + pytest.raises(ValueError, _exec_source._source_of_function, prototype) def test_function_without_known_source_fails(self) -> None: # this one won't be able to find the source @@ -396,7 +399,7 @@ def test_function_without_known_source_fails(self) -> None: exec("def fail(channel): pass", mess, mess) print(inspect.getsourcefile(mess["fail"])) with pytest.raises(ValueError): - gateway._source_of_function(mess["fail"]) + _exec_source._source_of_function(mess["fail"]) def test_function_with_closure_fails(self) -> None: mess: dict[str, Any] = {} @@ -405,13 +408,13 @@ def closure(channel: object) -> None: print(mess) with pytest.raises(ValueError): - gateway._source_of_function(closure) + _exec_source._source_of_function(closure) def test_source_of_nested_function(self) -> None: def working(channel: object) -> None: pass - send_source = gateway._source_of_function(working).lstrip("\r\n") + send_source = _exec_source._source_of_function(working).lstrip("\r\n") expected = "def working(channel: object) -> None:\n pass\n" assert send_source == expected @@ -420,7 +423,7 @@ class TestGlobalFinder: def check(self, func) -> list[str]: src = textwrap.dedent(inspect.getsource(func)) code = func.__code__ - return gateway._find_non_builtin_globals(src, code) + return _exec_source._find_non_builtin_globals(src, code) def test_local(self) -> None: def f(a, b, c): @@ -447,7 +450,7 @@ def test_function_with_global_fails(self) -> None: def func(channel) -> None: sys - pytest.raises(ValueError, gateway._source_of_function, func) + pytest.raises(ValueError, _exec_source._source_of_function, func) def test_method_call(self) -> None: # method names are reason @@ -460,7 +463,7 @@ def f(channel): @skip_win_pypy def test_remote_exec_function_with_kwargs( - anypython: str, makegateway: Callable[[str], gateway.Gateway] + anypython: str, makegateway: Callable[[str], execnet.Gateway] ) -> None: def func(channel, data) -> None: channel.send(data) @@ -473,17 +476,17 @@ def func(channel, data) -> None: assert result == 1 -def test_remote_exc__no_kwargs(makegateway: Callable[[], gateway.Gateway]) -> None: +def test_remote_exc__no_kwargs(makegateway: Callable[[], execnet.Gateway]) -> None: gw = makegateway() with pytest.raises(TypeError): - gw.remote_exec(gateway_base, kwarg=1) + gw.remote_exec(_message, kwarg=1) with pytest.raises(TypeError): gw.remote_exec("pass", kwarg=1) @skip_win_pypy def test_remote_exec_inspect_stack( - makegateway: Callable[[], gateway.Gateway], + makegateway: Callable[[], execnet.Gateway], ) -> None: gw = makegateway() ch = gw.remote_exec( diff --git a/testing/test_channel.py b/testing/test_channel.py index f1b0b454..8bb292a0 100644 --- a/testing/test_channel.py +++ b/testing/test_channel.py @@ -9,8 +9,8 @@ import pytest -from execnet.gateway import Gateway -from execnet.gateway_base import Channel +from execnet import Gateway +from execnet._channel import Channel needs_early_gc = pytest.mark.skipif("not hasattr(sys, 'getrefcount')") needs_osdup = pytest.mark.skipif("not hasattr(os, 'dup')") diff --git a/testing/test_channel_stress.py b/testing/test_channel_stress.py index 12f69d47..f5637efb 100644 --- a/testing/test_channel_stress.py +++ b/testing/test_channel_stress.py @@ -24,7 +24,7 @@ from hypothesis import given # noqa: E402 from hypothesis import strategies as st # noqa: E402 -from execnet.gateway import Gateway # noqa: E402 +from execnet import Gateway # noqa: E402 # High --stress levels replay one test many times; lift the per-test timeout # well above the default so that only a real hang (bounded by TESTTIMEOUT on diff --git a/testing/test_compatibility_regressions.py b/testing/test_compatibility_regressions.py index 837c0c6d..ed06be4e 100644 --- a/testing/test_compatibility_regressions.py +++ b/testing/test_compatibility_regressions.py @@ -1,8 +1,8 @@ -from execnet import gateway_base +from execnet import _serialize def test_opcodes() -> None: - data = vars(gateway_base.opcode) + data = vars(_serialize.opcode) computed = {k: v for k, v in data.items() if "__" not in k} assert computed == { "BUILDTUPLE": b"@", diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py index 84585ba9..4e568d7e 100644 --- a/testing/test_execmodel_trio.py +++ b/testing/test_execmodel_trio.py @@ -14,7 +14,7 @@ import execnet import execnet.trio -from execnet.gateway import Gateway +from execnet import Gateway TESTTIMEOUT = 10.0 diff --git a/testing/test_gateway.py b/testing/test_gateway.py index 6b6d5cf5..ef8e8aa9 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -16,8 +16,9 @@ import pytest import execnet -from execnet import gateway_base -from execnet.gateway import Gateway +from execnet import Gateway +from execnet import _execmodel +from execnet import _trace TESTTIMEOUT = 10.0 # seconds needs_osdup = pytest.mark.skipif("not hasattr(os, 'dup')") @@ -169,7 +170,7 @@ def run_me(channel=None): ch = gw.remote_exec(remotetest) try: ch.receive() - except execnet.gateway_base.RemoteError as e: + except execnet.RemoteError as e: assert 'remotetest.py", line 3, in run_me' in str(e) assert "ValueError: me" in str(e) finally: @@ -178,7 +179,7 @@ def run_me(channel=None): ch = gw.remote_exec(remotetest.run_me) try: ch.receive() - except execnet.gateway_base.RemoteError as e: + except execnet.RemoteError as e: assert 'remotetest.py", line 3, in run_me' in str(e) assert "ValueError: me" in str(e) finally: @@ -346,9 +347,9 @@ def test_chdir_separation( assert x.lower() == str(tmp_path).lower() def test_remoteerror_readable_traceback(self, gw: Gateway) -> None: - with pytest.raises(gateway_base.RemoteError) as e: + with pytest.raises(execnet.RemoteError) as e: gw.remote_exec("x y").waitclose() - assert "gateway_base" in e.value.formatted + assert "_gateway_base" in e.value.formatted def test_many_popen(self, makegateway: Callable[[str], Gateway]) -> None: num = 4 @@ -523,9 +524,7 @@ def test_popen_filetracing( monkeypatch.setenv("EXECNET_DEBUG", "1") gw = makegateway("popen") # hack out the debuffilename - fn = gw.remote_exec( - "import execnet;channel.send(execnet.gateway_base.fn)" - ).receive() + fn = gw.remote_exec("import execnet;channel.send(execnet._trace.fn)").receive() assert isinstance(fn, str) workerfile = pathlib.Path(fn) assert workerfile.exists() @@ -555,7 +554,7 @@ def test_popen_stderr_tracing( gw.exit() def test_no_tracing_by_default(self): - assert gateway_base.trace == gateway_base.notrace, ( + assert _trace.trace == _trace.notrace, ( "trace does not to default to empty tracing" ) @@ -584,7 +583,7 @@ def test_popen_args(spec: str, expected_args: list[str]) -> None: def test_assert_main_thread_only( - execmodel: gateway_base.ExecModel, makegateway: Callable[[str], Gateway] + execmodel: _execmodel.ExecModel, makegateway: Callable[[str], Gateway] ) -> None: if execmodel.backend != "main_thread_only": pytest.skip("can only run with main_thread_only") @@ -623,7 +622,7 @@ def test_assert_main_thread_only( def test_main_thread_only_concurrent_remote_exec_deadlock( - execmodel: gateway_base.ExecModel, makegateway: Callable[[str], Gateway] + execmodel: _execmodel.ExecModel, makegateway: Callable[[str], Gateway] ) -> None: if execmodel.backend != "main_thread_only": pytest.skip("can only run with main_thread_only") @@ -651,7 +650,7 @@ def test_main_thread_only_concurrent_remote_exec_deadlock( expected_results = ( True, - execnet.gateway_base.MAIN_THREAD_ONLY_DEADLOCK_TEXT, + execnet._errors.MAIN_THREAD_ONLY_DEADLOCK_TEXT, ) for expected, ch in zip(expected_results, channels, strict=True): try: diff --git a/testing/test_gevent.py b/testing/test_gevent.py index 91f74ef4..cbec77a9 100644 --- a/testing/test_gevent.py +++ b/testing/test_gevent.py @@ -155,9 +155,9 @@ def test_status_reports_gevent(self, worker_gw) -> None: def test_provisioning_adds_gevent_requirement() -> None: + from execnet import XSpec from execnet._provision import _extra_with_tokens from execnet._provision import worker_cli_arg - from execnet.xspec import XSpec spec = XSpec("popen//id=g1//execmodel=gevent") config = worker_cli_arg(spec) diff --git a/testing/test_multi.py b/testing/test_multi.py index 2f1b634e..156879b3 100644 --- a/testing/test_multi.py +++ b/testing/test_multi.py @@ -12,12 +12,12 @@ import pytest import execnet +from execnet import Gateway +from execnet import Group from execnet import XSpec -from execnet.gateway import Gateway -from execnet.gateway_base import Channel -from execnet.gateway_base import ExecModel -from execnet.multi import Group -from execnet.multi import safe_terminate +from execnet._channel import Channel +from execnet._execmodel import ExecModel +from execnet._multi import safe_terminate class TestMultiChannelAndGateway: diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index 0317bc2b..822f8d50 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -1,20 +1,33 @@ -"""The three public namespaces: execnet.sync / execnet.trio / execnet.portal. +"""The public namespaces, and the deprecation shims for the private modules. -Top-level ``execnet.*`` is an alias surface over ``execnet.sync``; the -trio namespace exposes the async-native core; the portal namespace the -cross-thread primitives. +Top-level ``execnet.*`` is an alias surface over ``execnet.sync``; the trio +and aio namespaces expose the async-native core and its asyncio bridge; the +portal namespace the cross-thread primitives. Everything else in the package +is private -- the pre-Trio module names survive only as warning shims. """ from __future__ import annotations +import importlib +import pkgutil import subprocess import sys +import warnings + +import pytest import execnet +import execnet.aio import execnet.portal import execnet.sync import execnet.trio +#: the only modules that may be reachable without a leading underscore +PUBLIC_NAMESPACES = ("aio", "portal", "sync", "trio") + +#: pre-Trio module names kept as deprecated forwarding shims +SHIMS = ("gateway", "gateway_base", "multi", "rsync", "rsync_remote", "xspec") + def test_top_level_names_alias_execnet_sync() -> None: for name in execnet.sync.__all__: @@ -22,7 +35,30 @@ def test_top_level_names_alias_execnet_sync() -> None: def test_top_level_all_matches_sync_surface() -> None: - assert set(execnet.__all__) == set(execnet.sync.__all__) | {"__version__"} + # the top level is the sync surface plus the two package-level names: + # the version, and can_send (a wire-format fact, not a gateway API) + assert set(execnet.__all__) == set(execnet.sync.__all__) | { + "__version__", + "can_send", + } + + +@pytest.mark.parametrize( + "namespace", + [execnet, *[importlib.import_module(f"execnet.{n}") for n in PUBLIC_NAMESPACES]], +) +def test_namespace_all_resolves(namespace: object) -> None: + missing = [n for n in namespace.__all__ if not hasattr(namespace, n)] # type: ignore[attr-defined] + assert not missing + + +def test_no_unexpected_public_modules() -> None: + found = { + info.name + for info in pkgutil.iter_modules(execnet.__path__) + if not info.name.startswith("_") + } + assert found == set(PUBLIC_NAMESPACES) | set(SHIMS) def test_trio_namespace_exposes_async_core() -> None: @@ -38,6 +74,35 @@ def test_trio_namespace_exposes_async_core() -> None: assert not hasattr(execnet.trio, "dumps") +def test_trio_namespace_hides_raw_plumbing() -> None: + # the raw-channel/stream layer is internal routing detail: reachable from + # execnet._trio_gateway, not advertised on the public namespace + for name in ("ByteStream", "RawChannel", "RawChannelStream", "serve_gateway"): + assert name not in execnet.trio.__all__, name + + +def test_can_send_lives_only_on_the_top_level() -> None: + assert execnet.can_send({"a": [1, 2.0, b"x", None, (True, frozenset({3}))]}) + assert not execnet.can_send(object()) + # the wire contract does not vary by surface, so it is not mirrored + for namespace in (execnet.sync, execnet.trio, execnet.aio, execnet.portal): + assert "can_send" not in namespace.__all__, namespace.__name__ + + +def test_dumps_is_a_temporary_xdist_shim() -> None: + # FOLLOW-UP: delete this test together with execnet._XDIST_COMPAT once + # pytest-xdist stops probing with ``execnet.dumps`` / ``except DumpError`` + # and uses ``execnet.can_send`` instead. + from execnet import _serialize + + assert execnet._XDIST_COMPAT == ("dumps",) + with pytest.warns(DeprecationWarning, match="temporary pytest-xdist"): + assert execnet.dumps is _serialize.dumps + # reachable, but never advertised + assert "dumps" not in execnet.__all__ + assert "dumps" not in dir(execnet) + + def test_portal_namespace() -> None: assert execnet.portal.__all__ == [ "LoopPortal", @@ -81,3 +146,48 @@ def test_import_execnet_does_not_import_trio() -> None: check=True, ) assert out.stdout.strip() == "False" + + +@pytest.mark.parametrize("shim", SHIMS) +def test_shim_reachable_as_package_attribute(shim: str) -> None: + # pytest-xdist reaches these as ``execnet.gateway_base.X`` after a plain + # ``import execnet``; that used to work because the import chain pulled + # them in, and must keep working for as long as the shims exist. + out = subprocess.run( + [sys.executable, "-c", f"import execnet; print(execnet.{shim}.__name__)"], + capture_output=True, + text=True, + check=True, + ) + assert out.stdout.strip() == f"execnet.{shim}" + + +def shim_attributes() -> list[tuple[str, str, str]]: + """``(shim, attribute, private module)`` for every forwarded name.""" + cases = [] + for shim in SHIMS: + module = importlib.import_module(f"execnet.{shim}") + for name, target in module._MOVED.items(): + cases.append((shim, name, target)) + return cases + + +@pytest.mark.parametrize(("shim", "name", "target"), shim_attributes(), ids=str) +def test_shim_warns_and_forwards(shim: str, name: str, target: str) -> None: + module = importlib.import_module(f"execnet.{shim}") + private = importlib.import_module(f"execnet{target}") + if not hasattr(private, name): + # ``trace``/``notrace``/``fn`` exist only under a given EXECNET_DEBUG + pytest.skip(f"execnet{target}.{name} not defined in this configuration") + with pytest.warns(DeprecationWarning, match=f"execnet.{shim} is private"): + value = getattr(module, name) + assert value is getattr(private, name) + + +@pytest.mark.parametrize("shim", SHIMS) +def test_shim_rejects_unknown_attribute(shim: str) -> None: + module = importlib.import_module(f"execnet.{shim}") + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + with pytest.raises(AttributeError, match="no attribute 'nonexistent'"): + getattr(module, "nonexistent") # noqa: B009 diff --git a/testing/test_rsync.py b/testing/test_rsync.py index 2112439f..8190efc7 100644 --- a/testing/test_rsync.py +++ b/testing/test_rsync.py @@ -7,8 +7,8 @@ import pytest import execnet +from execnet import Gateway from execnet import RSync -from execnet.gateway import Gateway @pytest.fixture(scope="module") diff --git a/testing/test_serializer.py b/testing/test_serializer.py index ec8e86ae..ae88f80d 100644 --- a/testing/test_serializer.py +++ b/testing/test_serializer.py @@ -9,8 +9,8 @@ import execnet -# The package parent: the serializer is imported as execnet.gateway_base -# (it needs its trio-free sibling execnet._boundary; only stdlib beyond that). +# The package parent: the serializer is imported as execnet._serialize +# (it needs its sibling execnet._errors; only stdlib beyond that). pyimportdir = os.fspath(Path(execnet.__file__).parent.parent) @@ -25,7 +25,7 @@ def dump(self, obj_rep: str) -> bytes: f""" import sys sys.path.insert(0, {pyimportdir!r}) -import execnet.gateway_base as serializer +import execnet._serialize as serializer sys.stdout = sys.stdout.detach() sys.stdout.write(serializer.dumps_internal({obj_rep})) """ @@ -41,7 +41,7 @@ def load(self, data: bytes) -> list[str]: rf""" import sys sys.path.insert(0, {pyimportdir!r}) -import execnet.gateway_base as serializer +import execnet._serialize as serializer from io import BytesIO data = {data!r} io = BytesIO(data) @@ -190,4 +190,4 @@ def test_tuple_nested_with_empty_in_between(dump, load) -> None: def test_py2_string_opcode_is_retired() -> None: """The py2 ``str`` opcode ``M`` is gone; only Python2 ever emitted it.""" with pytest.raises(execnet.DataFormatError, match="unknown opcode"): - execnet.gateway_base.loads(b"\x02M\x00\x00\x00\x01aQ") + execnet._serialize.loads(b"\x02M\x00\x00\x00\x01aQ") diff --git a/testing/test_termination.py b/testing/test_termination.py index 01c57427..03d6bbb8 100644 --- a/testing/test_termination.py +++ b/testing/test_termination.py @@ -12,8 +12,8 @@ from test_gateway import TESTTIMEOUT import execnet -from execnet.gateway import Gateway -from execnet.gateway_base import ExecModel +from execnet import Gateway +from execnet._execmodel import ExecModel execnetdir = pathlib.Path(execnet.__file__).parent.parent diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index ef52e232..35a62c50 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -15,14 +15,14 @@ import trio import trio.testing -from execnet import gateway_base +from execnet import _errors +from execnet._errors import RemoteError +from execnet._message import Message +from execnet._serialize import dumps_internal +from execnet._serialize import loads_internal from execnet._trio_gateway import AsyncGateway from execnet._trio_gateway import AsyncGroup from execnet._trio_gateway import open_popen_gateway -from execnet.gateway_base import Message -from execnet.gateway_base import RemoteError -from execnet.gateway_base import dumps_internal -from execnet.gateway_base import loads_internal @asynccontextmanager @@ -224,7 +224,7 @@ def test_channel_receive_timeout() -> None: async def main() -> None: async with gateway_pair() as (left, _right): channel = left.open_channel() - with pytest.raises(gateway_base.TimeoutError): + with pytest.raises(_errors.TimeoutError): await channel.receive(timeout=0.05) trio.run(main) diff --git a/testing/test_xspec.py b/testing/test_xspec.py index 850acbe1..3bcee98a 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -10,9 +10,9 @@ import pytest import execnet +from execnet import Gateway from execnet import XSpec from execnet import _provision -from execnet.gateway import Gateway skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), From 7aa17fed8ef4f703260050cd332ef9dd314106b5 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 28 Jul 2026 22:22:09 +0200 Subject: [PATCH 61/91] refactor!: retire the wakener extension point, add execnet.gevent There is no plan to let third parties plug in event loops, so the Wakener registry was machinery in search of a user: execnet.portal published the protocol and the carriers but not register_wakener, and the promised asyncio wakener was never built (P4 chose per-call bridging). What is left is two wait backends -- OS threads and gevent greenlets -- because every other concurrency library gets a facade. So make the boundary kit private (execnet.portal -> execnet._portal), collapse the registry into a two-branch make_wakener over a WaitBackend literal, and give gevent the facade it was missing. wait= disappears as a spec key: which primitive a *caller* parks on is what picking a namespace already says, and the worker's backend was derived from its profile anyway. Two warts fixed while the carriers were private again: they now raise execnet.TimeoutError instead of the builtin (dropping the translation in Channel.receive), and OneShot's double-resolve check is a RuntimeError rather than an assert that vanishes under -O. Co-Authored-By: Claude Opus 5 (1M context) --- src/execnet/__init__.py | 18 ++--- src/execnet/_boundary.py | 67 ++++++++---------- src/execnet/_channel.py | 6 +- src/execnet/_gateway.py | 2 - src/execnet/_gateway_base.py | 8 ++- src/execnet/_gevent_support.py | 13 ++-- src/execnet/_multi.py | 42 ++++++----- src/execnet/{portal.py => _portal.py} | 16 ++--- src/execnet/_provision.py | 6 +- src/execnet/_shim.py | 4 +- src/execnet/_trio_host.py | 14 ++-- src/execnet/_trio_worker.py | 2 +- src/execnet/_xspec.py | 1 - src/execnet/gateway_base.py | 2 +- src/execnet/gevent.py | 73 ++++++++++++++++++++ testing/{test_portal.py => test_boundary.py} | 18 +++-- testing/test_gevent.py | 11 +-- testing/test_namespaces.py | 60 ++++++++++------ testing/test_xspec.py | 10 +-- 19 files changed, 228 insertions(+), 145 deletions(-) rename src/execnet/{portal.py => _portal.py} (83%) create mode 100644 src/execnet/gevent.py rename testing/{test_portal.py => test_boundary.py} (76%) diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 5912ae9a..87626d99 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -4,16 +4,18 @@ pure python lib for connecting to local and remote Python Interpreters. -Four public namespaces: +One namespace per concurrency library you drive execnet from: -* :mod:`execnet.sync` — the blocking API; the top-level ``execnet.*`` - names below are aliases into it. +* :mod:`execnet.sync` — the blocking API for plain threads; the top-level + ``execnet.*`` names below are aliases into it. * :mod:`execnet.trio` — the trio-native API, awaited inside your own ``trio.run``. -* :mod:`execnet.aio` — the asyncio-native API, bridged over a Trio - host thread. -* :mod:`execnet.portal` — cross-thread / cross-loop communication - primitives shared by all of them. +* :mod:`execnet.aio` — the asyncio-native API. +* :mod:`execnet.gevent` — the blocking API with greenlet-parking waits. + +:mod:`execnet.trio` is the only one that runs gateways *directly* as tasks +in your own nursery. The other three drive a shared Trio host thread, so +their blocking calls must not be made from inside a running event loop. ``can_send`` sits here rather than on any one of them: the wire-format contract is the same whichever surface you drive a gateway from. @@ -68,7 +70,7 @@ #: that still do ``execnet.gateway_base.X`` must reach the warning shim. _LAZY_MODULES = ( "aio", - "portal", + "gevent", "trio", "gateway", "gateway_base", diff --git a/src/execnet/_boundary.py b/src/execnet/_boundary.py index 9c7c69f4..fcd4bae6 100644 --- a/src/execnet/_boundary.py +++ b/src/execnet/_boundary.py @@ -1,7 +1,12 @@ """Trio-free half of the boundary kit: Wakener, Mailbox, OneShot. Importable without loading any event loop (``import execnet`` must not -import trio); ``execnet.portal`` re-exports these next to LoopPortal. +import trio); :mod:`execnet._portal` re-exports these next to LoopPortal. + +All of this is internal. There are exactly two wait backends -- OS +threads and gevent greenlets -- and no plan to let third parties add +more: every other concurrency library gets a facade of its own +(:mod:`execnet.trio`, :mod:`execnet.aio`) rather than a wakener. """ from __future__ import annotations @@ -9,25 +14,30 @@ import queue import threading import time -from collections.abc import Callable from typing import Any from typing import Generic +from typing import Literal from typing import Protocol from typing import TypeVar from typing import cast +from ._errors import TimeoutError + __all__ = [ "Flag", "Mailbox", "OneShot", "ThreadWakener", + "WaitBackend", "Wakener", "make_wakener", - "register_wakener", ] T = TypeVar("T") +#: which primitive a facade's blocking waits park on +WaitBackend = Literal["thread", "gevent"] + class Wakener(Protocol): """Thread-safe consumer wakeup fired by the loop. @@ -36,8 +46,7 @@ class Wakener(Protocol): the only thing the loop side ever calls. The blocking mailbox/oneshot waits additionally need :meth:`wait`/:meth:`clear` executed in the consumer's own context (a thread here, a greenlet for a gevent - wakener); loop-native consumers such as asyncio wait on their own - side of ``notify`` instead. + wakener). """ def notify(self) -> None: ... @@ -141,14 +150,16 @@ def is_set(self) -> bool: def set(self, value: T) -> None: """Thread-safe; usable from a loop thread (never blocks).""" - assert not self._done, "OneShot already resolved" + if self._done: + raise RuntimeError("OneShot already resolved") self._value = value self._done = True self._wakener.notify() def set_error(self, error: BaseException) -> None: """Resolve with an error that :meth:`wait` will re-raise.""" - assert not self._done, "OneShot already resolved" + if self._done: + raise RuntimeError("OneShot already resolved") self._error = error self._done = True self._wakener.notify() @@ -203,35 +214,17 @@ def wait(self, timeout: float | None = None) -> bool: return self._flag -# wait= axis: named wakener factories; each call returns a fresh instance -# (carriers own their wakener exclusively, see Flag). Backends register -# here next to the built-in "thread"; entries in the lazy table import -# their module (which registers itself) on first use. -_WAKENER_FACTORIES: dict[str, Callable[[], Wakener]] = { - "thread": ThreadWakener, -} -_LAZY_WAKENER_MODULES: dict[str, str] = { - "gevent": "execnet._gevent_support", -} - - -def register_wakener(name: str, factory: Callable[[], Wakener]) -> None: - """Register a wakener factory for the ``wait=`` spec axis.""" - _WAKENER_FACTORIES[name] = factory +def make_wakener(backend: WaitBackend) -> Wakener: + """Create a fresh wakener for a wait backend. + Each carrier owns its wakener exclusively (see :class:`Flag`), so this + always returns a new instance. gevent is imported lazily -- it is an + optional dependency of the :mod:`execnet.gevent` facade. + """ + if backend == "thread": + return ThreadWakener() + if backend == "gevent": + from ._gevent_support import GeventWakener -def wakener_names() -> list[str]: - return sorted(set(_WAKENER_FACTORIES) | set(_LAZY_WAKENER_MODULES)) - - -def make_wakener(name: str) -> Wakener: - """Create a fresh wakener for the named wait backend.""" - if name not in _WAKENER_FACTORIES and name in _LAZY_WAKENER_MODULES: - import importlib - - importlib.import_module(_LAZY_WAKENER_MODULES[name]) - try: - factory = _WAKENER_FACTORIES[name] - except KeyError: - raise ValueError(f"unknown wait backend {name!r}") from None - return factory() + return GeventWakener() + raise ValueError(f"unknown wait backend {backend!r}") diff --git a/src/execnet/_channel.py b/src/execnet/_channel.py index e1f5449b..a13f846b 100644 --- a/src/execnet/_channel.py +++ b/src/execnet/_channel.py @@ -8,7 +8,6 @@ from __future__ import annotations -import builtins import threading import weakref from collections.abc import Callable @@ -326,10 +325,7 @@ def receive(self, timeout: float | None = None) -> Any: mailbox = self._mailbox if mailbox is None: raise OSError("cannot receive(), channel has receiver callback") - try: - x = mailbox.get(timeout) - except builtins.TimeoutError: - raise self.TimeoutError("no item after %r seconds" % timeout) from None + x = mailbox.get(timeout) if x is ENDMARKER: mailbox.put(x) # for other receivers raise self._getremoteerror() or EOFError() diff --git a/src/execnet/_gateway.py b/src/execnet/_gateway.py index facbdeca..b8cfe863 100644 --- a/src/execnet/_gateway.py +++ b/src/execnet/_gateway.py @@ -39,8 +39,6 @@ def __init__(self, io: IO, spec: XSpec) -> None: """ super().__init__(io=io, id=spec.id, _startcount=1) self.spec = spec - if spec.wait: - self._wait_backend = spec.wait @property def remoteaddress(self) -> str: diff --git a/src/execnet/_gateway_base.py b/src/execnet/_gateway_base.py index 16b21602..55e8bb09 100644 --- a/src/execnet/_gateway_base.py +++ b/src/execnet/_gateway_base.py @@ -23,6 +23,7 @@ from contextlib import suppress from typing import Any +from ._boundary import WaitBackend from ._boundary import Wakener from ._boundary import make_wakener from ._channel import Channel @@ -41,9 +42,10 @@ class BaseGateway: _trio_session: Any = None # Set by the receiver on EOF without a prior termination message. _error: BaseException | None = None - #: wait= axis: which wakener backend this gateway's blocking waits - #: park on (channels, write-acks, join) - _wait_backend: str = "thread" + #: which primitive this gateway's blocking waits park on (channels, + #: write-acks, join). Inherited from the facade coordinator-side, and + #: derived from the worker profile worker-side. + _wait_backend: WaitBackend = "thread" def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel diff --git a/src/execnet/_gevent_support.py b/src/execnet/_gevent_support.py index a999ab96..17cb7993 100644 --- a/src/execnet/_gevent_support.py +++ b/src/execnet/_gevent_support.py @@ -1,9 +1,9 @@ -"""The gevent wait backend (``wait=gevent``). +"""The gevent wait backend behind :mod:`execnet.gevent`. -Importing this module registers the ``gevent`` wakener factory; the -boundary kit imports it lazily when a spec asks for ``wait=gevent``. +The boundary kit imports this lazily (``make_wakener("gevent")``) so +gevent stays an optional dependency. -A blocking wait then parks only the calling greenlet: the carrier's +A blocking wait parks only the calling greenlet: the carrier's event lives in the waiting greenlet's hub, and the loop side's ``notify()`` crosses threads through a ``loop.async_`` watcher -- the one libev/libuv primitive gevent documents as safe to use from other @@ -18,8 +18,6 @@ import gevent.event from gevent.hub import get_hub -from ._boundary import register_wakener - class GeventWakener: """Wakener parking greenlets instead of OS threads. @@ -71,6 +69,3 @@ def clear(self) -> None: event = self._event if event is not None: event.clear() - - -register_wakener("gevent", GeventWakener) diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py index 429103f7..113ff1c2 100644 --- a/src/execnet/_multi.py +++ b/src/execnet/_multi.py @@ -23,7 +23,7 @@ from typing import TypeAlias from typing import overload -from ._boundary import wakener_names +from ._boundary import WaitBackend from ._channel import Channel from ._execmodel import EXECMODEL_PROFILES from ._execmodel import ExecModel @@ -43,6 +43,12 @@ class Group: defaultspec = "popen" + #: which primitive this group's blocking waits park on. Set by the + #: facade, not by a spec: it describes the *caller's* concurrency + #: library, which is exactly what picking a namespace already says. + #: ``execnet.gevent.Group`` overrides it. + _wait_backend: WaitBackend = "thread" + def __init__( self, xspecs: Iterable[XSpec | str | None] = (), execmodel: str = "thread" ) -> None: @@ -178,10 +184,6 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: f"unknown execmodel {spec.execmodel!r}" f" (known profiles: {list(EXECMODEL_PROFILES)})" ) - if spec.wait is not None and spec.wait not in wakener_names(): - raise ValueError( - f"unknown wait backend {spec.wait!r} (known: {wakener_names()})" - ) from . import _trio_host if not (spec.socket or spec.via or spec.ssh or spec.vagrant_ssh or spec.popen): @@ -256,26 +258,22 @@ def terminate(self, timeout: float | None = None) -> None: self._gateways_to_join[:] = [] def _host_terminate(self, timeout: float | None) -> None: - """Terminate the async group, parking correctly for wait backends. + """Terminate the async group, parking correctly for the wait backend. - A member gateway created with a non-thread ``wait=`` implies the - caller may be a greenlet: wait on a OneShot instead of blocking - the OS thread (which would stall the hub for the whole grace). + A non-thread backend implies the caller may be a greenlet: wait on + a OneShot instead of blocking the OS thread (which would stall the + hub for the whole grace). """ - backends = {gw._wait_backend for gw in self._gateways_to_join} | { - gw._wait_backend for gw in self - } - backends.discard("thread") - if backends: - from ._boundary import make_wakener - - self._trio_host.call_pending( - self._async_group.terminate, - timeout, - wakener=make_wakener(backends.pop()), - ).wait() - else: + if self._wait_backend == "thread": self._trio_host.call(self._async_group.terminate, timeout) + return + from ._boundary import make_wakener + + self._trio_host.call_pending( + self._async_group.terminate, + timeout, + wakener=make_wakener(self._wait_backend), + ).wait() def remote_exec( self, diff --git a/src/execnet/portal.py b/src/execnet/_portal.py similarity index 83% rename from src/execnet/portal.py rename to src/execnet/_portal.py index 5cc3e30e..6c1253c4 100644 --- a/src/execnet/portal.py +++ b/src/execnet/_portal.py @@ -1,17 +1,17 @@ """Cross-thread / cross-loop communication primitives — the boundary kit. -A :class:`LoopPortal` is a handle to a running trio loop that foreign -threads use to run functions on the loop or push work into it (the +Internal. A :class:`LoopPortal` is a handle to a running trio loop that +foreign threads use to run functions on the loop or push work into it (the consumer -> loop direction). Because each direction only needs the -*receiving* loop's token, two trio loops in two threads can communicate -by holding each other's portal. +*receiving* loop's token, two trio loops in two threads can communicate by +holding each other's portal. The loop -> consumer direction never blocks the loop and never knows who is listening: the loop fires a :class:`Wakener`, a single thread-safe -``notify()`` supplied by the consumer. Implementing that one method is -the entire integration surface for an event loop backend (threads here; -asyncio/gevent wakeners come with their facades). On top of it sit the -two carriers: +``notify()`` supplied by the consumer. There are exactly two wakeners -- +OS threads and gevent greenlets -- because every other concurrency library +gets a facade of its own (:mod:`execnet.trio`, :mod:`execnet.aio`) instead. +On top of the wakener sit the two carriers: * :class:`Mailbox` -- an item stream (channel payloads, exec requests), * :class:`OneShot` -- a single result (write acknowledgements, call diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index 4da4f689..dcfaf1a4 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -164,12 +164,12 @@ def worker_cli_arg(spec: Any) -> str: """ import execnet - # the gevent profile parks its greenlets on gevent wakeners - default_wait = "gevent" if spec.execmodel == "gevent" else "thread" config: dict[str, Any] = { "id": f"{spec.id}-worker", "execmodel": spec.execmodel, - "wait": spec.wait or default_wait, + # derived, not configurable: the gevent profile parks its greenlets + # on gevent wakeners, every other profile is thread-shaped. + "wait": "gevent" if spec.execmodel == "gevent" else "thread", "coordinator_version": execnet.__version__, } # Startup setup applied by the worker before serving (never through diff --git a/src/execnet/_shim.py b/src/execnet/_shim.py index 35e7682c..0839c2e3 100644 --- a/src/execnet/_shim.py +++ b/src/execnet/_shim.py @@ -8,7 +8,7 @@ whose ``__getattr__`` warns and forwards. The supported surfaces are :mod:`execnet` / :mod:`execnet.sync`, -:mod:`execnet.trio`, :mod:`execnet.aio` and :mod:`execnet.portal`. +:mod:`execnet.trio`, :mod:`execnet.aio` and :mod:`execnet.gevent`. """ from __future__ import annotations @@ -38,7 +38,7 @@ def __getattr__(name: str) -> Any: warnings.warn( f"execnet.{shim} is private and will be removed in {REMOVED_IN}; " f"{name} now lives in execnet{module}. The supported surfaces are " - f"execnet, execnet.sync, execnet.trio, execnet.aio and execnet.portal.", + f"execnet, execnet.sync, execnet.trio, execnet.aio and execnet.gevent.", DeprecationWarning, stacklevel=2, ) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index a1ea4acb..8e1d695b 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -45,8 +45,8 @@ from ._trio_gateway import open_popen_process from ._trio_gateway import read_handshake_ack from ._trio_gateway import ssh_transport_args -from .portal import LoopPortal -from .portal import OneShot +from ._portal import LoopPortal +from ._portal import OneShot #: default cap on concurrent threadpool threads running receiver callbacks DEFAULT_CALLBACK_THREADS = 40 @@ -719,6 +719,8 @@ def _make_gateway(self, stream: ByteStream, spec: Any) -> AsyncGateway: import execnet sync_gw = execnet.Gateway(_TempIO(self.group.execmodel), spec) + # the caller's concurrency library, inherited from the facade + sync_gw._wait_backend = self.group._wait_backend return SyncBridgeGateway( stream, id=spec.id, sync_gateway=sync_gw, host=self.host ) @@ -762,16 +764,16 @@ def makegateway_trio(group: Group, spec: Any) -> Gateway: """Create a sync-facade Gateway for ``spec`` on the group's Trio host.""" host: TrioHost = group._ensure_trio_host() async_group: FacadeAsyncGroup = group._ensure_async_group() - if spec.wait and spec.wait != "thread": + if group._wait_backend == "thread": + bridge = host.call(async_group.makegateway, spec) + else: # e.g. a gevent app: wait on a OneShot so only the calling # greenlet parks while the gateway comes up, not the whole hub. from ._boundary import make_wakener bridge = host.call_pending( - async_group.makegateway, spec, wakener=make_wakener(spec.wait) + async_group.makegateway, spec, wakener=make_wakener(group._wait_backend) ).wait() - else: - bridge = host.call(async_group.makegateway, spec) assert isinstance(bridge, SyncBridgeGateway) gw: Gateway = bridge.sync_gateway # type: ignore[assignment] gw._io = SyncIOHandle( diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 633b7d94..741c5eff 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -18,7 +18,7 @@ from ._gateway_base import WorkerGateway from ._serialize import loads_internal from ._trace import trace -from .portal import Mailbox +from ._boundary import Mailbox if TYPE_CHECKING: from . import _trio_host diff --git a/src/execnet/_xspec.py b/src/execnet/_xspec.py index afc58f4c..0559ed8c 100644 --- a/src/execnet/_xspec.py +++ b/src/execnet/_xspec.py @@ -29,7 +29,6 @@ class XSpec: ssh_config: str | None = None vagrant_ssh: str | None = None via: str | None = None - wait: str | None = None def __init__(self, string: str) -> None: self._spec = string diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index b54c68bf..ba1eb35f 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -6,7 +6,7 @@ to them with a :class:`DeprecationWarning`. Use :mod:`execnet` / :mod:`execnet.sync`, :mod:`execnet.trio`, -:mod:`execnet.aio` or :mod:`execnet.portal` instead. The standalone +:mod:`execnet.aio` or :mod:`execnet.gevent` instead. The standalone serializer stays internal; :func:`execnet.can_send` answers "can this value cross a channel?" without it. """ diff --git a/src/execnet/gevent.py b/src/execnet/gevent.py new file mode 100644 index 00000000..f3f12609 --- /dev/null +++ b/src/execnet/gevent.py @@ -0,0 +1,73 @@ +"""The blocking execnet API for gevent applications. + +Identical to :mod:`execnet.sync` except that every blocking wait parks the +calling *greenlet* rather than its OS thread:: + + import execnet.gevent + + group = execnet.gevent.Group() + gateway = group.makegateway("popen") + channel = gateway.remote_exec("channel.send(6 * 7)") + print(channel.receive()) # parks this greenlet, not the hub + +Protocol IO runs on the shared Trio host thread as it does for every +blocking surface; the difference is only which primitive a waiter parks +on, so a slow ``receive`` no longer stalls the whole hub. Requires +gevent (``execnet[gevent]``). + +This is about the *caller*: the worker's own shape is the ``profile=`` +spec key, and ``profile=gevent`` is an independent choice. Importing this +module does not monkey-patch anything -- do that yourself, as early as +usual. +""" + +from __future__ import annotations + +import gevent # noqa: F401 -- fail at import time when gevent is missing + +from ._errors import DataFormatError +from ._errors import DumpError +from ._errors import HostNotFound +from ._errors import LoadError +from ._errors import RemoteError +from ._errors import TimeoutError +from ._multi import Group as _SyncGroup +from ._multi import MultiChannel +from ._xspec import XSpec +from .sync import Channel +from .sync import Gateway +from .sync import RSync + +__all__ = [ + "Channel", + "DataFormatError", + "DumpError", + "Gateway", + "Group", + "HostNotFound", + "LoadError", + "MultiChannel", + "RSync", + "RemoteError", + "TimeoutError", + "XSpec", + "default_group", + "makegateway", +] + + +class Group(_SyncGroup): + """A gateway group whose blocking waits park greenlets. + + Every gateway it creates inherits the gevent wait backend, so + ``channel.receive()``, ``waitclose()``, sends waiting on their write + acknowledgement, ``join()`` and ``Group.terminate()`` all yield to the + hub instead of blocking the thread running it. + """ + + _wait_backend = "gevent" + + +#: convenience group for scripts; real applications should own a Group +default_group = Group() +makegateway = default_group.makegateway diff --git a/testing/test_portal.py b/testing/test_boundary.py similarity index 76% rename from testing/test_portal.py rename to testing/test_boundary.py index 1a3759af..ce2a5302 100644 --- a/testing/test_portal.py +++ b/testing/test_boundary.py @@ -7,8 +7,9 @@ import pytest -from execnet.portal import Mailbox -from execnet.portal import OneShot +from execnet import TimeoutError as ExecnetTimeoutError +from execnet._boundary import Mailbox +from execnet._boundary import OneShot class TestMailbox: @@ -24,8 +25,10 @@ def test_get_nowait_empty(self) -> None: box.get_nowait() def test_get_timeout(self) -> None: + # the carriers speak execnet's TimeoutError, so Channel.receive + # does not have to translate the builtin one box: Mailbox[int] = Mailbox() - with pytest.raises(TimeoutError): + with pytest.raises(ExecnetTimeoutError): box.get(timeout=0.01) def test_get_blocks_until_put_from_other_thread(self) -> None: @@ -54,7 +57,7 @@ def test_set_then_wait(self) -> None: def test_wait_timeout(self) -> None: shot: OneShot[int] = OneShot() - with pytest.raises(TimeoutError): + with pytest.raises(ExecnetTimeoutError): shot.wait(timeout=0.01) assert not shot.is_set() @@ -69,8 +72,11 @@ def test_wait_blocks_until_set_from_other_thread(self) -> None: threading.Timer(0.05, shot.set, args=["done"]).start() assert shot.wait(timeout=5.0) == "done" - def test_single_resolution_asserted(self) -> None: + def test_single_resolution_enforced(self) -> None: + # a real error, not an assert: it must survive python -O shot: OneShot[int] = OneShot() shot.set(1) - with pytest.raises(AssertionError): + with pytest.raises(RuntimeError, match="already resolved"): shot.set(2) + with pytest.raises(RuntimeError, match="already resolved"): + shot.set_error(RuntimeError("boom")) diff --git a/testing/test_gevent.py b/testing/test_gevent.py index cbec77a9..5d8a84c3 100644 --- a/testing/test_gevent.py +++ b/testing/test_gevent.py @@ -1,4 +1,4 @@ -"""The gevent wait backend (wait=gevent): greenlet-parking blocking waits. +"""The execnet.gevent facade: greenlet-parking blocking waits. Opt-in: requires the ``gevent`` dependency group (``uv sync --group gevent``); skipped when gevent is not installed. No monkey-patching is @@ -15,6 +15,7 @@ gevent = pytest.importorskip("gevent") import execnet # noqa: E402 +import execnet.gevent # noqa: E402 from execnet._boundary import Flag # noqa: E402 from execnet._boundary import Mailbox # noqa: E402 from execnet._boundary import make_wakener # noqa: E402 @@ -24,9 +25,9 @@ @pytest.fixture def gevent_gw(): - group = execnet.Group() + group = execnet.gevent.Group() try: - yield group.makegateway("popen//wait=gevent") + yield group.makegateway("popen") finally: group.terminate(timeout=5.0) @@ -88,7 +89,7 @@ def test_waitclose_and_endmarker(self, gevent_gw: execnet.Gateway) -> None: def test_makegateway_parks_greenlet_not_hub(self) -> None: # management ops (makegateway/terminate) from a greenlet must not # stall the hub: they wait on a OneShot with a gevent wakener. - group = execnet.Group() + group = execnet.gevent.Group() progressed: list[int] = [] def other() -> None: @@ -98,7 +99,7 @@ def other() -> None: try: ticker = gevent.spawn(other) - maker = gevent.spawn(group.makegateway, "popen//wait=gevent") + maker = gevent.spawn(group.makegateway, "popen") gw = maker.get(timeout=TESTTIMEOUT) channel = gw.remote_exec("channel.send(42)") assert gevent.spawn(channel.receive, TESTTIMEOUT).get(TESTTIMEOUT) == 42 diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index 822f8d50..49f47bb4 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -1,9 +1,11 @@ """The public namespaces, and the deprecation shims for the private modules. -Top-level ``execnet.*`` is an alias surface over ``execnet.sync``; the trio -and aio namespaces expose the async-native core and its asyncio bridge; the -portal namespace the cross-thread primitives. Everything else in the package -is private -- the pre-Trio module names survive only as warning shims. +There is one namespace per concurrency library the caller drives execnet +from: top-level ``execnet.*`` is an alias surface over ``execnet.sync`` +(threads), ``execnet.trio`` and ``execnet.aio`` expose the async-native core +and its asyncio bridge, and ``execnet.gevent`` the greenlet-parking blocking +surface. Everything else in the package is private -- the pre-Trio module +names survive only as warning shims. """ from __future__ import annotations @@ -18,12 +20,14 @@ import execnet import execnet.aio -import execnet.portal import execnet.sync import execnet.trio #: the only modules that may be reachable without a leading underscore -PUBLIC_NAMESPACES = ("aio", "portal", "sync", "trio") +PUBLIC_NAMESPACES = ("aio", "gevent", "sync", "trio") + +#: those importable without an optional dependency (execnet.gevent needs gevent) +ALWAYS_IMPORTABLE = tuple(n for n in PUBLIC_NAMESPACES if n != "gevent") #: pre-Trio module names kept as deprecated forwarding shims SHIMS = ("gateway", "gateway_base", "multi", "rsync", "rsync_remote", "xspec") @@ -45,13 +49,22 @@ def test_top_level_all_matches_sync_surface() -> None: @pytest.mark.parametrize( "namespace", - [execnet, *[importlib.import_module(f"execnet.{n}") for n in PUBLIC_NAMESPACES]], + [execnet, *[importlib.import_module(f"execnet.{n}") for n in ALWAYS_IMPORTABLE]], ) def test_namespace_all_resolves(namespace: object) -> None: missing = [n for n in namespace.__all__ if not hasattr(namespace, n)] # type: ignore[attr-defined] assert not missing +def test_gevent_namespace_all_resolves() -> None: + pytest.importorskip("gevent") + namespace = importlib.import_module("execnet.gevent") + assert not [n for n in namespace.__all__ if not hasattr(namespace, n)] + # the facade is the sync surface with greenlet parking wired in + assert namespace.Group._wait_backend == "gevent" + assert issubclass(namespace.Group, execnet.Group) + + def test_no_unexpected_public_modules() -> None: found = { info.name @@ -85,7 +98,7 @@ def test_can_send_lives_only_on_the_top_level() -> None: assert execnet.can_send({"a": [1, 2.0, b"x", None, (True, frozenset({3}))]}) assert not execnet.can_send(object()) # the wire contract does not vary by surface, so it is not mirrored - for namespace in (execnet.sync, execnet.trio, execnet.aio, execnet.portal): + for namespace in (execnet.sync, execnet.trio, execnet.aio): assert "can_send" not in namespace.__all__, namespace.__name__ @@ -103,32 +116,35 @@ def test_dumps_is_a_temporary_xdist_shim() -> None: assert "dumps" not in dir(execnet) -def test_portal_namespace() -> None: - assert execnet.portal.__all__ == [ - "LoopPortal", - "Mailbox", - "OneShot", - "ThreadWakener", - "Wakener", - ] - assert execnet.portal.LoopPortal is not None +def test_boundary_kit_is_private() -> None: + # There is no third-party event-loop extension point: the two wait + # backends are threads and gevent, and every other concurrency library + # gets a facade instead of a wakener. + assert not hasattr(execnet, "portal") + for name in ("Wakener", "Mailbox", "OneShot", "LoopPortal"): + assert not hasattr(execnet, name), name + from execnet import _boundary + + assert not hasattr(_boundary, "register_wakener") + assert _boundary.make_wakener("thread") is not None + with pytest.raises(ValueError, match="unknown wait backend"): + _boundary.make_wakener("nope") # type: ignore[arg-type] def test_lazy_submodule_attribute_access() -> None: - # After ``import execnet`` alone, execnet.trio / execnet.portal are - # reachable as attributes (PEP 562) without having been imported. + # After ``import execnet`` alone, execnet.trio is reachable as an + # attribute (PEP 562) without having been imported. out = subprocess.run( [ sys.executable, "-c", - "import execnet; print(execnet.trio.AsyncGroup.__name__);" - " print(execnet.portal.LoopPortal.__name__)", + "import execnet; print(execnet.trio.AsyncGroup.__name__)", ], capture_output=True, text=True, check=True, ) - assert out.stdout.split() == ["AsyncGroup", "LoopPortal"] + assert out.stdout.strip() == "AsyncGroup" def test_import_execnet_does_not_import_trio() -> None: diff --git a/testing/test_xspec.py b/testing/test_xspec.py index 3bcee98a..891b6a1a 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -119,10 +119,12 @@ class TestMakegateway: def test_no_type(self, makegateway: Callable[[str], Gateway]) -> None: pytest.raises(ValueError, lambda: makegateway("hello")) - def test_wait_axis(self, makegateway: Callable[[str], Gateway]) -> None: - with pytest.raises(ValueError, match="unknown wait backend"): - makegateway("popen//wait=nope") - gw = makegateway("popen//wait=thread") + def test_wait_backend_comes_from_the_facade( + self, makegateway: Callable[[str], Gateway] + ) -> None: + # not a spec key: the blocking surface decides how *it* parks, and + # a thread-shaped worker profile parks on threads either way + gw = makegateway("popen") assert gw._wait_backend == "thread" channel = gw.remote_exec("channel.send(channel.gateway._wait_backend)") assert channel.receive() == "thread" From c0faa503ab5ef2ab86c61dd23cae655126d57e91 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 29 Jul 2026 08:59:33 +0200 Subject: [PATCH 62/91] feat!: rename execmodel= to profile=, deprecate main_thread_only "execmodel" named a *local* execution model that the Trio port deleted: Group(execmodel=), set_execmodel() and group.execmodel have had no behavioural effect for a while, while the spec key of the same name selects something else entirely -- the worker's profile. So the spec key becomes profile=, execmodel= stays as a permanently accepted alias (pytest-xdist passes both the kwarg and the spec key), and the local half is deprecated down to the remote default it actually sets. main_thread_only goes with it, as an accepted alias for thread. The restored hybrid thread profile already hands the first remote_exec the worker's real main thread, which is the GUI/signal property the profile existed for; what it additionally did was refuse a *second* concurrent remote_exec, and that was a deadlock guard built on a 1s timeout that misread a merely slow predecessor under load. Behaviour change worth noting: such a request now runs on a pool thread instead of failing. Fixes a latent livelock this surfaced: execnet.dumps warned on every access, and xdist calls it from serialize_warning_message -- i.e. from inside pytest's warning-recording hook -- so a single DeprecationWarning in a worker recorded a warning that recorded a warning, unbounded, and wedged the run. The shim now warns once per process. Co-Authored-By: Claude Opus 5 (1M context) --- src/execnet/__init__.py | 32 ++++++--- src/execnet/_errors.py | 3 - src/execnet/_execmodel.py | 51 +++++++++++--- src/execnet/_gateway.py | 10 ++- src/execnet/_gateway_base.py | 9 +-- src/execnet/_multi.py | 119 +++++++++++++++++++++++---------- src/execnet/_provision.py | 9 ++- src/execnet/_trio_gateway.py | 14 +++- src/execnet/_trio_host.py | 7 +- src/execnet/_trio_worker.py | 99 ++++++++++----------------- src/execnet/_xspec.py | 20 +++++- src/execnet/gateway_base.py | 1 - src/execnet/sync.py | 2 + testing/conftest.py | 29 ++++---- testing/test_execmodel_trio.py | 28 ++++++-- testing/test_gateway.py | 60 ++++++----------- testing/test_namespaces.py | 13 ++++ 17 files changed, 305 insertions(+), 201 deletions(-) diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 87626d99..315dd0bf 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -42,6 +42,7 @@ from .sync import default_group from .sync import makegateway from .sync import set_execmodel +from .sync import set_profile __all__ = [ "Channel", @@ -61,6 +62,7 @@ "default_group", "makegateway", "set_execmodel", + "set_profile", ] @@ -92,6 +94,14 @@ #: then delete this and its test. Nothing else may be added here. _XDIST_COMPAT = ("dumps",) +#: Warn about the shim at most once per process. xdist reaches it from +#: ``serialize_warning_message``, i.e. from inside pytest's +#: warning-recording hook: warning every time makes recording one warning +#: record another, which records another -- an unbounded loop that wedges +#: the worker. Once is also simply the right amount of noise for a name +#: whose caller is a library, not the user. +_xdist_compat_warned: set[str] = set() + def __getattr__(name: str) -> Any: if name in _LAZY_MODULES: @@ -100,14 +110,18 @@ def __getattr__(name: str) -> Any: return importlib.import_module(f".{name}", __name__) if name in _XDIST_COMPAT: import importlib - import warnings - - warnings.warn( - f"execnet.{name} is a temporary pytest-xdist compatibility shim and" - " will be removed; the standalone serializer is not public. Use" - " execnet.can_send(obj) to test whether a value can cross a channel.", - DeprecationWarning, - stacklevel=2, - ) + + if name not in _xdist_compat_warned: + import warnings + + _xdist_compat_warned.add(name) + warnings.warn( + f"execnet.{name} is a temporary pytest-xdist compatibility shim" + " and will be removed; the standalone serializer is not public." + " Use execnet.can_send(obj) to test whether a value can cross a" + " channel.", + DeprecationWarning, + stacklevel=2, + ) return getattr(importlib.import_module("._serialize", __name__), name) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/execnet/_errors.py b/src/execnet/_errors.py index e90ee2d0..276fbb5e 100644 --- a/src/execnet/_errors.py +++ b/src/execnet/_errors.py @@ -17,9 +17,6 @@ sysex = (KeyboardInterrupt, SystemExit) INTERRUPT_TEXT = "keyboard-interrupted" -MAIN_THREAD_ONLY_DEADLOCK_TEXT = ( - "concurrent remote_exec would cause deadlock for main_thread_only execmodel" -) class GatewayReceivedTerminate(Exception): diff --git a/src/execnet/_execmodel.py b/src/execnet/_execmodel.py index 9cb1d75c..adb43950 100644 --- a/src/execnet/_execmodel.py +++ b/src/execnet/_execmodel.py @@ -1,15 +1,19 @@ -"""Deprecated execution-model presets. +"""Worker profiles, and the deprecated ExecModel shim. -The machinery behind execution models was retired during the Trio port; what -survives is the preset *name*, which maps onto the worker config axes -(``loop=`` / ``exec=`` / ``wait=``). Kept because pytest-xdist's remote worker -still builds its test queue on ``channel.gateway.execmodel.RLock()``/``Event()``. +The machinery behind "execution models" was retired during the Trio port. +What the name actually selected -- where exec'd code runs relative to the +worker's protocol loop -- survives as the ``profile=`` spec key and +:data:`WORKER_PROFILES`. + +:class:`ExecModel` itself is kept only because pytest-xdist's remote worker +builds its test queue on ``channel.gateway.execmodel.RLock()``/``Event()``. """ from __future__ import annotations import os import threading +import warnings class ExecModel: @@ -76,17 +80,46 @@ def Event(self) -> threading.Event: #: worker profiles: where exec'd code runs relative to the protocol loop -EXECMODEL_PROFILES = ( +WORKER_PROFILES = ( "thread", # hybrid: primary on the main thread, overflow on pool threads - "main_thread_only", # exec serialized on the main thread (GUI/pytest) "trio", # pure async: loop owns the main thread, async sources as tasks "gevent", # greenlets on a main-thread hub, one per remote_exec ) +#: profiles kept as accepted spellings, mapped to what they now select. +#: ``main_thread_only`` predates the restored hybrid ``thread`` profile, +#: which already hands the first remote_exec the real main thread -- the +#: GUI/signal property it existed for. What it additionally did was refuse +#: a *second* concurrent remote_exec instead of overflowing to a pool +#: thread; that guard is gone. +DEPRECATED_PROFILES = {"main_thread_only": "thread"} + + +def resolve_profile(name: str) -> str: + """Validate a ``profile=`` value, mapping deprecated spellings.""" + replacement = DEPRECATED_PROFILES.get(name) + if replacement is not None: + warnings.warn( + f"the {name!r} worker profile is deprecated and now behaves like" + f" {replacement!r}, which already runs the first remote_exec on" + " the worker's main thread. A second concurrent remote_exec no" + " longer fails -- it runs on a pool thread.", + DeprecationWarning, + stacklevel=3, + ) + return replacement + if name not in WORKER_PROFILES: + raise ValueError( + f"unknown profile {name!r} (known: {list(WORKER_PROFILES)})" + ) + return name + + def get_execmodel(backend: str | ExecModel) -> ExecModel: + """Deprecated: build the xdist-facing shim for a profile name.""" if isinstance(backend, ExecModel): return backend - if backend in EXECMODEL_PROFILES: + if backend in WORKER_PROFILES or backend in DEPRECATED_PROFILES: return ExecModel(backend) - raise ValueError(f"unknown execmodel {backend!r}") + raise ValueError(f"unknown profile {backend!r}") diff --git a/src/execnet/_gateway.py b/src/execnet/_gateway.py index b8cfe863..bbbb936c 100644 --- a/src/execnet/_gateway.py +++ b/src/execnet/_gateway.py @@ -6,6 +6,7 @@ from __future__ import annotations import types +import warnings from collections.abc import Callable from typing import TYPE_CHECKING from typing import Any @@ -53,7 +54,7 @@ def __repr__(self) -> str: except AttributeError: r = "uninitialized" i = "no" - return f"<{self.__class__.__name__} id={self.id!r} {r}, {self.execmodel.backend} model, {i} active channels>" + return f"<{self.__class__.__name__} id={self.id!r} {r}, {self.spec.profile} profile, {i} active channels>" def exit(self) -> None: """Trigger gateway exit. @@ -137,7 +138,12 @@ def remote_exec( def remote_init_threads(self, num: int | None = None) -> None: """DEPRECATED. Is currently a NO-OPERATION already.""" - print("WARNING: remote_init_threads() is a no-operation in execnet-1.2") + warnings.warn( + "remote_init_threads() has been a no-operation since execnet 1.2" + " and will be removed; drop the call.", + DeprecationWarning, + stacklevel=2, + ) class RInfo: diff --git a/src/execnet/_gateway_base.py b/src/execnet/_gateway_base.py index 55e8bb09..4d0f762e 100644 --- a/src/execnet/_gateway_base.py +++ b/src/execnet/_gateway_base.py @@ -17,7 +17,6 @@ import os import sys -import threading from _thread import interrupt_main from collections.abc import Callable from contextlib import suppress @@ -69,7 +68,7 @@ def _attach_trio_session(self, session: Any) -> None: session.bind_sync_channel(channel) def _new_wakener(self) -> Wakener: - """A fresh wakener for one blocking-wait carrier (wait= axis).""" + """A fresh wakener for one blocking-wait carrier.""" return make_wakener(self._wait_backend) def _bind_channel(self, channel: Channel) -> None: @@ -171,7 +170,6 @@ class WorkerGateway(BaseGateway): _trio_exec: Any = None # The exec pool (a TrioWorkerExec duck-typed as WorkerPool for STATUS). _execpool: Any = None - _executetask_complete: threading.Event | None = None def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: trio_exec = self._trio_exec @@ -232,8 +230,3 @@ def executetask( channel.close(errortext) return channel.close() - if self._executetask_complete is not None: - # Indicate that this task has finished executing, meaning - # that there is no possibility of it triggering a deadlock - # for the next spawn call. - self._executetask_complete.set() diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py index 113ff1c2..ee751597 100644 --- a/src/execnet/_multi.py +++ b/src/execnet/_multi.py @@ -11,6 +11,7 @@ import threading import time import types +import warnings from collections.abc import Callable from collections.abc import Iterable from collections.abc import Iterator @@ -25,9 +26,9 @@ from ._boundary import WaitBackend from ._channel import Channel -from ._execmodel import EXECMODEL_PROFILES from ._execmodel import ExecModel from ._execmodel import get_execmodel +from ._execmodel import resolve_profile from ._trace import trace from ._xspec import XSpec @@ -50,24 +51,36 @@ class Group: _wait_backend: WaitBackend = "thread" def __init__( - self, xspecs: Iterable[XSpec | str | None] = (), execmodel: str = "thread" + self, + xspecs: Iterable[XSpec | str | None] = (), + profile: str | None = None, + *, + execmodel: str | None = None, ) -> None: """Initialize a group and make gateways as specified. - execmodel can be one of the supported execution models. + ``profile`` is the default worker profile for gateways created + without an explicit ``profile=`` in their spec. ``execmodel`` is + the deprecated spelling (pytest-xdist still passes it). """ + if execmodel is not None: + if profile is not None: + raise TypeError("pass either profile= or execmodel=, not both") + warnings.warn( + "Group(execmodel=...) is deprecated; use Group(profile=...)." + " execnet has no local execution model any more -- the value" + " only selects the worker profile.", + DeprecationWarning, + stacklevel=2, + ) + profile = execmodel self._gateways: list[Gateway] = [] self._autoidcounter = 0 self._autoidlock = Lock() self._gateways_to_join: list[Gateway] = [] self._trio_host: Any = None self._async_group: Any = None - # we use the same execmodel for all of the Gateway objects - # we spawn on our side. Probably we should not allow different - # execmodels between different groups but not clear. - # Note that "other side" execmodels may differ and is typically - # specified by the spec passed to makegateway. - self.set_execmodel(execmodel) + self.set_profile("thread" if profile is None else profile) for xspec in xspecs: self.makegateway(xspec) atexit.register(self._cleanup_atexit) @@ -94,33 +107,63 @@ async def _start() -> Any: self._async_group = host.call(_start) return self._async_group + @property + def profile(self) -> str: + """Default worker profile for gateways created by this group.""" + return self._profile + + def set_profile(self, profile: str) -> None: + """Set the default worker profile for newly created gateways. + + NOTE: only settable before any gateway is created. + """ + if self._gateways: + raise ValueError( + "can not set the profile if gateways have been created already" + ) + self._profile = resolve_profile(profile) + @property def execmodel(self) -> ExecModel: - return self._execmodel + """Deprecated: there is no local execution model any more.""" + warnings.warn( + "Group.execmodel is deprecated: execnet has no local execution" + " model. Use Group.profile for the worker profile.", + DeprecationWarning, + stacklevel=2, + ) + return get_execmodel(self._profile) @property def remote_execmodel(self) -> ExecModel: - return self._remote_execmodel + """Deprecated alias for :attr:`profile`, as an ExecModel shim.""" + warnings.warn( + "Group.remote_execmodel is deprecated; use Group.profile.", + DeprecationWarning, + stacklevel=2, + ) + return get_execmodel(self._profile) def set_execmodel( self, execmodel: str, remote_execmodel: str | None = None ) -> None: - """Set the execution model for local and remote site. + """Deprecated alias for :meth:`set_profile`. - execmodel can be one of the supported execution models. - It determines the execution model for any newly created gateway. - If remote_execmodel is not specified it takes on the value of execmodel. - - NOTE: Execution models can only be set before any gateway is created. + The *local* execution model it used to set no longer exists -- all + protocol IO runs on the Trio host -- so only the worker profile is + taken from these arguments (``remote_execmodel`` when given, else + ``execmodel``). """ - if self._gateways: - raise ValueError( - "can not set execution models if gateways have been created already" - ) - if remote_execmodel is None: - remote_execmodel = execmodel - self._execmodel = get_execmodel(execmodel) - self._remote_execmodel = get_execmodel(remote_execmodel) + warnings.warn( + "Group.set_execmodel is deprecated; use Group.set_profile(profile)." + " execnet has no local execution model, so only the remote value" + " has an effect.", + DeprecationWarning, + stacklevel=2, + ) + self.set_profile( + execmodel if remote_execmodel is None else remote_execmodel + ) def __repr__(self) -> str: idgateways = [gw.id for gw in self] @@ -161,11 +204,14 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: id= specifies the gateway id python= specifies which python interpreter to execute - execmodel=name worker profile: where exec'd code runs relative - to the worker's protocol loop. 'thread' (pool - threads) or 'main_thread_only' (serialized on - the worker main thread, GUI/signal-safe). - wait=backend wakener for blocking waits ('thread' default) + profile=name worker profile: where exec'd code runs relative + to the worker's protocol loop. 'thread' + (default; the first remote_exec claims the + worker main thread, further ones overflow to + pool threads), 'trio' (async sources as tasks, + single-threaded) or 'gevent' (a greenlet per + remote_exec). Spelled 'execmodel=' before + execnet 3.0; that spelling still works. chdir= specifies to which directory to change nice= specifies process priority of new process env:NAME=value specifies a remote environment variable setting. @@ -177,13 +223,10 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: if not isinstance(spec, XSpec): spec = XSpec(spec) self.allocate_id(spec) - if spec.execmodel is None: - spec.execmodel = self.remote_execmodel.backend - elif spec.execmodel not in EXECMODEL_PROFILES: - raise ValueError( - f"unknown execmodel {spec.execmodel!r}" - f" (known profiles: {list(EXECMODEL_PROFILES)})" - ) + if spec.profile is None: + spec.profile = self._profile + else: + spec.profile = resolve_profile(spec.profile) from . import _trio_host if not (spec.socket or spec.via or spec.ssh or spec.vagrant_ssh or spec.popen): @@ -414,4 +457,6 @@ def run_term() -> None: default_group = Group() makegateway = default_group.makegateway +set_profile = default_group.set_profile +#: deprecated alias, see Group.set_execmodel set_execmodel = default_group.set_execmodel diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index dcfaf1a4..67b02d17 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -166,10 +166,13 @@ def worker_cli_arg(spec: Any) -> str: config: dict[str, Any] = { "id": f"{spec.id}-worker", - "execmodel": spec.execmodel, + "profile": spec.profile, + # pre-3.0 spelling, same value: a worker from an older execnet + # reads this one. Drop with the XSpec alias. + "execmodel": spec.profile, # derived, not configurable: the gevent profile parks its greenlets # on gevent wakeners, every other profile is thread-shaped. - "wait": "gevent" if spec.execmodel == "gevent" else "thread", + "wait": "gevent" if spec.profile == "gevent" else "thread", "coordinator_version": execnet.__version__, } # Startup setup applied by the worker before serving (never through @@ -213,7 +216,7 @@ def _extra_with_tokens(config: str) -> list[str]: backend needs gevent importable in the worker. """ parsed = json.loads(config) - if parsed.get("execmodel") == "gevent" or parsed.get("wait") == "gevent": + if parsed.get("profile") == "gevent" or parsed.get("wait") == "gevent": return ["--with", "gevent"] return [] diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 90246fb7..db60920e 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -42,6 +42,7 @@ from ._errors import HostNotFound from ._errors import RemoteError from ._errors import TimeoutError +from ._execmodel import resolve_profile from ._exec_source import normalize_exec_source from ._message import FrameDecoder from ._message import Message @@ -667,6 +668,8 @@ def _dispatch(self, message: Message) -> None: status = { "numchannels": len(self._channels), "numexecuting": task_exec.active_count() if task_exec else 0, + "profile": "trio", + # legacy key, same value -- pytest-xdist reads it "execmodel": "trio", } self._send_nowait(Message.CHANNEL_DATA, channelid, dumps_internal(status)) @@ -905,9 +908,14 @@ async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: raise RuntimeError(f"{self!r} is not entered") if not isinstance(spec, XSpec): spec = XSpec(spec) - if spec.execmodel is None: - # the sync Group normally stamps this before spawning - spec.execmodel = "thread" + if spec.profile is None: + # An async coordinator does not imply an async worker: the + # worker's shape is its own choice, and the default stays the + # thread profile. Pass ``profile=trio`` for a worker that runs + # exec'd async sources as tasks. + spec.profile = "thread" + else: + spec.profile = resolve_profile(spec.profile) if spec.id is None: spec.id = "gw%d" % len(self._gateways) process: trio.Process | None = None diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 8e1d695b..82ad9c37 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -31,6 +31,7 @@ from ._errors import GatewayReceivedTerminate from ._errors import RemoteError from ._execmodel import ExecModel +from ._execmodel import get_execmodel from ._message import FrameDecoder from ._message import Message from ._message import gateway_info @@ -224,6 +225,8 @@ def _answer_status(self, message: Message) -> None: d = { "numchannels": len(gateway._channelfactory._channels), "numexecuting": execpool.active_count() if execpool is not None else 0, + "profile": gateway.execmodel.backend, + # legacy key, same value -- pytest-xdist reads it "execmodel": gateway.execmodel.backend, } gateway._send(Message.CHANNEL_DATA, message.channelid, dumps_internal(d)) @@ -718,7 +721,7 @@ def __init__(self, group: Group, host: TrioHost) -> None: def _make_gateway(self, stream: ByteStream, spec: Any) -> AsyncGateway: import execnet - sync_gw = execnet.Gateway(_TempIO(self.group.execmodel), spec) + sync_gw = execnet.Gateway(_TempIO(get_execmodel(spec.profile)), spec) # the caller's concurrency library, inherited from the facade sync_gw._wait_backend = self.group._wait_backend return SyncBridgeGateway( @@ -777,7 +780,7 @@ def makegateway_trio(group: Group, spec: Any) -> Gateway: assert isinstance(bridge, SyncBridgeGateway) gw: Gateway = bridge.sync_gateway # type: ignore[assignment] gw._io = SyncIOHandle( - group.execmodel, + get_execmodel(spec.profile), bridge, process=async_group._processes.get(bridge), remoteaddress=bridge.remoteaddress, diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 741c5eff..0146c00b 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -2,7 +2,6 @@ from __future__ import annotations -import functools import os import sys import threading @@ -12,7 +11,6 @@ import trio -from ._errors import MAIN_THREAD_ONLY_DEADLOCK_TEXT from ._errors import geterrortext from ._execmodel import get_execmodel from ._gateway_base import WorkerGateway @@ -59,25 +57,15 @@ def trigger_shutdown(self) -> None: pass -#: How long admission waits for the previous main-thread exec to finish -#: before declaring a deadlock. Kept small: a genuine second concurrent -#: remote_exec never returns, so this only affects how fast that is -#: reported. (Under heavy CPU contention a merely slow predecessor can be -#: misread as a deadlock, but main_thread_only is an xdist-only transitional -#: profile slated for removal, so the whole guard goes away with it.) -MAIN_THREAD_ONLY_ADMIT_TIMEOUT = 1.0 +class PrimaryThreadPump: + """Runs exec requests handed to it on the process main thread. - -class MainExec: - """Exec strategy: serialize each request onto the process main thread. - - The ``main_thread_only`` profile (GUI/signal-safe: pytest under xdist - runs this way). Admission waits for the previous request to finish and - closes the channel with the deadlock text when it cannot within - ``MAIN_THREAD_ONLY_ADMIT_TIMEOUT`` (a second concurrent remote_exec would - deadlock the requester). + The building block for every strategy that needs a real main thread: + :meth:`integrate_as_primary_thread` parks there draining a mailbox, and + :meth:`run` hands one request over and awaits its completion. """ + #: whether _run_worker must hand this strategy the process main thread needs_primary_thread = True def __init__(self, gateway: WorkerGateway) -> None: @@ -87,15 +75,7 @@ def __init__(self, gateway: WorkerGateway) -> None: ) async def admit(self, channel: Channel, item: ExecItem) -> bool: - complete = self.gateway._executetask_complete - assert complete is not None - wait_slot = functools.partial( - complete.wait, timeout=MAIN_THREAD_ONLY_ADMIT_TIMEOUT - ) - if not await trio.to_thread.run_sync(wait_slot, abandon_on_cancel=True): - channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) - return False - complete.clear() + """FIFO admission gate; always open.""" return True async def run(self, channel: Channel, item: ExecItem) -> None: @@ -119,14 +99,19 @@ def trigger_shutdown(self) -> None: self._primary.put(None) -class HybridExec(MainExec): +class HybridExec(PrimaryThreadPump): """Exec strategy: primary on the main thread, overflow on pool threads. - The classic ``thread`` execmodel shape: a request arriving while the + The classic ``thread`` profile shape: a request arriving while the main thread is idle claims it (pytest and friends get a true main thread); requests arriving while it is busy run on worker threads instead of queueing. The claim is decided during FIFO admission so the *first* request always gets the main thread. + + This is also what the retired ``main_thread_only`` profile now maps to: + it existed for the main-thread guarantee, which the claim provides, + and its extra behaviour -- refusing a second concurrent remote_exec + rather than overflowing -- was a deadlock guard, not a feature. """ def __init__(self, gateway: WorkerGateway) -> None: @@ -160,8 +145,8 @@ async def run(self, channel: Channel, item: ExecItem) -> None: class GreenletExec: """Exec strategy: greenlets on a gevent hub owning the main thread. - ``execmodel=gevent``: each request runs as a greenlet spawned by the - integrate loop; the worker's ``wait=gevent`` wakeners make channel + ``profile=gevent``: each request runs as a greenlet spawned by the + integrate loop; the worker's gevent wakeners make channel operations park the greenlet, so concurrent remote_execs cooperate on the one main thread. Requires gevent in the worker environment (provisioning adds the ``gevent`` requirement automatically). @@ -207,18 +192,17 @@ def trigger_shutdown(self) -> None: self._primary.put(None) -# execmodel profile -> exec strategy for the sync-facade worker; the -# "trio" profile serves a plain AsyncGateway instead (TaskExec below). +# worker profile -> exec strategy for the sync-facade worker; the "trio" +# profile serves a plain AsyncGateway instead (TaskExec below). # Future placement strategies slot in here (e.g. subinterpreters). WORKER_EXEC_STRATEGIES: dict[str, Callable[[WorkerGateway], Any]] = { "thread": HybridExec, - "main_thread_only": MainExec, "gevent": GreenletExec, } class TaskExec: - """Exec strategy for the pure-async profile (``execmodel=trio``). + """Exec strategy for the pure-async profile (``profile=trio``). Sources run as tasks on the worker's own loop, in the one and only thread of the process, and receive an ``AsyncChannel``. Sources must @@ -261,7 +245,7 @@ async def _run_exec(self, channel: Any, item: ExecItem) -> None: toplevel_await = bool(co.co_flags & inspect.CO_COROUTINE) if not toplevel_await and not call_name: await channel.aclose( - "sync source under execmodel=trio: the source must use" + "sync source under profile=trio: the source must use" " top-level await (or pass an async function)" ) return @@ -274,7 +258,7 @@ async def _run_exec(self, channel: Any, item: ExecItem) -> None: result = function(channel, **kwargs) if not hasattr(result, "__await__"): await channel.aclose( - f"sync function {call_name!r} under execmodel=trio:" + f"sync function {call_name!r} under profile=trio:" " remote_exec functions must be async" ) return @@ -299,7 +283,7 @@ class TrioWorkerExec: Exec requests flow through a single pump task so admission happens strictly in message-arrival order (trio task scheduling order is deliberately unordered, so per-request tasks would race for e.g. the - main_thread_only slot). Where an admitted request runs is the + main-thread claim). Where an admitted request runs is the strategy's business (:data:`WORKER_EXEC_STRATEGIES`). """ @@ -454,7 +438,7 @@ def _build_worker_gateway( strategy_factory = WORKER_EXEC_STRATEGIES[model.backend] except KeyError: raise ValueError( - f"execmodel {model.backend!r} has no worker exec strategy " + f"profile {model.backend!r} has no worker exec strategy " f"(known: {sorted(WORKER_EXEC_STRATEGIES)})" ) from None strategy = strategy_factory(gateway) @@ -462,10 +446,6 @@ def _build_worker_gateway( # Duck-type as the exec pool for STATUS / _terminate_execution. gateway._execpool = trio_exec gateway._trio_exec = trio_exec - gateway._executetask_complete = None - if strategy.needs_primary_thread: - gateway._executetask_complete = threading.Event() - gateway._executetask_complete.set() return gateway, trio_exec @@ -504,7 +484,7 @@ async def _make_fd_io(read_fd: int, write_fd: int) -> Any: async def _serve_async_worker(stream: Any, id: str) -> None: - """Serve a plain AsyncGateway with task-based exec (execmodel=trio). + """Serve a plain AsyncGateway with task-based exec (profile=trio). The whole worker is this one trio run on the process main thread: the dispatch loop and every exec'd source share it. Termination cancels @@ -523,7 +503,7 @@ async def _serve_async_worker(stream: Any, id: str) -> None: def serve_popen_async(id: str) -> None: - """Serve the pure-async worker over the stdio pipes (execmodel=trio).""" + """Serve the pure-async worker over the stdio pipes (profile=trio).""" from ._trio_gateway import staple_fd_stream read_fd, write_fd = _prepare_protocol_fds() @@ -548,11 +528,11 @@ async def main() -> None: os._exit(0) -def serve_popen_trio(id: str, execmodel: str = "thread", wait: str = "thread") -> None: +def serve_popen_trio(id: str, profile: str = "thread", wait: str = "thread") -> None: """Serve a WorkerGateway over the stdio pipes (popen / ssh worker).""" from . import _trio_host - model = get_execmodel(execmodel) + model = get_execmodel(profile) read_fd, write_fd = _prepare_protocol_fds() # Bootstrap handshake: the coordinator waits for this byte on our stdout # before starting the Message protocol. We are launched as a plain module @@ -566,7 +546,7 @@ def serve_popen_trio(id: str, execmodel: str = "thread", wait: str = "thread") - def serve_socket_trio( - id: str, execmodel: str, socket_fd: int, wait: str = "thread" + id: str, profile: str, socket_fd: int, wait: str = "thread" ) -> None: """Serve a WorkerGateway over an inherited socket fd. @@ -576,7 +556,7 @@ def serve_socket_trio( """ from . import _trio_host - model = get_execmodel(execmodel) + model = get_execmodel(profile) host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") host.start() io = host.call(_trio_host.adopt_socket, socket_fd) @@ -640,7 +620,7 @@ def _main() -> None: """Entry point for ``python -m execnet._trio_worker [--socket-fd N]``. ```` is the coordinator's ``_provision.worker_cli_arg`` payload: - ``{"id", "execmodel", "coordinator_version"}``. With ``--socket-fd`` the + ``{"id", "profile", "wait", "coordinator_version"}``. With ``--socket-fd`` the worker serves over that inherited socket (socketserver); otherwise over the stdio pipes (popen / ssh). The worker imports execnet + trio from the environment; no source is sent over the wire to bootstrap it. @@ -658,25 +638,20 @@ def _main() -> None: config = json.loads(ns.config) _check_version(config["coordinator_version"]) _apply_worker_setup(config) - if config["execmodel"] == "trio": + # "execmodel" is the pre-3.0 spelling; accept both so a version-skewed + # coordinator still connects. + profile = config.get("profile") or config["execmodel"] + wait = config.get("wait", "thread") + if profile == "trio": # pure-async profile: one thread, the loop owns the main thread if ns.socket_fd is not None: serve_socket_async(config["id"], ns.socket_fd) else: serve_popen_async(config["id"]) elif ns.socket_fd is not None: - serve_socket_trio( - config["id"], - config["execmodel"], - ns.socket_fd, - wait=config.get("wait", "thread"), - ) + serve_socket_trio(config["id"], profile, ns.socket_fd, wait=wait) else: - serve_popen_trio( - id=config["id"], - execmodel=config["execmodel"], - wait=config.get("wait", "thread"), - ) + serve_popen_trio(id=config["id"], profile=profile, wait=wait) if __name__ == "__main__": diff --git a/src/execnet/_xspec.py b/src/execnet/_xspec.py index 0559ed8c..f13d0521 100644 --- a/src/execnet/_xspec.py +++ b/src/execnet/_xspec.py @@ -15,10 +15,15 @@ class XSpec: * If no "=value" is given, assume a boolean True value """ + #: keys accepted under an older spelling, mapped to the canonical one. + #: ``execmodel`` described a *local* execution model that no longer + #: exists; what the key actually selects is the worker's profile. + _ALIASES = {"execmodel": "profile"} + # XXX allow customization, for only allow specific key names chdir: str | None = None dont_write_bytecode: bool | None = None - execmodel: str | None = None + profile: str | None = None id: str | None = None installvia: str | None = None nice: str | None = None @@ -42,7 +47,9 @@ def __init__(self, string: str) -> None: key, value = keyvalue[:i], keyvalue[i + 1 :] if key[0] == "_": raise AttributeError("%r not a valid XSpec key" % key) - if key in self.__dict__: + # duplicates are checked on the canonical name, so + # ``execmodel=x//profile=y`` is rejected like any other clash + if self._ALIASES.get(key, key) in self.__dict__: raise ValueError(f"duplicate key: {key!r} in {string!r}") if key.startswith("env:"): self.env[key[4:]] = value @@ -54,6 +61,15 @@ def __getattr__(self, name: str) -> None | bool | str: raise AttributeError(name) return None + @property + def execmodel(self) -> str | None: + """Deprecated alias for :attr:`profile` (accepted indefinitely).""" + return self.profile + + @execmodel.setter + def execmodel(self, value: str | None) -> None: + self.profile = value + def __repr__(self) -> str: return f"" diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index ba1eb35f..a82a1c0f 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -24,7 +24,6 @@ # errors and error texts "sysex": "._errors", "INTERRUPT_TEXT": "._errors", - "MAIN_THREAD_ONLY_DEADLOCK_TEXT": "._errors", "GatewayReceivedTerminate": "._errors", "HostNotFound": "._errors", "geterrortext": "._errors", diff --git a/src/execnet/sync.py b/src/execnet/sync.py index 37259bb7..1160b89c 100644 --- a/src/execnet/sync.py +++ b/src/execnet/sync.py @@ -19,6 +19,7 @@ from ._multi import default_group from ._multi import makegateway from ._multi import set_execmodel +from ._multi import set_profile from ._rsync import RSync from ._xspec import XSpec @@ -38,4 +39,5 @@ "default_group", "makegateway", "set_execmodel", + "set_profile", ] diff --git a/testing/conftest.py b/testing/conftest.py index de1e2866..a9797ada 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -179,46 +179,49 @@ def group() -> Iterator[execnet.Group]: @pytest.fixture def gw( request: pytest.FixtureRequest, - execmodel: ExecModel, + profile: str, group: execnet.Group, ) -> Gateway: try: return group[request.param] except KeyError: if request.param == "popen": - gw = group.makegateway("popen//id=popen//execmodel=%s" % execmodel.backend) + gw = group.makegateway("popen//id=popen//profile=%s" % profile) elif request.param == "socket": - # if execmodel.backend != "thread": - # pytest.xfail( - # "cannot set remote non-thread execmodel for sockets") pname = "sproxy1" if pname not in group: proxygw = group.makegateway("popen//id=%s" % pname) # assert group['proxygw'].remote_status().receiving gw = group.makegateway( - f"socket//id=socket//installvia={pname}//execmodel={execmodel.backend}" + f"socket//id=socket//installvia={pname}//profile={profile}" ) # TODO(typing): Clarify this assignment. gw.proxygw = proxygw # type: ignore[attr-defined] assert pname in group elif request.param == "ssh": sshhost = request.getfixturevalue("specssh").ssh - # we don't use execmodel.backend here - # but you can set it when specifying the ssh spec + # the profile is not forced here; set it in the ssh spec instead gw = group.makegateway(f"ssh={sshhost}//id=ssh") elif request.param == "proxy": group.makegateway("popen//id=proxy-transport") gw = group.makegateway( - "popen//via=proxy-transport//id=proxy//execmodel=%s" % execmodel.backend + "popen//via=proxy-transport//id=proxy//profile=%s" % profile ) else: - assert 0, f"unknown execmodel: {request.param}" + assert 0, f"unknown gateway type: {request.param}" return gw -@pytest.fixture(params=["thread", "main_thread_only"], scope="session") -def execmodel(request: pytest.FixtureRequest) -> ExecModel: - return get_execmodel(request.param) +@pytest.fixture(params=["thread"], scope="session") +def profile(request: pytest.FixtureRequest) -> str: + """The worker profile gateways in this test run are created with.""" + return request.param + + +@pytest.fixture(scope="session") +def execmodel(profile: str) -> ExecModel: + """The deprecated ExecModel shim for ``profile`` (pytest-xdist compat).""" + return get_execmodel(profile) @pytest.fixture diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py index 4e568d7e..2ef456ef 100644 --- a/testing/test_execmodel_trio.py +++ b/testing/test_execmodel_trio.py @@ -1,4 +1,4 @@ -"""The pure-async worker profile (execmodel=trio). +"""The pure-async worker profile (profile=trio). One single thread in the worker: the trio loop owns the main thread and exec'd sources run as tasks on it, talking through AsyncChannels. Sync @@ -21,7 +21,7 @@ @pytest.fixture def trio_gw(makegateway: Callable[[str], Gateway]) -> Gateway: - return makegateway("popen//execmodel=trio") + return makegateway("popen//profile=trio") class TestSyncCoordinator: @@ -95,21 +95,37 @@ async def source(channel) -> None: channel.receive(TESTTIMEOUT) def test_status_and_rinfo(self, trio_gw: Gateway) -> None: - assert trio_gw.remote_status().execmodel == "trio" + status = trio_gw.remote_status() + assert status.profile == "trio" + # legacy STATUS key, kept for pytest-xdist + assert status.execmodel == "trio" rinfo = trio_gw._rinfo() assert rinfo.pid assert rinfo.version_info -def test_unknown_execmodel_rejected(makegateway: Callable[[str], Gateway]) -> None: - with pytest.raises(ValueError, match="unknown execmodel"): +def test_unknown_profile_rejected(makegateway: Callable[[str], Gateway]) -> None: + with pytest.raises(ValueError, match="unknown profile"): + makegateway("popen//profile=nope") + # the pre-3.0 spelling routes to the same validation + with pytest.raises(ValueError, match="unknown profile"): makegateway("popen//execmodel=nope") +def test_execmodel_is_an_accepted_alias_for_profile( + makegateway: Callable[[str], Gateway], +) -> None: + gw = makegateway("popen//execmodel=trio") + assert gw.spec.profile == "trio" + assert gw.spec.execmodel == "trio" + with pytest.raises(ValueError, match="duplicate key"): + execnet.XSpec("popen//execmodel=trio//profile=thread") + + def test_trio_native_coordinator() -> None: async def main() -> None: async with execnet.trio.AsyncGroup() as group: - gateway = await group.makegateway("popen//execmodel=trio") + gateway = await group.makegateway("popen//profile=trio") channel = await gateway.remote_exec( "await channel.send(await channel.receive() * 2)" ) diff --git a/testing/test_gateway.py b/testing/test_gateway.py index ef8e8aa9..e28a98eb 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -582,22 +582,15 @@ def test_popen_args(spec: str, expected_args: list[str]) -> None: assert args[len(expected_args) :][:3] == ["-u", "-m", "execnet._trio_worker"] -def test_assert_main_thread_only( - execmodel: _execmodel.ExecModel, makegateway: Callable[[str], Gateway] +def test_sequential_remote_exec_claims_the_main_thread( + makegateway: Callable[[str], Gateway], ) -> None: - if execmodel.backend != "main_thread_only": - pytest.skip("can only run with main_thread_only") - - gw = makegateway(f"execmodel={execmodel.backend}//popen") - + # The `thread` profile hands each request the worker main thread while + # it is idle -- the GUI/signal-safety property `main_thread_only` + # existed for. Sequential remote_execs therefore all land there. + gw = makegateway("profile=thread//popen") try: - # Submit multiple remote_exec requests in quick succession and - # assert that all tasks execute in the main thread. It is - # necessary to call receive on each channel before the next - # remote_exec call, since the channel will raise an error if - # concurrent remote_exec requests are submitted as in - # test_main_thread_only_concurrent_remote_exec_deadlock. - for i in range(10): + for _ in range(10): ch = gw.remote_exec( """ import time, threading @@ -605,7 +598,6 @@ def test_assert_main_thread_only( channel.send(threading.current_thread() is threading.main_thread()) """ ) - try: res = ch.receive() finally: @@ -621,43 +613,29 @@ def test_assert_main_thread_only( gw.join() -def test_main_thread_only_concurrent_remote_exec_deadlock( - execmodel: _execmodel.ExecModel, makegateway: Callable[[str], Gateway] +def test_main_thread_only_is_deprecated_and_overflows( + makegateway: Callable[[str], Gateway], ) -> None: - if execmodel.backend != "main_thread_only": - pytest.skip("can only run with main_thread_only") - - gw = makegateway(f"execmodel={execmodel.backend}//popen") + # `main_thread_only` now maps to `thread`. Where it used to refuse a + # second concurrent remote_exec with a deadlock error, the request now + # overflows to a pool thread -- the first one still gets the main + # thread, which is what the profile was for. + with pytest.warns(DeprecationWarning, match="main_thread_only"): + gw = makegateway("profile=main_thread_only//popen") channels = [] try: - # Submit multiple remote_exec requests in quick succession and - # assert that MAIN_THREAD_ONLY_DEADLOCK_TEXT is raised if - # concurrent remote_exec requests are submitted for the - # main_thread_only execmodel (as compensation for the lack of - # back pressure in remote_exec calls which do not attempt to - # block until the remote main thread is idle). - for i in range(2): + for _ in range(2): channels.append( gw.remote_exec( """ import threading channel.send(threading.current_thread() is threading.main_thread()) - # Wait forever, ensuring that the deadlock case triggers. - threading.Event().wait() + channel.receive() """ ) ) - - expected_results = ( - True, - execnet._errors.MAIN_THREAD_ONLY_DEADLOCK_TEXT, - ) - for expected, ch in zip(expected_results, channels, strict=True): - try: - res = ch.receive() - except execnet.RemoteError as e: - res = e.formatted - assert res == expected + # both run: first on the main thread, second on an overflow thread + assert [ch.receive(TESTTIMEOUT) for ch in channels] == [True, False] finally: for ch in channels: ch.close() diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index 49f47bb4..dc6f2d9d 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -109,6 +109,7 @@ def test_dumps_is_a_temporary_xdist_shim() -> None: from execnet import _serialize assert execnet._XDIST_COMPAT == ("dumps",) + execnet._xdist_compat_warned.discard("dumps") with pytest.warns(DeprecationWarning, match="temporary pytest-xdist"): assert execnet.dumps is _serialize.dumps # reachable, but never advertised @@ -116,6 +117,18 @@ def test_dumps_is_a_temporary_xdist_shim() -> None: assert "dumps" not in dir(execnet) +def test_dumps_shim_warns_only_once() -> None: + # xdist reaches this from serialize_warning_message, i.e. from inside + # pytest's warning-recording hook: warning every time makes recording + # one warning record another, ad infinitum, and the worker wedges. + execnet._xdist_compat_warned.discard("dumps") + with pytest.warns(DeprecationWarning, match="temporary pytest-xdist"): + execnet.dumps # noqa: B018 + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + execnet.dumps # noqa: B018 + + def test_boundary_kit_is_private() -> None: # There is no third-party event-loop extension point: the two wait # backends are threads and gevent, and every other concurrency library From e75cd0ad92b005e6591f2cc51c6a0731042c3447 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 29 Jul 2026 12:38:53 +0200 Subject: [PATCH 63/91] feat!: share one Trio host, guard loop misuse, fix aio cancellation Three things that only make sense together, because they are all about the boundary between execnet and the caller's concurrency library. execnet.trio is the only surface that runs gateways directly, as tasks in the caller's own nursery. sync/aio/gevent have no loop to put them on, so they drive a Trio host thread -- and every Group used to spawn its own. A host is a thread and an event loop, not something groups need isolated from each other, so there is now one shared host per process, started lazily and stopped at exit, with an explicit execnet.Host for callers who want isolation or deterministic teardown. Driving a blocking surface from inside a running loop used to hang: the wait parks the loop's own thread. Blocking coordinator operations now detect a running asyncio or trio loop and raise, naming the namespace to use instead. Worker-side channels are deliberately exempt -- exec'd code may run its own loop and talk to its channel from inside it. execnet.aio lost data on cancellation: the bridge abandoned the asyncio future while the host task ran on, so `asyncio.timeout` around a receive consumed an item and dropped it -- on the most idiomatic asyncio code path. Each bridged call now owns a trio CancelScope that a cancelled await cancels, while send/send_eof/aclose/terminate are shielded so they cannot tear halfway. Its classes gain the Async prefix to match execnet.trio, AsyncGroup can be driven with start()/aclose() from lifespan hooks, and open_popen_gateway becomes open_gateway on both async surfaces (it always accepted any spec). Co-Authored-By: Claude Opus 5 (1M context) --- src/execnet/__init__.py | 2 + src/execnet/_channel.py | 3 + src/execnet/_gateway.py | 1 + src/execnet/_gateway_base.py | 10 ++ src/execnet/_host.py | 184 ++++++++++++++++++++++++ src/execnet/_multi.py | 39 +++-- src/execnet/_trio_gateway.py | 2 +- src/execnet/_trio_host.py | 12 +- src/execnet/aio.py | 272 ++++++++++++++++++++++++----------- src/execnet/gevent.py | 2 + src/execnet/sync.py | 2 + src/execnet/trio.py | 4 +- testing/test_aio.py | 90 ++++++++++-- testing/test_basics.py | 3 + testing/test_host.py | 201 ++++++++++++++++++++++++++ testing/test_namespaces.py | 2 +- testing/test_trio_gateway.py | 12 +- 17 files changed, 714 insertions(+), 127 deletions(-) create mode 100644 src/execnet/_host.py create mode 100644 testing/test_host.py diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 315dd0bf..2ed1293f 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -32,6 +32,7 @@ from .sync import DumpError from .sync import Gateway from .sync import Group +from .sync import Host from .sync import HostNotFound from .sync import LoadError from .sync import MultiChannel @@ -50,6 +51,7 @@ "DumpError", "Gateway", "Group", + "Host", "HostNotFound", "LoadError", "MultiChannel", diff --git a/src/execnet/_channel.py b/src/execnet/_channel.py index a13f846b..98c5d862 100644 --- a/src/execnet/_channel.py +++ b/src/execnet/_channel.py @@ -286,6 +286,7 @@ def waitclose(self, timeout: float | None = None) -> None: # For a callback channel wait on the consumer task finishing (so every # callback, including the endmarker, has run); otherwise wait for the # non-"opened" state directly. + self.gateway._check_event_loop("channel.waitclose()") signal = ( self._consumer_done if self._consumer_done is not None @@ -309,6 +310,7 @@ def send(self, item: object) -> None: """ if self.isclosed(): raise OSError(f"cannot send to {self!r}") + self.gateway._check_event_loop("channel.send()") self.gateway._send(Message.CHANNEL_DATA, self.id, dumps_internal(item)) def receive(self, timeout: float | None = None) -> Any: @@ -325,6 +327,7 @@ def receive(self, timeout: float | None = None) -> Any: mailbox = self._mailbox if mailbox is None: raise OSError("cannot receive(), channel has receiver callback") + self.gateway._check_event_loop("channel.receive()") x = mailbox.get(timeout) if x is ENDMARKER: mailbox.put(x) # for other receivers diff --git a/src/execnet/_gateway.py b/src/execnet/_gateway.py index bbbb936c..8f992e09 100644 --- a/src/execnet/_gateway.py +++ b/src/execnet/_gateway.py @@ -31,6 +31,7 @@ class Gateway(BaseGateway): """Gateway to a local or remote Python Interpreter.""" _group: Group + _guard_event_loop = True def __init__(self, io: IO, spec: XSpec) -> None: """:private: diff --git a/src/execnet/_gateway_base.py b/src/execnet/_gateway_base.py index 4d0f762e..11920dae 100644 --- a/src/execnet/_gateway_base.py +++ b/src/execnet/_gateway_base.py @@ -45,6 +45,16 @@ class BaseGateway: #: write-acks, join). Inherited from the facade coordinator-side, and #: derived from the worker profile worker-side. _wait_backend: WaitBackend = "thread" + #: whether blocking operations refuse to run inside a foreign event + #: loop. Only coordinator-side: exec'd code in a worker may legitimately + #: run its own loop and talk to its channel from inside it. + _guard_event_loop = False + + def _check_event_loop(self, what: str) -> None: + if self._guard_event_loop: + from ._host import check_not_in_event_loop + + check_not_in_event_loop(what) def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel diff --git a/src/execnet/_host.py b/src/execnet/_host.py new file mode 100644 index 00000000..56af3b4f --- /dev/null +++ b/src/execnet/_host.py @@ -0,0 +1,184 @@ +"""The Trio host thread that the blocking surfaces drive. + +:mod:`execnet.trio` runs gateways *directly*, as tasks in the caller's own +nursery. Every other surface -- :mod:`execnet.sync`, :mod:`execnet.aio`, +:mod:`execnet.gevent` -- has no loop of its own to put them on, so protocol +IO runs on a :class:`Host`: one OS thread running ``trio.run``. + +There is one shared host per process by default, because a host is a +thread and a trio loop, not a resource groups need isolated from each +other. Pass an explicit :class:`Host` when you do want isolation or +deterministic teardown:: + + with execnet.Host() as host: + group = execnet.Group(host=host) + ... + # the thread is joined here, rather than at interpreter exit + +This module stays free of ``import trio`` so ``import execnet`` does not +load the event loop machinery: the real :class:`~execnet._trio_host.TrioHost` +is built on first use. +""" + +from __future__ import annotations + +import atexit +import os +import sys +import threading +from types import TracebackType +from typing import Any + +__all__ = ["Host", "check_not_in_event_loop", "default_host"] + +#: default cap on concurrent threadpool threads running receiver callbacks +DEFAULT_CALLBACK_THREADS = 40 + + +def _running_event_loop() -> str | None: + """``"asyncio"`` / ``"trio"`` when called from inside one, else None. + + Both checks go through ``sys.modules`` first, so a program that never + imported asyncio or trio pays two dict lookups. asyncio is probed with + the private ``_get_running_loop`` because it *returns* None rather than + raising -- this sits in front of every blocking channel operation, and + building an exception per send is not free. + """ + asyncio = sys.modules.get("asyncio") + if asyncio is not None: + get_running = getattr(asyncio.events, "_get_running_loop", None) + if get_running is not None: + if get_running() is not None: + return "asyncio" + else: # pragma: no cover - every supported CPython has the private one + try: + asyncio.get_running_loop() + except RuntimeError: + pass + else: + return "asyncio" + trio = sys.modules.get("trio") + if trio is not None: + try: + trio.lowlevel.current_trio_token() + except RuntimeError: + pass + else: + return "trio" + return None + + +#: which namespace to point a caller at, per detected loop +_SURFACE_FOR_LOOP = { + "asyncio": "execnet.aio (AsyncGroup)", + "trio": "execnet.trio (AsyncGroup)", +} + + +def check_not_in_event_loop(what: str) -> None: + """Raise if ``what`` is about to block a running event loop's thread. + + The blocking surfaces park the calling thread on a wakener, which + inside a running loop stalls every task on it -- usually as a hang, so + it is worth turning into an error that names the right namespace. + """ + loop = _running_event_loop() + if loop is None: + return + raise RuntimeError( + f"{what} blocks the calling thread and you are inside a running" + f" {loop} event loop, which would stall every task on it." + f" Use {_SURFACE_FOR_LOOP[loop]} instead, or run this in a" + " worker thread." + ) + + +class Host: + """A Trio event loop on a dedicated thread, shared by gateway groups. + + Starting is lazy: constructing a Host costs nothing, and the thread + comes up when a group first needs it. + """ + + def __init__( + self, + name: str = "execnet-host", + callback_threads: int = DEFAULT_CALLBACK_THREADS, + ) -> None: + self.name = name + self.callback_threads = callback_threads + self._lock = threading.Lock() + self._trio_host: Any = None + + def __repr__(self) -> str: + state = "running" if self._trio_host is not None else "idle" + return f"" + + @property + def running(self) -> bool: + return self._trio_host is not None + + def _ensure_started(self) -> Any: + """The started :class:`~execnet._trio_host.TrioHost` (internal).""" + with self._lock: + if self._trio_host is None: + from . import _trio_host + + trio_host = _trio_host.TrioHost( + name=self.name, callback_threads=self.callback_threads + ) + trio_host.start() + self._trio_host = trio_host + return self._trio_host + + def close(self, timeout: float | None = 5.0) -> None: + """Stop the loop and join the thread (no-op when not running). + + Gateways served by this host must already be terminated; closing + does not terminate them for you. + """ + with self._lock: + trio_host, self._trio_host = self._trio_host, None + if trio_host is not None: + trio_host.stop(timeout=timeout) + + def __enter__(self) -> Host: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + self.close() + + +_default_lock = threading.Lock() +_default: Host | None = None +#: the pid _default was created in; a forked child inherits a dead thread +_default_pid: int | None = None + + +def default_host() -> Host: + """The process-wide host, started lazily and stopped at interpreter exit. + + After ``os.fork()`` the child inherits a Host whose thread does not + exist there, so the first use in a child builds a fresh one. + """ + global _default, _default_pid + with _default_lock: + pid = os.getpid() + if _default is None or _default_pid != pid: + _default = Host() + _default_pid = pid + return _default + + +def _close_default_atexit() -> None: + host = _default + if host is not None and _default_pid == os.getpid(): + host.close(timeout=1.0) + + +atexit.register(_close_default_atexit) diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py index ee751597..c739eedc 100644 --- a/src/execnet/_multi.py +++ b/src/execnet/_multi.py @@ -29,6 +29,9 @@ from ._execmodel import ExecModel from ._execmodel import get_execmodel from ._execmodel import resolve_profile +from ._host import Host +from ._host import check_not_in_event_loop +from ._host import default_host from ._trace import trace from ._xspec import XSpec @@ -55,13 +58,16 @@ def __init__( xspecs: Iterable[XSpec | str | None] = (), profile: str | None = None, *, + host: Host | None = None, execmodel: str | None = None, ) -> None: """Initialize a group and make gateways as specified. ``profile`` is the default worker profile for gateways created - without an explicit ``profile=`` in their spec. ``execmodel`` is - the deprecated spelling (pytest-xdist still passes it). + without an explicit ``profile=`` in their spec. ``host`` is the + Trio host thread to serve this group's protocol IO on; it defaults + to the process-wide one. ``execmodel`` is the deprecated spelling + of ``profile`` (pytest-xdist still passes it). """ if execmodel is not None: if profile is not None: @@ -78,20 +84,20 @@ def __init__( self._autoidcounter = 0 self._autoidlock = Lock() self._gateways_to_join: list[Gateway] = [] - self._trio_host: Any = None + self._host = default_host() if host is None else host self._async_group: Any = None self.set_profile("thread" if profile is None else profile) for xspec in xspecs: self.makegateway(xspec) atexit.register(self._cleanup_atexit) - def _ensure_trio_host(self) -> Any: - if self._trio_host is None: - from . import _trio_host + @property + def host(self) -> Host: + """The Trio host thread this group's protocol IO runs on.""" + return self._host - self._trio_host = _trio_host.TrioHost(name="execnet-trio-group") - self._trio_host.start() - return self._trio_host + def _ensure_trio_host(self) -> Any: + return self._host._ensure_started() def _ensure_async_group(self) -> Any: """The FacadeAsyncGroup owning the async side, running on the host.""" @@ -218,6 +224,7 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: If no spec is given, self.defaultspec is used. """ + check_not_in_event_loop("Group.makegateway()") if not spec: spec = self.defaultspec if not isinstance(spec, XSpec): @@ -260,15 +267,16 @@ def _unregister(self, gateway: Gateway) -> None: self._gateways_to_join.append(gateway) def _cleanup_atexit(self) -> None: + # The host is shared and stops itself at exit; a group only owns + # its gateways and the async group task running on that host. trace(f"=== atexit cleanup {self!r} ===") self.terminate(timeout=1.0) if self._async_group is not None: with suppress(Exception): - self._trio_host.call_sync(self._async_group.shutdown.set) + self._host._ensure_started().call_sync( + self._async_group.shutdown.set + ) self._async_group = None - if self._trio_host is not None: - self._trio_host.stop(timeout=1.0) - self._trio_host = None def terminate(self, timeout: float | None = None) -> None: """Trigger exit of member gateways and wait for termination @@ -307,12 +315,13 @@ def _host_terminate(self, timeout: float | None) -> None: a OneShot instead of blocking the OS thread (which would stall the hub for the whole grace). """ + trio_host = self._host._ensure_started() if self._wait_backend == "thread": - self._trio_host.call(self._async_group.terminate, timeout) + trio_host.call(self._async_group.terminate, timeout) return from ._boundary import make_wakener - self._trio_host.call_pending( + trio_host.call_pending( self._async_group.terminate, timeout, wakener=make_wakener(self._wait_backend), diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index db60920e..b1fccbf5 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -1033,7 +1033,7 @@ async def _terminate_one( @asynccontextmanager -async def open_popen_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGateway]: +async def open_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGateway]: """Spawn one popen worker and serve an AsyncGateway over its stdio. Runs inside the caller's own trio run -- no host thread involved. diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 82ad9c37..ed5ee8f0 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -46,11 +46,10 @@ from ._trio_gateway import open_popen_process from ._trio_gateway import read_handshake_ack from ._trio_gateway import ssh_transport_args +from ._host import DEFAULT_CALLBACK_THREADS from ._portal import LoopPortal from ._portal import OneShot -#: default cap on concurrent threadpool threads running receiver callbacks -DEFAULT_CALLBACK_THREADS = 40 #: bound on how long the endmarker callback may run during host shutdown CONSUMER_ENDMARKER_GRACE = 10.0 @@ -550,8 +549,13 @@ def is_alive(self) -> bool: class TrioHost: """Dedicated OS thread running ``trio.run`` for protocol IO.""" - def __init__(self, name: str = "execnet-trio-host") -> None: + def __init__( + self, + name: str = "execnet-trio-host", + callback_threads: int = DEFAULT_CALLBACK_THREADS, + ) -> None: self._name = name + self._callback_threads = callback_threads self._thread: threading.Thread | None = None self._portal: LoopPortal | None = None self._nursery: trio.Nursery | None = None @@ -591,7 +595,7 @@ def _run(self) -> None: async def _main(self) -> None: self._portal = LoopPortal() self._shutdown = trio.Event() - self._callback_limiter = trio.CapacityLimiter(DEFAULT_CALLBACK_THREADS) + self._callback_limiter = trio.CapacityLimiter(self._callback_threads) try: async with trio.open_nursery() as nursery: self._nursery = nursery diff --git a/src/execnet/aio.py b/src/execnet/aio.py index 36bd8751..3e3deee6 100644 --- a/src/execnet/aio.py +++ b/src/execnet/aio.py @@ -1,4 +1,4 @@ -"""The asyncio-native execnet API, bridged over a Trio host thread. +"""The asyncio-native execnet API. Everything here is awaited inside your own asyncio event loop:: @@ -6,22 +6,25 @@ import execnet.aio async def main(): - async with execnet.aio.Group() as group: + async with execnet.aio.AsyncGroup() as group: gateway = await group.makegateway("popen") channel = await gateway.remote_exec("channel.send(6 * 7)") print(await channel.receive()) asyncio.run(main()) -Protocol IO keeps running on a dedicated Trio host thread (the same -engine as the blocking and trio-native APIs, all transports included); -each awaited operation runs as a task on that host and resolves an -asyncio future via ``loop.call_soon_threadsafe``. No anyio port and no -executor threads per call. +Protocol IO keeps running on a Trio host thread (the same engine as the +blocking and trio-native APIs, all transports included); each awaited +operation runs as a task on that host and resolves an asyncio future via +``loop.call_soon_threadsafe``. No anyio port and no executor threads per +call. -Note: cancelling a bridged await abandons the operation on the asyncio -side only -- the host-side task runs to completion (a cancelled -``receive`` may still consume the next item). +Cancellation crosses the bridge. Cancelling an awaited ``receive`` (say +by ``asyncio.timeout``) cancels the host-side operation too, so no item +is consumed and dropped. Operations that must not tear halfway -- +``send``, ``send_eof``, ``aclose``, ``terminate`` -- are shielded +instead: the ``CancelledError`` reaches you, but the operation still +completes on the host. The error types are shared with :mod:`execnet.sync` and :mod:`execnet.trio`. Items you send must already be simple builtin data @@ -52,41 +55,59 @@ async def main(): from ._errors import LoadError from ._errors import RemoteError from ._errors import TimeoutError -from ._trio_gateway import AsyncChannel -from ._trio_gateway import AsyncGateway -from ._trio_gateway import AsyncGroup -from ._trio_host import TrioHost +from ._host import Host +from ._host import default_host +from ._trio_gateway import AsyncChannel as _TrioChannel +from ._trio_gateway import AsyncGateway as _TrioGateway +from ._trio_gateway import AsyncGroup as _TrioGroup from ._xspec import XSpec if TYPE_CHECKING: from typing_extensions import Self __all__ = [ - "Channel", + "AsyncChannel", + "AsyncGateway", + "AsyncGroup", "DataFormatError", "DumpError", - "Gateway", - "Group", + "Host", "HostNotFound", "LoadError", "RemoteError", "TimeoutError", "XSpec", - "open_popen_gateway", + "default_host", + "open_gateway", ] T = TypeVar("T") class _HostBridge: - """Await trio-native coroutines on a TrioHost from asyncio.""" + """Await trio-native coroutines on a Trio host from asyncio.""" - def __init__(self, host: TrioHost) -> None: - self._host = host + def __init__(self, trio_host: Any) -> None: + self._host = trio_host - async def call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: + async def call( + self, + async_fn: Callable[..., Awaitable[T]], + *args: Any, + shield: bool = False, + ) -> T: + """Run ``async_fn`` on the host and await its result. + + Unless ``shield``, cancelling the await cancels the host-side + operation too, so a cancelled ``receive`` does not consume an item + that nobody will ever see. + """ loop = asyncio.get_running_loop() future: asyncio.Future[T] = loop.create_future() + # Built here rather than inside the task: the cancel post can + # otherwise overtake the task's first step, and cancelling a scope + # nobody has entered yet still cancels it once entered. + scope = trio.CancelScope() def resolve(result: Any, error: BaseException | None) -> None: if future.cancelled(): @@ -104,15 +125,19 @@ def post_result(result: Any, error: BaseException | None) -> None: async def runner() -> None: try: - result = await async_fn(*args) + with scope: + result = await async_fn(*args) except trio.Cancelled: # host shutdown: the nursery cancel must propagate post_result(None, RuntimeError("execnet aio host was shut down")) raise except BaseException as exc: post_result(None, exc) - else: - post_result(result, None) + return + if scope.cancelled_caught: + # cancelled by us -- the awaiter is already gone + return + post_result(result, None) def spawn() -> None: self._host.start_soon(runner) @@ -121,10 +146,20 @@ def spawn() -> None: self._host.portal.post(spawn) except trio.RunFinishedError: raise RuntimeError("execnet aio group is not running") from None - return await future + + if shield: + # The host-side work runs to completion either way; shielding + # keeps a cancelled caller from abandoning a half-done send. + return await asyncio.shield(future) + try: + return await future + except asyncio.CancelledError: + with suppress(trio.RunFinishedError): + self._host.portal.post(scope.cancel) + raise -class _HostedGroup(AsyncGroup): +class _HostedGroup(_TrioGroup): """Trio AsyncGroup living as a task on the host nursery.""" def __init__(self, termination_timeout: float) -> None: @@ -141,13 +176,13 @@ async def run(self, task_status: trio.TaskStatus[_HostedGroup]) -> None: self.finished.set() -class Channel: - """asyncio facade over a trio-native :class:`AsyncChannel`.""" +class AsyncChannel: + """asyncio facade over a trio-native channel.""" RemoteError = RemoteError TimeoutError = TimeoutError - def __init__(self, bridge: _HostBridge, channel: AsyncChannel) -> None: + def __init__(self, bridge: _HostBridge, channel: _TrioChannel) -> None: self._bridge = bridge self._channel = channel @@ -156,41 +191,49 @@ def id(self) -> int: return self._channel.id def __repr__(self) -> str: - return f"" + return f"" def isclosed(self) -> bool: """Return True if the channel is closed for sending.""" return self._channel.isclosed() async def send(self, item: object) -> None: - """Serialize ``item`` and send it to the other side.""" - await self._bridge.call(self._channel.send, item) + """Serialize ``item`` and send it to the other side. + + Shielded: cancelling raises in the caller but the item is still + sent, rather than leaving a half-written frame on the wire. + """ + await self._bridge.call(self._channel.send, item, shield=True) async def receive(self, timeout: float | None = None) -> Any: """Receive the next item sent from the other side. EOFError once the peer closed or sent EOF, RemoteError for a peer close-with-error, TimeoutError after ``timeout`` seconds. A - received channel reference arrives as an :class:`~execnet.aio.Channel`. + received channel reference arrives as an + :class:`~execnet.aio.AsyncChannel`. + + Cancellable: no item is consumed if the await is cancelled, so + ``asyncio.timeout`` is equivalent to passing ``timeout``. """ result = await self._bridge.call(self._channel.receive, timeout) - if isinstance(result, AsyncChannel): - return Channel(self._bridge, result) + if isinstance(result, _TrioChannel): + return AsyncChannel(self._bridge, result) return result async def send_eof(self) -> None: """Signal that no more items follow (peer keeps its send side).""" - await self._bridge.call(self._channel.send_eof) + await self._bridge.call(self._channel.send_eof, shield=True) async def aclose(self, error: str | None = None) -> None: """Close the channel; ``error`` reaches the peer as a RemoteError.""" - await self._bridge.call(self._channel.aclose, error) + await self._bridge.call(self._channel.aclose, error, shield=True) async def wait_closed(self) -> None: """Wait until the peer closed or sent EOF; reraise remote errors.""" await self._bridge.call(self._channel.wait_closed) - def __aiter__(self) -> Channel: + def __aiter__(self) -> AsyncChannel: return self async def __anext__(self) -> Any: @@ -200,10 +243,10 @@ async def __anext__(self) -> Any: raise StopAsyncIteration from None -class Gateway: - """asyncio facade over a trio-native :class:`AsyncGateway`.""" +class AsyncGateway: + """asyncio facade over a trio-native gateway.""" - def __init__(self, bridge: _HostBridge, gateway: AsyncGateway) -> None: + def __init__(self, bridge: _HostBridge, gateway: _TrioGateway) -> None: self._bridge = bridge self._gateway = gateway @@ -216,13 +259,13 @@ def remoteaddress(self) -> str | None: return self._gateway.remoteaddress def __repr__(self) -> str: - return f"" + return f"" async def remote_exec( self, source: str | types.FunctionType | Callable[..., object] | types.ModuleType, **kwargs: object, - ) -> Channel: + ) -> AsyncChannel: """Connect a new channel to remote execution of ``source``. Accepts the same source kinds as ``Gateway.remote_exec``: a source @@ -232,86 +275,141 @@ async def remote_exec( channel = await self._bridge.call( functools.partial(self._gateway.remote_exec, source, **kwargs) ) - return Channel(self._bridge, channel) + return AsyncChannel(self._bridge, channel) async def terminate(self) -> None: """Send GATEWAY_TERMINATE to the peer, then close this side.""" - await self._bridge.call(self._gateway.terminate) + await self._bridge.call(self._gateway.terminate, shield=True) -class Group: - """asyncio-native gateway group over a dedicated Trio host thread. +class AsyncGroup: + """asyncio-native gateway group served on a Trio host thread. - An async context manager mirroring :class:`execnet.trio.AsyncGroup`; - leaving the ``async with`` block terminates every gateway with the - same bounded contract, then stops the host thread. + Usable as an async context manager, or driven explicitly with + :meth:`start` / :meth:`aclose` from application lifespan hooks. + Either way, shutting down terminates every gateway with the same + bounded contract as :class:`execnet.trio.AsyncGroup`. """ - def __init__(self, termination_timeout: float = 10.0) -> None: + def __init__( + self, + termination_timeout: float = 10.0, + *, + host: Host | None = None, + ) -> None: self._termination_timeout = termination_timeout - self._host: TrioHost | None = None + self._host = default_host() if host is None else host self._bridge: _HostBridge | None = None self._group: _HostedGroup | None = None def __repr__(self) -> str: state = "running" if self._group is not None else "idle" - return f"" + return f"" - async def __aenter__(self) -> Self: - assert self._host is None, "group already entered" - loop = asyncio.get_running_loop() - host = TrioHost(name="execnet-aio-group") - # start() blocks until the host loop is ready; keep it off the - # asyncio loop thread. - await loop.run_in_executor(None, host.start) - self._host = host - self._bridge = _HostBridge(host) + @property + def host(self) -> Host: + """The Trio host thread this group's protocol IO runs on.""" + return self._host + + async def start(self) -> None: + """Bring the host up and start the group task on it.""" + if self._group is not None: + raise RuntimeError(f"{self!r} is already started") + trio_host = await _start_host(self._host) + bridge = _HostBridge(trio_host) async def start_group() -> _HostedGroup: # runs on the host loop group = _HostedGroup(self._termination_timeout) - assert host._nursery is not None - started: _HostedGroup = await host._nursery.start(group.run) + assert trio_host._nursery is not None + started: _HostedGroup = await trio_host._nursery.start(group.run) return started - self._group = await self._bridge.call(start_group) - return self + self._bridge = bridge + self._group = await bridge.call(start_group, shield=True) - async def __aexit__(self, *exc_info: object) -> None: - group, bridge, host = self._group, self._bridge, self._host + async def aclose(self) -> None: + """Terminate every gateway and stop the group task (idempotent). + + The host thread is shared, so it keeps running for other groups. + """ + group, bridge = self._group, self._bridge self._group = self._bridge = None - if group is not None and bridge is not None: + if group is None or bridge is None: + return - async def stop_group() -> None: - group.shutdown.set() - await group.finished.wait() + async def stop_group() -> None: + group.shutdown.set() + await group.finished.wait() - with suppress(RuntimeError): - await bridge.call(stop_group) - if host is not None: - self._host = None - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, functools.partial(host.stop, 5.0)) + with suppress(RuntimeError): + await bridge.call(stop_group, shield=True) + + async def __aenter__(self) -> Self: + await self.start() + return self + + async def __aexit__(self, *exc_info: object) -> None: + await self.aclose() - async def makegateway(self, spec: str | XSpec = "popen") -> Gateway: + async def makegateway(self, spec: str | XSpec = "popen") -> AsyncGateway: """Create a gateway for ``spec`` served on the group's host. All transports are supported: popen (including uv-provisioned ``python=``), ``ssh=``, ``vagrant_ssh=``, ``socket=`` (with - ``installvia=``), and ``via=`` sub-gateways. + ``installvia=``), and ``via=`` sub-gateways. The worker profile + defaults to ``thread``; pass ``profile=trio`` for a worker that + runs exec'd async sources as tasks. """ group, bridge = self._group, self._bridge if group is None or bridge is None: - raise RuntimeError(f"{self!r} is not entered") + raise RuntimeError(f"{self!r} is not started") gateway = await bridge.call(group.makegateway, spec) - return Gateway(bridge, gateway) + return AsyncGateway(bridge, gateway) + + +async def _start_host(host: Host) -> Any: + """Start ``host`` without blocking the asyncio loop or its executor. + + ``Host._ensure_started`` blocks until the trio loop is ready, so it + runs on a throwaway thread whose completion is posted back to the + loop -- never on the default executor, which belongs to the caller's + application. + """ + if host.running: + return host._ensure_started() + import threading + + loop = asyncio.get_running_loop() + future: asyncio.Future[Any] = loop.create_future() + + def start() -> None: + try: + trio_host = host._ensure_started() + except BaseException as exc: # noqa: BLE001 + loop.call_soon_threadsafe(_set_future_error, future, exc) + else: + loop.call_soon_threadsafe(_set_future_result, future, trio_host) + + threading.Thread(target=start, name="execnet-host-start", daemon=True).start() + return await future + + +def _set_future_result(future: asyncio.Future[Any], value: Any) -> None: + if not future.cancelled(): + future.set_result(value) + + +def _set_future_error(future: asyncio.Future[Any], error: BaseException) -> None: + if not future.cancelled(): + future.set_exception(error) @asynccontextmanager -async def open_popen_gateway(spec: str | XSpec = "popen") -> AsyncIterator[Gateway]: - """Spawn one popen worker and yield an asyncio Gateway to it. +async def open_gateway(spec: str | XSpec = "popen") -> AsyncIterator[AsyncGateway]: + """Spawn one worker for ``spec`` and yield an asyncio gateway to it. - Convenience for a single-gateway :class:`Group`. + Convenience for a single-gateway :class:`AsyncGroup`. """ - async with Group() as group: + async with AsyncGroup() as group: yield await group.makegateway(spec) diff --git a/src/execnet/gevent.py b/src/execnet/gevent.py index f3f12609..df5aadfb 100644 --- a/src/execnet/gevent.py +++ b/src/execnet/gevent.py @@ -36,6 +36,7 @@ from ._xspec import XSpec from .sync import Channel from .sync import Gateway +from .sync import Host from .sync import RSync __all__ = [ @@ -44,6 +45,7 @@ "DumpError", "Gateway", "Group", + "Host", "HostNotFound", "LoadError", "MultiChannel", diff --git a/src/execnet/sync.py b/src/execnet/sync.py index 1160b89c..e32951d8 100644 --- a/src/execnet/sync.py +++ b/src/execnet/sync.py @@ -14,6 +14,7 @@ from ._errors import RemoteError from ._errors import TimeoutError from ._gateway import Gateway +from ._host import Host from ._multi import Group from ._multi import MultiChannel from ._multi import default_group @@ -29,6 +30,7 @@ "DumpError", "Gateway", "Group", + "Host", "HostNotFound", "LoadError", "MultiChannel", diff --git a/src/execnet/trio.py b/src/execnet/trio.py index 9704b6ef..0fc3b5df 100644 --- a/src/execnet/trio.py +++ b/src/execnet/trio.py @@ -29,7 +29,7 @@ async def main(): from ._trio_gateway import AsyncChannel from ._trio_gateway import AsyncGateway from ._trio_gateway import AsyncGroup -from ._trio_gateway import open_popen_gateway +from ._trio_gateway import open_gateway from ._xspec import XSpec __all__ = [ @@ -43,5 +43,5 @@ async def main(): "RemoteError", "TimeoutError", "XSpec", - "open_popen_gateway", + "open_gateway", ] diff --git a/testing/test_aio.py b/testing/test_aio.py index 6bdb7724..8c51ecf2 100644 --- a/testing/test_aio.py +++ b/testing/test_aio.py @@ -22,7 +22,7 @@ def run(main: Awaitable[T]) -> T: def test_popen_roundtrip() -> None: async def main() -> None: - async with execnet.aio.Group() as group: + async with execnet.aio.AsyncGroup() as group: gateway = await group.makegateway("popen") channel = await gateway.remote_exec("channel.send(channel.receive() + 1)") await channel.send(41) @@ -32,9 +32,9 @@ async def main() -> None: run(main()) -def test_open_popen_gateway_iteration() -> None: +def test_open_gateway_iteration() -> None: async def main() -> list[int]: - async with execnet.aio.open_popen_gateway() as gateway: + async with execnet.aio.open_gateway() as gateway: channel = await gateway.remote_exec( "for i in range(4): channel.send(i * 2)" ) @@ -45,7 +45,7 @@ async def main() -> list[int]: def test_receive_timeout() -> None: async def main() -> None: - async with execnet.aio.open_popen_gateway() as gateway: + async with execnet.aio.open_gateway() as gateway: channel = await gateway.remote_exec("channel.receive()") with pytest.raises(channel.TimeoutError): await channel.receive(timeout=0.05) @@ -56,7 +56,7 @@ async def main() -> None: def test_remote_error() -> None: async def main() -> None: - async with execnet.aio.open_popen_gateway() as gateway: + async with execnet.aio.open_gateway() as gateway: channel = await gateway.remote_exec("raise ValueError(17)") with pytest.raises(execnet.aio.RemoteError, match="ValueError"): await channel.receive() @@ -66,7 +66,7 @@ async def main() -> None: def test_channel_passing_wraps_aio() -> None: async def main() -> None: - async with execnet.aio.open_popen_gateway() as gateway: + async with execnet.aio.open_gateway() as gateway: channel = await gateway.remote_exec( """ c = channel.gateway.newchannel() @@ -75,7 +75,7 @@ async def main() -> None: """ ) passed = await channel.receive() - assert isinstance(passed, execnet.aio.Channel) + assert isinstance(passed, execnet.aio.AsyncChannel) assert await passed.receive() == 42 run(main()) @@ -83,7 +83,7 @@ async def main() -> None: def test_multiple_gateways_and_send_each() -> None: async def main() -> list[Any]: - async with execnet.aio.Group() as group: + async with execnet.aio.AsyncGroup() as group: gateways = [await group.makegateway("popen") for _ in range(2)] channels = [ await gw.remote_exec("channel.send(channel.receive() * 2)") @@ -98,8 +98,8 @@ async def main() -> list[Any]: def test_group_not_entered() -> None: async def main() -> None: - group = execnet.aio.Group() - with pytest.raises(RuntimeError, match="not entered"): + group = execnet.aio.AsyncGroup() + with pytest.raises(RuntimeError, match="not started"): await group.makegateway("popen") run(main()) @@ -107,10 +107,78 @@ async def main() -> None: def test_terminate_gateway_explicitly() -> None: async def main() -> None: - async with execnet.aio.Group() as group: + async with execnet.aio.AsyncGroup() as group: gateway = await group.makegateway("popen") channel = await gateway.remote_exec("channel.send(1)") assert await channel.receive() == 1 await gateway.terminate() run(main()) + + +def test_cancelled_receive_does_not_consume_an_item() -> None: + # The bridge cancels the host-side receive, so the item stays queued + # instead of being consumed and dropped -- asyncio.timeout around a + # receive must behave like passing timeout=. + async def main() -> None: + async with execnet.aio.open_gateway() as gateway: + channel = await gateway.remote_exec( + """ + channel.receive() + for i in range(3): + channel.send(i) + """ + ) + with pytest.raises(asyncio.TimeoutError): + async with asyncio.timeout(0.05): + await channel.receive() + # release the worker; nothing was consumed by the cancelled wait + await channel.send("go") + assert [await channel.receive() for _ in range(3)] == [0, 1, 2] + + run(main()) + + +def test_cancelled_send_still_arrives() -> None: + # send is shielded: the caller sees CancelledError but the item is on + # the wire, rather than a frame half-written to the peer. + async def main() -> None: + async with execnet.aio.open_gateway() as gateway: + channel = await gateway.remote_exec( + "channel.send(channel.receive() * 2)" + ) + task = asyncio.ensure_future(channel.send(21)) + await asyncio.sleep(0) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert await channel.receive() == 42 + + run(main()) + + +def test_group_start_and_aclose_explicitly() -> None: + # asyncio apps drive this from lifespan hooks rather than "async with" + async def main() -> None: + group = execnet.aio.AsyncGroup() + await group.start() + try: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(7)") + assert await channel.receive() == 7 + finally: + await group.aclose() + await group.aclose() # idempotent + with pytest.raises(RuntimeError, match="not started"): + await group.makegateway("popen") + + run(main()) + + +def test_groups_share_the_default_host() -> None: + async def main() -> None: + async with execnet.aio.AsyncGroup() as a, execnet.aio.AsyncGroup() as b: + assert a.host is b.host + assert a.host is execnet.aio.default_host() + + run(main()) diff --git a/testing/test_basics.py b/testing/test_basics.py index a81ed1f6..0afcdd0d 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -346,6 +346,9 @@ class FakeGateway: _trio_session = None _new_wakener = staticmethod(_boundary.ThreadWakener) + def _check_event_loop(self, what: str) -> None: + pass + def _trace(self, *args) -> None: pass diff --git a/testing/test_host.py b/testing/test_host.py new file mode 100644 index 00000000..686c022f --- /dev/null +++ b/testing/test_host.py @@ -0,0 +1,201 @@ +"""The Trio host thread: sharing, explicit override, and loop-misuse guards. + +``execnet.trio`` runs gateways directly in the caller's own nursery; every +other surface drives a :class:`execnet.Host`. One is shared per process, +and blocking on it from inside a running event loop is an error rather +than a hang. +""" + +from __future__ import annotations + +import asyncio +import os +import sys +import threading + +import pytest +import trio + +import execnet +from execnet._host import Host +from execnet._host import default_host + +TESTTIMEOUT = 30.0 + + +def host_thread_names() -> list[str]: + return [t.name for t in threading.enumerate() if t.name.startswith("execnet-host")] + + +class TestSharedHost: + def test_groups_share_the_default_host(self) -> None: + a = execnet.Group() + b = execnet.Group() + assert a.host is b.host is default_host() + + def test_many_groups_run_one_thread(self) -> None: + groups = [execnet.Group() for _ in range(3)] + try: + for group in groups: + group.makegateway("popen") + assert len(host_thread_names()) == 1 + for group in groups: + channel = group[0].remote_exec("channel.send(1)") + assert channel.receive(TESTTIMEOUT) == 1 + finally: + for group in groups: + group.terminate(timeout=5.0) + + def test_explicit_host_is_isolated_and_closes(self) -> None: + host = Host(name="execnet-host-isolated") + assert not host.running + group = execnet.Group(host=host) + assert group.host is host + assert group.host is not default_host() + try: + gateway = group.makegateway("popen") + channel = gateway.remote_exec("channel.send(6 * 7)") + assert channel.receive(TESTTIMEOUT) == 42 + assert host.running + assert "execnet-host-isolated" in host_thread_names() + finally: + group.terminate(timeout=5.0) + host.close() + assert not host.running + assert "execnet-host-isolated" not in host_thread_names() + + def test_host_context_manager_closes(self) -> None: + with Host(name="execnet-host-ctx") as host: + group = execnet.Group(host=host) + group.makegateway("popen") + group.terminate(timeout=5.0) + assert not host.running + + def test_starting_is_lazy(self) -> None: + host = Host(name="execnet-host-lazy") + execnet.Group(host=host) + # constructing a group must not cost a thread + assert not host.running + assert "execnet-host-lazy" not in host_thread_names() + + @pytest.mark.skipif( + not hasattr(os, "fork"), reason="requires os.fork" + ) + def test_forked_child_gets_a_fresh_default_host(self) -> None: + # the child inherits a Host object whose thread does not exist + # there, so the first use must build a new one + default_host()._ensure_started() + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: # pragma: no cover - runs in the child + code = 1 + try: + os.close(read_fd) + child_host = default_host() + group = execnet.Group() + gateway = group.makegateway("popen") + got = gateway.remote_exec("channel.send(3)").receive(TESTTIMEOUT) + group.terminate(timeout=5.0) + code = 0 if (got == 3 and child_host.running) else 1 + finally: + os.write(write_fd, bytes([code])) + os._exit(0) + os.close(write_fd) + try: + result = os.read(read_fd, 1) + finally: + os.close(read_fd) + os.waitpid(pid, 0) + assert result == b"\x00" + + +class TestEventLoopGuard: + """Blocking on the host from inside a running loop must not hang.""" + + def test_makegateway_inside_asyncio_raises(self) -> None: + async def main() -> None: + with pytest.raises(RuntimeError, match="execnet.aio"): + execnet.Group().makegateway("popen") + + asyncio.run(main()) + + def test_makegateway_inside_trio_raises(self) -> None: + async def main() -> None: + with pytest.raises(RuntimeError, match="execnet.trio"): + execnet.Group().makegateway("popen") + + trio.run(main) + + def test_channel_receive_inside_asyncio_raises(self) -> None: + group = execnet.Group() + try: + gateway = group.makegateway("popen") + channel = gateway.remote_exec("channel.send(1)") + + async def main() -> None: + with pytest.raises(RuntimeError, match="execnet.aio"): + channel.receive(TESTTIMEOUT) + with pytest.raises(RuntimeError, match="execnet.aio"): + channel.send(1) + with pytest.raises(RuntimeError, match="execnet.aio"): + channel.waitclose(TESTTIMEOUT) + + asyncio.run(main()) + # still usable from a plain thread afterwards + assert channel.receive(TESTTIMEOUT) == 1 + finally: + group.terminate(timeout=5.0) + + def test_worker_channels_are_not_guarded(self) -> None: + # exec'd code may run its own event loop and talk to its channel + # from inside it -- that is the caller's own loop to block + group = execnet.Group() + try: + gateway = group.makegateway("popen") + channel = gateway.remote_exec( + """ + import asyncio + + async def main(): + channel.send(channel.receive() + 1) + + asyncio.run(main()) + """ + ) + channel.send(41) + assert channel.receive(TESTTIMEOUT) == 42 + finally: + group.terminate(timeout=5.0) + + def test_a_worker_thread_is_still_fine(self) -> None: + group = execnet.Group() + result: list[object] = [] + + def work() -> None: + gateway = group.makegateway("popen") + result.append(gateway.remote_exec("channel.send(5)").receive(TESTTIMEOUT)) + + async def main() -> None: + # inside a loop, but the blocking call happens off it + await asyncio.to_thread(work) + + try: + asyncio.run(main()) + assert result == [5] + finally: + group.terminate(timeout=5.0) + + def test_no_asyncio_import_no_cost(self) -> None: + # the probe must not import asyncio/trio into a program that has + # neither; it goes through sys.modules first + code = ( + "import sys, execnet;" + " execnet._host.check_not_in_event_loop('x');" + " print('asyncio' in sys.modules, 'trio' in sys.modules)" + ) + import subprocess + + out = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + assert out.stdout.split() == ["False", "False"] diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index dc6f2d9d..856268ea 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -80,7 +80,7 @@ def test_trio_namespace_exposes_async_core() -> None: assert execnet.trio.AsyncGroup is _trio_gateway.AsyncGroup assert execnet.trio.AsyncGateway is _trio_gateway.AsyncGateway assert execnet.trio.AsyncChannel is _trio_gateway.AsyncChannel - assert execnet.trio.open_popen_gateway is _trio_gateway.open_popen_gateway + assert execnet.trio.open_gateway is _trio_gateway.open_gateway # error types are shared with the sync surface; the standalone serializer # is intentionally not exposed on any public namespace assert execnet.trio.RemoteError is execnet.RemoteError diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index 35a62c50..9473df89 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -22,7 +22,7 @@ from execnet._serialize import loads_internal from execnet._trio_gateway import AsyncGateway from execnet._trio_gateway import AsyncGroup -from execnet._trio_gateway import open_popen_gateway +from execnet._trio_gateway import open_gateway @asynccontextmanager @@ -284,7 +284,7 @@ class TestPopenAsyncGateway: def test_remote_exec_roundtrip(self) -> None: async def main() -> None: - async with open_popen_gateway() as gateway: + async with open_gateway() as gateway: channel = await gateway.remote_exec( "channel.send(channel.receive() + 1)" ) @@ -296,7 +296,7 @@ async def main() -> None: def test_remote_exec_function_with_kwargs(self) -> None: async def main() -> None: - async with open_popen_gateway() as gateway: + async with open_gateway() as gateway: channel = await gateway.remote_exec(_remote_add, a=40, b=2) assert await channel.receive() == 42 @@ -304,7 +304,7 @@ async def main() -> None: def test_remote_error_propagates(self) -> None: async def main() -> None: - async with open_popen_gateway() as gateway: + async with open_gateway() as gateway: channel = await gateway.remote_exec("raise ValueError('kaboom')") with pytest.raises(RemoteError, match="kaboom"): await channel.receive() @@ -313,7 +313,7 @@ async def main() -> None: def test_exec_finish_closes_channel_ending_iteration(self) -> None: async def main() -> None: - async with open_popen_gateway() as gateway: + async with open_gateway() as gateway: channel = await gateway.remote_exec( "for i in range(3): channel.send(i)" ) @@ -323,7 +323,7 @@ async def main() -> None: def test_concurrent_remote_execs(self) -> None: async def main() -> None: - async with open_popen_gateway() as gateway: + async with open_gateway() as gateway: results = [] async def run_one(value: int) -> None: From ddce312abf368cbb1141e26dab6e84d7e123731e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Wed, 29 Jul 2026 16:49:46 +0200 Subject: [PATCH 64/91] docs: record the surface review in the changelog and handoffs Adds the changelog entries for the four namespaces, the shared Host, the loop guards, profile=/main_thread_only and the aio cancellation fix, and corrects the two earlier entries that still advertised execnet.portal. The phase C handoff gains a "Surface review" section that wins over everything below it, and the boundary-protocol handoff is marked superseded where it promised a Wakener extension point. Also drops the EXECMODEL_PROFILES entry from the gateway_base shim: that name was added on this branch, so it never existed pre-Trio and nothing can be forwarding to it. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 61 ++++++++++++++++++++-- handoff-boundary-protocol-rethink.md | 20 +++++-- handoff-phase-c-worker-axes.md | 78 ++++++++++++++++++++++++---- src/execnet/_execmodel.py | 4 +- src/execnet/_host.py | 6 ++- src/execnet/_multi.py | 8 +-- src/execnet/_trio_gateway.py | 2 +- src/execnet/_trio_host.py | 6 +-- src/execnet/_trio_worker.py | 22 +++++--- src/execnet/aio.py | 2 +- src/execnet/gateway_base.py | 1 - testing/conftest.py | 3 +- testing/test_aio.py | 9 ++-- testing/test_execmodel_trio.py | 19 +++++++ testing/test_gateway.py | 1 - testing/test_host.py | 16 +++--- 16 files changed, 203 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0a966560..93a52284 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,61 @@ 2.2.0 (UNRELEASED) ------------------ +* One namespace per concurrency library you drive execnet from: + ``execnet.sync`` (plain threads; the top-level ``execnet.*`` aliases), + ``execnet.trio``, ``execnet.aio`` and the new ``execnet.gevent``, whose blocking + waits park the calling greenlet instead of its OS thread (needs ``execnet[gevent]``). + + ``execnet.trio`` is the only surface that runs gateways *directly*, as tasks in your + own nursery. The others drive a Trio host thread, so their blocking calls now raise + when made from inside a running asyncio or trio loop -- naming the namespace to use + instead -- rather than stalling that loop. Channels inside a worker are exempt: + exec'd code may run its own event loop and talk to its channel from within it. +* The ``execnet.portal`` namespace is gone. It exposed the ``Wakener`` protocol and the + ``Mailbox``/``OneShot``/``LoopPortal`` primitives but not the registry needed to plug + a ``Wakener`` in, and execnet does not offer third-party event-loop integration: a new + concurrency library gets a namespace of its own, as gevent just did. The primitives + are internal again. +* Gateway groups share one Trio host thread per process instead of starting one each. + Pass ``execnet.Host()`` as ``Group(host=...)`` (or ``AsyncGroup(host=...)``) for an + isolated loop with deterministic teardown -- ``Host`` is a context manager and joins + its thread on exit, where the shared one stops at interpreter exit. +* The ``execmodel=`` spec key is now ``profile=``; ``execmodel=`` remains an accepted + alias. It always selected the *worker* profile -- where exec'd code runs relative to + the worker's protocol loop -- while the local execution model it was named after no + longer exists. Accordingly ``Group(execmodel=...)``, ``Group.set_execmodel()``, + ``Group.execmodel`` and ``Group.remote_execmodel`` are deprecated in favour of + ``Group(profile=...)``, ``Group.set_profile()`` and ``Group.profile``; only the + remote default they set has any effect. ``remote_status()`` reports both + ``profile`` and ``execmodel``. +* The ``main_thread_only`` profile is deprecated and now behaves like ``thread``, which + already hands the first ``remote_exec`` the worker's real main thread -- the + GUI/signal-safety property it was added for in 2.1.0. Its other behaviour is gone: + a second concurrent ``remote_exec`` used to close the channel with + ``concurrent remote_exec would cause deadlock``, and now runs on a pool thread. That + guard was a one-second timeout that reported a merely slow predecessor as a deadlock. +* The ``wait=`` spec key added earlier in this release cycle is gone. Which primitive a + blocking wait parks on describes the *caller*, which is what choosing a namespace + already says; a worker's own backend is derived from its profile. +* ``execnet.aio`` now propagates cancellation. Cancelling an awaited ``receive`` (with + ``asyncio.timeout``, say) cancels the host-side operation, where it previously + abandoned only the asyncio side and let the operation consume an item that was then + discarded. ``send``, ``send_eof``, ``aclose`` and ``terminate`` are shielded instead, + so they cannot tear halfway. + + Its classes gained the ``Async`` prefix -- ``AsyncGroup``, ``AsyncGateway``, + ``AsyncChannel`` -- matching ``execnet.trio``, and ``AsyncGroup`` can be driven with + ``start()``/``aclose()`` from application lifespan hooks instead of ``async with``. + ``open_popen_gateway`` is renamed ``open_gateway`` on both async namespaces, since it + always accepted any spec. +* ``execnet.dumps``, the temporary pytest-xdist compatibility shim, now warns once per + process rather than on every access. xdist reaches it from ``serialize_warning_message``, + i.e. from inside pytest's warning-recording hook, so a single ``DeprecationWarning`` + raised in a worker made recording that warning record another, unbounded, wedging the + run. +* ``Gateway.remote_init_threads()`` raises a ``DeprecationWarning`` instead of printing + to stdout. It has been a no-operation since execnet 1.2. + * `#380 `__: Add support for Python 3.13 and 3.14, and drop EOL 3.8 and 3.9. * Trio host-thread Message IO for local ``popen`` + import bootstrap (coordinator and worker). Adds a hard ``trio`` dependency. Disable with ``EXECNET_TRIO_HOST=0``. @@ -31,7 +86,8 @@ load. Opcode bytes are unchanged for every type that survives. * The supported API is now exactly five namespaces: ``execnet`` (aliases of ``execnet.sync``), ``execnet.sync``, ``execnet.trio``, ``execnet.aio`` and - ``execnet.portal``. The pre-Trio modules ``execnet.gateway_base``, ``execnet.gateway``, + ``execnet.gevent`` -- one per concurrency library you drive execnet from. The + pre-Trio modules ``execnet.gateway_base``, ``execnet.gateway``, ``execnet.multi``, ``execnet.rsync``, ``execnet.rsync_remote`` and ``execnet.xspec`` were only ever reachable because ``import execnet`` pulled them in transitively; they are now deprecated forwarding shims that warn on attribute access and will be removed @@ -55,8 +111,7 @@ ``can_send``. * ``execnet.trio`` no longer exports ``ByteStream``, ``RawChannel``, ``RawChannelStream`` or ``serve_gateway``; the raw-channel layer is internal routing - detail. No names were added to ``execnet.sync``, ``execnet.aio`` or - ``execnet.portal``. + detail. No names were added to ``execnet.sync`` or ``execnet.aio``. 2.1.2 (2025-11-11) diff --git a/handoff-boundary-protocol-rethink.md b/handoff-boundary-protocol-rethink.md index 70146c7d..5f108df2 100644 --- a/handoff-boundary-protocol-rethink.md +++ b/handoff-boundary-protocol-rethink.md @@ -128,9 +128,23 @@ port. Phase E (anyio-native core) becomes optional purity/perf work. in the first waiter's hub (carriers may be constructed on the host loop); tests behind the `gevent` dependency group. -Next: Phase C `loop=main` / `exec=task` on the smaller core -(handoff-phase-c-worker-axes.md), and Phase D docs cover the four -namespaces + the axes/presets. +## SUPERSEDED 2026-07-29 by the surface review + +Phase C landed, and reviewing the whole public surface at once (before +Phase D docs froze it) retired several of the decisions above. See +"Surface review" in `handoff-phase-c-worker-axes.md` for what stands. In +short: + +- **The Wakener extension point is gone.** There was never a plan to let + third parties add event loops, and the asyncio wakener sketched here was + deliberately never built (P4 chose per-call bridging). What is left is + two wait backends, threads and gevent, behind a two-branch + `make_wakener`. `execnet.portal` is now private `execnet._portal`. +- **`wait=` is gone as a spec key.** It described the *caller's* + concurrency library, which is what picking a namespace already says. + gevent got the facade it was missing: `execnet.gevent`. +- **The `loop=` / `exec=` axes stayed dropped**; `execmodel=` became + `profile=` (see below). ## Invariants (unchanged, re-mapped) diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index 102cb78f..ba289844 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -17,6 +17,63 @@ pytest testing/ -n 12` passes (~7s) since the session-attach race fix `testing/test_ssh_local.py` (asyncssh server; system ssh client needed). Known flake: `test_socket_installvia` EOFs rarely under load. +## Surface review — LANDED 2026-07-29 (`7aa17fe..e75cd0a`) + +The public surface had settled commit by commit and was never reviewed as +a whole. Doing that before the Phase D docs froze it produced the +following, which **wins over anything below or in +`handoff-boundary-protocol-rethink.md` that contradicts it**. + +**Namespaces are now one per concurrency library you drive execnet from**: +`execnet.sync` (threads; the top-level aliases), `execnet.trio`, +`execnet.aio`, `execnet.gevent`. `execnet.portal` is gone -- +`execnet._portal` plus the trio-free `execnet._boundary`. + +| was | is | why | +|---|---|---| +| `execnet.portal` public | `execnet._portal` private | it exported `Wakener`/`Mailbox`/`OneShot`/`LoopPortal` but *not* `register_wakener`, so the advertised extension point was unreachable -- and there is no plan to let third parties add event loops at all | +| Wakener registry (`register_wakener`, lazy module table) | two-branch `make_wakener(Literal["thread","gevent"])` | exactly two backends exist; every other library gets a facade | +| `wait=` spec key | gone; `Group._wait_backend`, set by the facade | it described the *caller*, which the namespace already says. Worker-side wait was always derived from the profile | +| gevent via `wait=gevent` | `execnet.gevent.Group` | symmetric with the other surfaces | +| `execmodel=` spec key | `profile=` (`execmodel=` a permanent alias) | the key selects the *worker profile*; the local execution model it was named after no longer exists | +| `Group(execmodel=)`, `set_execmodel`, `group.execmodel` | deprecated; only the remote default survives | they had no behavioural effect. **xdist passes `Group(execmodel=...)` as a keyword** -- that must keep working | +| `main_thread_only` profile | deprecated alias for `thread` | `HybridExec` already gives the first remote_exec the real main thread. Its extra behaviour (refusing a second concurrent remote_exec) was a 1s-timeout deadlock guard, now deleted | +| one `TrioHost` per `Group` | one shared `execnet.Host` per process, `Group(host=...)` to override | a host is a thread and a loop, not something groups need isolated | +| blocking inside a running loop hangs | raises, naming `execnet.aio` / `execnet.trio` | worker-side channels stay exempt: exec'd code may run its own loop | +| `aio.Group`/`Gateway`/`Channel` | `aio.AsyncGroup`/`AsyncGateway`/`AsyncChannel` | matches `execnet.trio`; swapping the import ports the code | +| `open_popen_gateway` | `open_gateway` (both async surfaces) | it always accepted any spec | + +Behaviour changes worth a changelog line: + +- a second concurrent `remote_exec` under `main_thread_only` used to close + the channel with `MAIN_THREAD_ONLY_DEADLOCK_TEXT`; it now runs on a pool + thread. `_executetask_complete`, `MAIN_THREAD_ONLY_ADMIT_TIMEOUT` and + the error text are deleted; `MainExec` became `PrimaryThreadPump`. +- `execnet.aio` cancellation is now real: a cancelled `receive` cancels + the host-side operation instead of consuming and discarding an item. + `send`/`send_eof`/`aclose`/`terminate` are shielded instead. +- the boundary carriers raise `execnet.TimeoutError`, not the builtin; + `OneShot` double-resolve is a `RuntimeError`, not an `assert`. +- `STATUS` answers both `profile` and (legacy) `execmodel`. +- **Latent livelock fixed**: `execnet.dumps` warned on *every* access, and + xdist calls it from `serialize_warning_message` -- i.e. from inside + pytest's warning-recording hook. One DeprecationWarning in a worker + therefore recorded a warning that recorded a warning, unbounded, and + wedged the run. The shim warns once per process + (`execnet._xdist_compat_warned`). Anything that warns in a worker can + hit this class of bug; keep it in mind. + +New tests: `testing/test_host.py` (sharing, explicit `Host`, fork, the +loop guards), `testing/test_boundary.py` (renamed from `test_portal.py`), +aio cancellation contracts in `testing/test_aio.py`. The `execmodel` +fixture parametrization collapsed to a single `profile` fixture, so the +suite is ~540 items rather than ~765. + +Still open from the review, deliberately not done: the async surfaces have +no `remote_status()`, no `MultiChannel`, no group iteration, and no +`RSync`. `AsyncGroup.makegateway` defaults workers to the `thread` +profile (the coordinator's shape does not dictate the worker's). + ## Where the repo stands (2026-07-25, after `a69b844`) One protocol engine: `AsyncGateway` (`_trio_gateway.py`). The sync API @@ -45,12 +102,12 @@ File map (src/execnet/): | file | role | |---|---| -| `gateway_base.py` | `Message` wire protocol + sans-IO `FrameDecoder`, serializer (CHANNEL opcode incl. duck-typed `save_AsyncChannel`), sync `Channel`/`ChannelFactory` (raw-receiver registry), `BaseGateway`/`WorkerGateway` (`_send` via bridge, `_send_nonblocking` for GC), `ExecModel`, `WorkerPool`, `HostNotFound` | +| `_message.py` / `_serialize.py` / `_channel.py` / `_gateway_base.py` / `_errors.py` / `_execmodel.py` | split by concern since this table was written: wire protocol + sans-IO `FrameDecoder`; serializer (CHANNEL opcode incl. duck-typed `save_AsyncChannel`); sync `Channel`/`ChannelFactory`; `BaseGateway`/`WorkerGateway`; error types; `WORKER_PROFILES` + `resolve_profile` + the deprecated `ExecModel` xdist shim. `gateway_base.py` is now only a warning shim | | `_trio_gateway.py` | async core: `ByteStream` Protocol, `RawChannel`/`AsyncChannel`, `AsyncGateway` (outbound queue of `(frame, on_written)`; `_finalize` hook), `AsyncGroup` (all transports, overridable `_make_gateway`/`_open_via_stream`/`_resolve_socket_address`, reapers, bounded terminate), stream/argv helpers | | `_trio_host.py` | `TrioHost` (loop thread), `SyncBridgeGateway`, `FacadeAsyncGroup`, `makegateway_trio`, `SyncIOHandle`, `RawTunnelStream` (via tunnel; `aclose` feeds own reader EOF), `GATEWAY_START_*` handlers | | `_trio_worker.py` | worker entry (`_main`/`serve_popen_trio`/`serve_socket_trio`), `TrioWorkerExec` (FIFO `_pump` admission; `integrate_as_primary_thread`), `_prepare_protocol_fds`, `_check_version` | | `gateway.py` / `multi.py` | sync `Gateway` / sync `Group` facade, `MultiChannel`, `safe_terminate` (WorkerPool-based, kept for tests) | -| `sync.py` / `trio.py` / `portal.py` | the three public namespaces | +| `sync.py` / `trio.py` / `aio.py` / `gevent.py` | the four public namespaces; `_host.py` holds the shared `Host`, `_portal.py`/`_boundary.py` the (private) boundary kit | | `_exec_source.py`, `_provision.py`, `xspec.py`, `rsync.py` | source normalization, uv provisioning + argv builders, spec parsing, rsync | ## Semantics that MUST survive (xdist depends on them) @@ -84,20 +141,23 @@ different use-cases get named profiles, `execmodel=` is the public mode key (xdist already passes it), and `wait=` is the only other public knob. Implemented in commits `b2f43c3..cbce183`: -| `execmodel=` | loop thread | exec'd code runs | channel | worker wait | extra deps | +| `profile=` (was `execmodel=`) | loop thread | exec'd code runs | channel | worker wait | extra deps | |---|---|---|---|---|---| | `thread` (default) | side thread | **classic hybrid restored**: primary on the main thread, overflow on pool threads (claim decided during FIFO admission) | sync | thread | — | -| `main_thread_only` | side thread | main thread, serialized (deadlock guard) | sync | thread | — | +| ~~`main_thread_only`~~ | *deprecated 2026-07-29, aliases to `thread`* | | | | | | `trio` (new) | **main thread** | async sources as tasks — one single thread total; top-level await or async def; sync sources rejected; termination cancels tasks | AsyncChannel | (loop) | — | | `gevent` (revived) | side thread | greenlets on a main-thread hub, one per remote_exec | sync | gevent (derived) | `execnet[gevent]`, auto-added by uv provisioning | +(The coordinator-side counterpart of the last row is now `execnet.gevent`, +not `wait=gevent`.) + Architecture: `TrioWorkerExec` is a pure FIFO admission pump delegating to strategy objects (`WORKER_EXEC_STRATEGIES` in `_trio_worker.py`: PoolExec building block, MainExec, HybridExec, GreenletExec; TaskExec serves a plain AsyncGateway via its pluggable `_exec_handler` — no sync bridge at all in the trio profile). Subinterpreters: future strategy -slot, not built. `EXECMODEL_PROFILES` (gateway_base) validates -coordinator-side in makegateway. +slot, not built. `WORKER_PROFILES` (`_execmodel.py`) validates +coordinator-side in makegateway, via `resolve_profile`. **Native info/setup** (the pytest fix): `Message.GATEWAY_INFO` (code 10) answers `_rinfo()` from the dispatch loop; chdir/nice/env ship in the @@ -108,9 +168,9 @@ post-start remote_exec setup block are gone. Coordinator-gevent integration: `TrioHost.call_pending` (OneShot from a host task) backs makegateway / `SyncIOHandle.wait/kill` / -`Group.terminate` whenever the wait backend is not `thread`, so a gevent -app's management ops park only the calling greenlet. `wait=thread` -keeps the KI-deferred `portal.run` path. +`Group.terminate` whenever the group's wait backend is not `thread`, so a +gevent app's management ops park only the calling greenlet. The default +`thread` backend keeps the KI-deferred `portal.run` path. Still decided/standing: core stays trio-only (anyio/asyncio-core port rejected; asyncio apps use `execnet.aio`); eventlet stays dead; exec'd diff --git a/src/execnet/_execmodel.py b/src/execnet/_execmodel.py index adb43950..aee2ddcb 100644 --- a/src/execnet/_execmodel.py +++ b/src/execnet/_execmodel.py @@ -110,9 +110,7 @@ def resolve_profile(name: str) -> str: ) return replacement if name not in WORKER_PROFILES: - raise ValueError( - f"unknown profile {name!r} (known: {list(WORKER_PROFILES)})" - ) + raise ValueError(f"unknown profile {name!r} (known: {list(WORKER_PROFILES)})") return name diff --git a/src/execnet/_host.py b/src/execnet/_host.py index 56af3b4f..e57511d8 100644 --- a/src/execnet/_host.py +++ b/src/execnet/_host.py @@ -27,8 +27,12 @@ import sys import threading from types import TracebackType +from typing import TYPE_CHECKING from typing import Any +if TYPE_CHECKING: + from typing_extensions import Self + __all__ = ["Host", "check_not_in_event_loop", "default_host"] #: default cap on concurrent threadpool threads running receiver callbacks @@ -142,7 +146,7 @@ def close(self, timeout: float | None = 5.0) -> None: if trio_host is not None: trio_host.stop(timeout=timeout) - def __enter__(self) -> Host: + def __enter__(self) -> Self: return self def __exit__( diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py index c739eedc..70559bfa 100644 --- a/src/execnet/_multi.py +++ b/src/execnet/_multi.py @@ -167,9 +167,7 @@ def set_execmodel( DeprecationWarning, stacklevel=2, ) - self.set_profile( - execmodel if remote_execmodel is None else remote_execmodel - ) + self.set_profile(execmodel if remote_execmodel is None else remote_execmodel) def __repr__(self) -> str: idgateways = [gw.id for gw in self] @@ -273,9 +271,7 @@ def _cleanup_atexit(self) -> None: self.terminate(timeout=1.0) if self._async_group is not None: with suppress(Exception): - self._host._ensure_started().call_sync( - self._async_group.shutdown.set - ) + self._host._ensure_started().call_sync(self._async_group.shutdown.set) self._async_group = None def terminate(self, timeout: float | None = None) -> None: diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index b1fccbf5..735ee842 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -42,8 +42,8 @@ from ._errors import HostNotFound from ._errors import RemoteError from ._errors import TimeoutError -from ._execmodel import resolve_profile from ._exec_source import normalize_exec_source +from ._execmodel import resolve_profile from ._message import FrameDecoder from ._message import Message from ._message import gateway_info diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index ed5ee8f0..567851b0 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -32,9 +32,12 @@ from ._errors import RemoteError from ._execmodel import ExecModel from ._execmodel import get_execmodel +from ._host import DEFAULT_CALLBACK_THREADS from ._message import FrameDecoder from ._message import Message from ._message import gateway_info +from ._portal import LoopPortal +from ._portal import OneShot from ._serialize import dumps_internal from ._serialize import loads_internal from ._trace import trace @@ -46,9 +49,6 @@ from ._trio_gateway import open_popen_process from ._trio_gateway import read_handshake_ack from ._trio_gateway import ssh_transport_args -from ._host import DEFAULT_CALLBACK_THREADS -from ._portal import LoopPortal -from ._portal import OneShot #: bound on how long the endmarker callback may run during host shutdown CONSUMER_ENDMARKER_GRACE = 10.0 diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 0146c00b..9f824412 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -11,12 +11,13 @@ import trio +from ._boundary import Mailbox +from ._boundary import WaitBackend from ._errors import geterrortext from ._execmodel import get_execmodel from ._gateway_base import WorkerGateway from ._serialize import loads_internal from ._trace import trace -from ._boundary import Mailbox if TYPE_CHECKING: from . import _trio_host @@ -426,7 +427,10 @@ def kill(self) -> None: def _build_worker_gateway( - host: _trio_host.TrioHost, id: str, model: ExecModel, wait: str = "thread" + host: _trio_host.TrioHost, + id: str, + model: ExecModel, + wait: WaitBackend = "thread", ) -> tuple[WorkerGateway, TrioWorkerExec]: """Construct the WorkerGateway + exec pump/strategy (no IO yet).""" trace(f"creating workergateway on trio id={id!r}") @@ -450,7 +454,11 @@ def _build_worker_gateway( def _run_worker( - host: _trio_host.TrioHost, io: Any, id: str, model: ExecModel, wait: str = "thread" + host: _trio_host.TrioHost, + io: Any, + id: str, + model: ExecModel, + wait: WaitBackend = "thread", ) -> None: """Attach ``io`` as the gateway session and serve until shutdown.""" gateway, trio_exec = _build_worker_gateway(host, id, model, wait) @@ -528,7 +536,9 @@ async def main() -> None: os._exit(0) -def serve_popen_trio(id: str, profile: str = "thread", wait: str = "thread") -> None: +def serve_popen_trio( + id: str, profile: str = "thread", wait: WaitBackend = "thread" +) -> None: """Serve a WorkerGateway over the stdio pipes (popen / ssh worker).""" from . import _trio_host @@ -546,7 +556,7 @@ def serve_popen_trio(id: str, profile: str = "thread", wait: str = "thread") -> def serve_socket_trio( - id: str, profile: str, socket_fd: int, wait: str = "thread" + id: str, profile: str, socket_fd: int, wait: WaitBackend = "thread" ) -> None: """Serve a WorkerGateway over an inherited socket fd. @@ -641,7 +651,7 @@ def _main() -> None: # "execmodel" is the pre-3.0 spelling; accept both so a version-skewed # coordinator still connects. profile = config.get("profile") or config["execmodel"] - wait = config.get("wait", "thread") + wait: WaitBackend = config.get("wait", "thread") if profile == "trio": # pure-async profile: one thread, the loop owns the main thread if ns.socket_fd is not None: diff --git a/src/execnet/aio.py b/src/execnet/aio.py index 3e3deee6..fb4f6c2a 100644 --- a/src/execnet/aio.py +++ b/src/execnet/aio.py @@ -386,7 +386,7 @@ async def _start_host(host: Host) -> Any: def start() -> None: try: trio_host = host._ensure_started() - except BaseException as exc: # noqa: BLE001 + except BaseException as exc: loop.call_soon_threadsafe(_set_future_error, future, exc) else: loop.call_soon_threadsafe(_set_future_result, future, trio_host) diff --git a/src/execnet/gateway_base.py b/src/execnet/gateway_base.py index a82a1c0f..40eea46c 100644 --- a/src/execnet/gateway_base.py +++ b/src/execnet/gateway_base.py @@ -34,7 +34,6 @@ "LoadError": "._errors", # execution model presets "ExecModel": "._execmodel", - "EXECMODEL_PROFILES": "._execmodel", "get_execmodel": "._execmodel", # wire protocol "WriteIO": "._message", diff --git a/testing/conftest.py b/testing/conftest.py index a9797ada..29231d27 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -215,7 +215,8 @@ def gw( @pytest.fixture(params=["thread"], scope="session") def profile(request: pytest.FixtureRequest) -> str: """The worker profile gateways in this test run are created with.""" - return request.param + param: str = request.param + return param @pytest.fixture(scope="session") diff --git a/testing/test_aio.py b/testing/test_aio.py index 8c51ecf2..688643f3 100644 --- a/testing/test_aio.py +++ b/testing/test_aio.py @@ -118,7 +118,7 @@ async def main() -> None: def test_cancelled_receive_does_not_consume_an_item() -> None: # The bridge cancels the host-side receive, so the item stays queued - # instead of being consumed and dropped -- asyncio.timeout around a + # instead of being consumed and dropped -- an asyncio timeout around a # receive must behave like passing timeout=. async def main() -> None: async with execnet.aio.open_gateway() as gateway: @@ -130,8 +130,7 @@ async def main() -> None: """ ) with pytest.raises(asyncio.TimeoutError): - async with asyncio.timeout(0.05): - await channel.receive() + await asyncio.wait_for(channel.receive(), 0.05) # release the worker; nothing was consumed by the cancelled wait await channel.send("go") assert [await channel.receive() for _ in range(3)] == [0, 1, 2] @@ -144,9 +143,7 @@ def test_cancelled_send_still_arrives() -> None: # the wire, rather than a frame half-written to the peer. async def main() -> None: async with execnet.aio.open_gateway() as gateway: - channel = await gateway.remote_exec( - "channel.send(channel.receive() * 2)" - ) + channel = await gateway.remote_exec("channel.send(channel.receive() * 2)") task = asyncio.ensure_future(channel.send(21)) await asyncio.sleep(0) task.cancel() diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py index 2ef456ef..3f624d16 100644 --- a/testing/test_execmodel_trio.py +++ b/testing/test_execmodel_trio.py @@ -134,3 +134,22 @@ async def main() -> None: assert await channel.receive() == 42 trio_lib.run(main) + + +def test_async_coordinator_defaults_to_a_thread_worker() -> None: + # An async coordinator does not imply an async worker: the worker's + # shape is its own choice, so the default stays the thread profile. + async def main() -> None: + async with execnet.trio.AsyncGroup() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec( + "import threading;" + " channel.send(threading.current_thread() is" + " threading.main_thread())" + ) + with trio_lib.fail_after(TESTTIMEOUT): + # a sync source at all proves this is not the trio profile, + # which rejects them + assert await channel.receive() is True + + trio_lib.run(main) diff --git a/testing/test_gateway.py b/testing/test_gateway.py index e28a98eb..3c6f2fe1 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -17,7 +17,6 @@ import execnet from execnet import Gateway -from execnet import _execmodel from execnet import _trace TESTTIMEOUT = 10.0 # seconds diff --git a/testing/test_host.py b/testing/test_host.py index 686c022f..f5678a31 100644 --- a/testing/test_host.py +++ b/testing/test_host.py @@ -48,7 +48,6 @@ def test_many_groups_run_one_thread(self) -> None: def test_explicit_host_is_isolated_and_closes(self) -> None: host = Host(name="execnet-host-isolated") - assert not host.running group = execnet.Group(host=host) assert group.host is host assert group.host is not default_host() @@ -56,7 +55,6 @@ def test_explicit_host_is_isolated_and_closes(self) -> None: gateway = group.makegateway("popen") channel = gateway.remote_exec("channel.send(6 * 7)") assert channel.receive(TESTTIMEOUT) == 42 - assert host.running assert "execnet-host-isolated" in host_thread_names() finally: group.terminate(timeout=5.0) @@ -78,9 +76,7 @@ def test_starting_is_lazy(self) -> None: assert not host.running assert "execnet-host-lazy" not in host_thread_names() - @pytest.mark.skipif( - not hasattr(os, "fork"), reason="requires os.fork" - ) + @pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork") def test_forked_child_gets_a_fresh_default_host(self) -> None: # the child inherits a Host object whose thread does not exist # there, so the first use must build a new one @@ -114,14 +110,14 @@ class TestEventLoopGuard: def test_makegateway_inside_asyncio_raises(self) -> None: async def main() -> None: - with pytest.raises(RuntimeError, match="execnet.aio"): + with pytest.raises(RuntimeError, match=r"execnet\.aio"): execnet.Group().makegateway("popen") asyncio.run(main()) def test_makegateway_inside_trio_raises(self) -> None: async def main() -> None: - with pytest.raises(RuntimeError, match="execnet.trio"): + with pytest.raises(RuntimeError, match=r"execnet\.trio"): execnet.Group().makegateway("popen") trio.run(main) @@ -133,11 +129,11 @@ def test_channel_receive_inside_asyncio_raises(self) -> None: channel = gateway.remote_exec("channel.send(1)") async def main() -> None: - with pytest.raises(RuntimeError, match="execnet.aio"): + with pytest.raises(RuntimeError, match=r"execnet\.aio"): channel.receive(TESTTIMEOUT) - with pytest.raises(RuntimeError, match="execnet.aio"): + with pytest.raises(RuntimeError, match=r"execnet\.aio"): channel.send(1) - with pytest.raises(RuntimeError, match="execnet.aio"): + with pytest.raises(RuntimeError, match=r"execnet\.aio"): channel.waitclose(TESTTIMEOUT) asyncio.run(main()) From 320c89ef7ad93986203c6a3500c46879c516bde2 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 30 Jul 2026 07:16:55 +0200 Subject: [PATCH 65/91] ci: run the pytest-xdist test suite against this execnet Our own suite runs xdist as a *tool*, which exercises none of the paths xdist actually drives: crashed-worker replacement, the main-thread worker profile, report and warning serialization. Adding a job that runs xdist's own suite against the execnet built from the branch found 16 real regressions on the first run (baseline: released execnet 2.1.2 gives 220 passed, 0 failed). Both root causes are fixed here. makegateway wrote the *normalized* profile back onto the caller's XSpec. xdist reuses one spec object across gateways and re-reads spec.execmodel to decide whether it still needs its "execmodel=main_thread_only//" prefix, so after normalization it prefixed a second time and built a spec with a duplicate key -- every crashed-worker-replacement test failed. resolve_profile still validates and warns, but its result is no longer assigned to a caller-owned spec; effective_profile maps silently where the value is consumed. Filling in a missing spec value is idempotent and fine; rewriting one the caller set is not. execnet.dumps no longer warns at all. xdist reaches it once per warning a *user's* test raises, from inside pytest's warning-recording hook, so the warning landed in that user's warnings summary attributed to their test -- about a probe only xdist can port. Warning once per process stopped the livelock but not the noise. One expected failure remains, deselected via a documented list: test_remote_inner_argv asserts sys.argv == ["-c"] and documents the old `python -c` bootstrap, which no-source-shipping deliberately replaced. That needs an xdist PR. Co-Authored-By: Claude Opus 5 --- .github/workflows/test.yml | 70 ++++++++++++++++++++++++++++++++ .github/xdist-known-failures.txt | 13 ++++++ CHANGELOG.rst | 17 +++++--- handoff-phase-c-worker-axes.md | 27 ++++++++++++ src/execnet/__init__.py | 32 +++++---------- src/execnet/_execmodel.py | 25 +++++++++++- src/execnet/_multi.py | 4 +- src/execnet/_provision.py | 12 ++++-- src/execnet/_trio_gateway.py | 2 +- src/execnet/_trio_worker.py | 4 +- testing/test_execmodel_trio.py | 28 +++++++++++++ testing/test_namespaces.py | 22 +++++----- 12 files changed, 210 insertions(+), 46 deletions(-) create mode 100644 .github/xdist-known-failures.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ad25a355..b600032e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,6 +10,13 @@ on: branches: - "main" + workflow_dispatch: + inputs: + xdist-ref: + description: "pytest-xdist ref to test against" + default: "main" + type: string + # Cancel running jobs for the same workflow and branch. concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -64,3 +71,66 @@ jobs: shell: bash run: | tox run -e py --installpkg `find dist/*.tar.gz` + + # pytest-xdist drives execnet's local parallel testing: popen gateways, + # the main-thread worker profile, crashed-worker replacement, and the + # report/warning serialization. Our own suite runs xdist as a *tool*, + # which does not exercise any of that -- so run xdist's own test suite + # against the execnet built from this branch. + # + # To reproduce locally: + # git clone --depth 1 https://github.com/pytest-dev/pytest-xdist + # python -m venv .xdist-venv + # .xdist-venv/bin/pip install ./pytest-xdist[testing] pytest . + # cd pytest-xdist && ../.xdist-venv/bin/python -m pytest + xdist: + + needs: [package] + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v7 + with: + path: execnet + + - uses: actions/checkout@v7 + with: + repository: pytest-dev/pytest-xdist + ref: ${{ inputs.xdist-ref || 'main' }} + path: pytest-xdist + + - name: Download Package + uses: actions/download-artifact@v8 + with: + name: Packages + path: dist + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.13" + + - name: Install pytest-xdist and this execnet + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install "./pytest-xdist[testing]" pytest + # after xdist, so its execnet>=2.1 requirement does not win + python -m pip install --force-reinstall `find dist/*.tar.gz` + python -c "import execnet; print('execnet', execnet.__version__)" + + - name: Run the pytest-xdist test suite + shell: bash + working-directory: pytest-xdist + run: | + deselect=() + # `|| [ -n "$line" ]` so a file without a trailing newline still + # contributes its last entry + while read -r line || [ -n "$line" ]; do + line="${line%%#*}" + line="$(echo "$line" | xargs)" + [ -n "$line" ] && deselect+=(--deselect "$line") + done < ../execnet/.github/xdist-known-failures.txt + printf 'deselecting %d known failure(s)\n' $(( ${#deselect[@]} / 2 )) + python -m pytest -ra "${deselect[@]}" diff --git a/.github/xdist-known-failures.txt b/.github/xdist-known-failures.txt new file mode 100644 index 00000000..6dd34d86 --- /dev/null +++ b/.github/xdist-known-failures.txt @@ -0,0 +1,13 @@ +# pytest-xdist tests that are expected to fail against this execnet, with +# the reason. Everything not listed here must pass -- the whole point of +# the `xdist` CI job is that this list stays short and justified. +# +# Blank lines and `#` comments are ignored; every other line is one pytest +# node id, passed to `pytest --deselect`. + +# Asserts `sys.argv == ["-c"]`, and its own docstring says it is +# documenting "the behavior due to execnet using `python -c`". execnet no +# longer ships source over the wire: workers are launched as +# `python -m execnet._trio_worker `, so sys.argv legitimately +# differs. Needs an xdist-side update, not an execnet fix. +testing/test_remote.py::test_remote_inner_argv diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 93a52284..49dd647b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,12 @@ Pass ``execnet.Host()`` as ``Group(host=...)`` (or ``AsyncGroup(host=...)``) for an isolated loop with deterministic teardown -- ``Host`` is a context manager and joins its thread on exit, where the shared one stops at interpreter exit. +* A spec's ``profile=``/``execmodel=`` value is no longer rewritten in place when it + names a deprecated profile. pytest-xdist reuses one ``XSpec`` across gateways and + re-reads ``spec.execmodel`` to decide whether it still needs prefixing, so normalizing + the value it set made the second use build a spec with a duplicate key -- which broke + crashed-worker replacement. The spec keeps what the caller spelled; the mapping happens + where the value is consumed. * The ``execmodel=`` spec key is now ``profile=``; ``execmodel=`` remains an accepted alias. It always selected the *worker* profile -- where exec'd code runs relative to the worker's protocol loop -- while the local execution model it was named after no @@ -48,11 +54,12 @@ ``start()``/``aclose()`` from application lifespan hooks instead of ``async with``. ``open_popen_gateway`` is renamed ``open_gateway`` on both async namespaces, since it always accepted any spec. -* ``execnet.dumps``, the temporary pytest-xdist compatibility shim, now warns once per - process rather than on every access. xdist reaches it from ``serialize_warning_message``, - i.e. from inside pytest's warning-recording hook, so a single ``DeprecationWarning`` - raised in a worker made recording that warning record another, unbounded, wedging the - run. +* ``execnet.dumps``, the temporary pytest-xdist compatibility shim, no longer warns. + xdist reaches it from ``serialize_warning_message`` -- from inside pytest's + warning-recording hook, once per warning a *user's* test raises. Warning there put a + spurious execnet ``DeprecationWarning`` in that user's warnings summary, attributed to + their test, about a probe only xdist can port; and warning on every access made + recording one warning record another, unbounded, wedging the run. * ``Gateway.remote_init_threads()`` raises a ``DeprecationWarning`` instead of printing to stdout. It has been a no-operation since execnet 1.2. diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index ba289844..ca036254 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -69,6 +69,33 @@ aio cancellation contracts in `testing/test_aio.py`. The `execmodel` fixture parametrization collapsed to a single `profile` fixture, so the suite is ~540 items rather than ~765. +### D.3 done: the xdist suite runs in CI (2026-07-30) + +The `xdist` job in `.github/workflows/test.yml` runs pytest-xdist's own +test suite against the execnet built from the branch. **Doing this for +the first time found 16 real regressions** that our suite could not see +(it runs xdist as a *tool*, which exercises none of the crash-replacement +or report-serialization paths). Baseline: released execnet 2.1.2 gives +220 passed / 0 failed. Two root causes, both fixed: + +1. `makegateway` wrote the *normalized* profile back onto the caller's + XSpec. xdist reuses one spec object and re-reads `spec.execmodel` to + decide whether it still needs the `execmodel=main_thread_only//` + prefix, so after normalization it prefixed a second time and built + `execmodel=...//execmodel=...//popen` -> `ValueError: duplicate key`. + Every crashed-worker-replacement test failed. Fix: `resolve_profile` + validates and warns but callers no longer assign its result to a + caller-owned spec; `effective_profile` (no warning) maps at the + consumption points. **Rule: filling in a missing spec value is + idempotent and fine; rewriting one the caller set is not.** +2. the `execnet.dumps` deprecation warning (see above). + +Remaining known failure, deselected via `.github/xdist-known-failures.txt` +(the job is green otherwise): `test_remote_inner_argv` asserts +`sys.argv == ["-c"]` and documents "the behavior due to execnet using +`python -c`" -- which the no-source-shipping worker launch deliberately +changed. That one needs an xdist PR. + Still open from the review, deliberately not done: the async surfaces have no `remote_status()`, no `MultiChannel`, no group iteration, and no `RSync`. `AsyncGroup.makegateway` defaults workers to the `thread` diff --git a/src/execnet/__init__.py b/src/execnet/__init__.py index 2ed1293f..e88baf6f 100644 --- a/src/execnet/__init__.py +++ b/src/execnet/__init__.py @@ -89,21 +89,21 @@ #: serializability with ``try: execnet.dumps(x) / except execnet.DumpError`` #: before shipping warning args and report attrs. The standalone serializer #: is internal and :func:`can_send` replaces that probe, but dropping the name -#: outright breaks every released xdist, so it stays reachable -- warning, and -#: deliberately absent from ``__all__``. +#: outright breaks every released xdist, so it stays reachable -- deliberately +#: absent from ``__all__`` and from ``dir()``. +#: +#: It does NOT warn, on purpose. xdist reaches it from +#: ``serialize_warning_message``, i.e. from inside pytest's warning-recording +#: hook and once per warning a *user's* test raises. A warning there is +#: attributed to that test, in a run the user cannot change the outcome of +#: (porting the probe is xdist's call, not theirs) -- and warning on every +#: access made recording one warning record another, unbounded, wedging the +#: run. The deprecation lives in the changelog and in the xdist issue. #: #: FOLLOW-UP (after the execnet release): port xdist to ``execnet.can_send``, #: then delete this and its test. Nothing else may be added here. _XDIST_COMPAT = ("dumps",) -#: Warn about the shim at most once per process. xdist reaches it from -#: ``serialize_warning_message``, i.e. from inside pytest's -#: warning-recording hook: warning every time makes recording one warning -#: record another, which records another -- an unbounded loop that wedges -#: the worker. Once is also simply the right amount of noise for a name -#: whose caller is a library, not the user. -_xdist_compat_warned: set[str] = set() - def __getattr__(name: str) -> Any: if name in _LAZY_MODULES: @@ -113,17 +113,5 @@ def __getattr__(name: str) -> Any: if name in _XDIST_COMPAT: import importlib - if name not in _xdist_compat_warned: - import warnings - - _xdist_compat_warned.add(name) - warnings.warn( - f"execnet.{name} is a temporary pytest-xdist compatibility shim" - " and will be removed; the standalone serializer is not public." - " Use execnet.can_send(obj) to test whether a value can cross a" - " channel.", - DeprecationWarning, - stacklevel=2, - ) return getattr(importlib.import_module("._serialize", __name__), name) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/execnet/_execmodel.py b/src/execnet/_execmodel.py index aee2ddcb..7c25448c 100644 --- a/src/execnet/_execmodel.py +++ b/src/execnet/_execmodel.py @@ -96,8 +96,31 @@ def Event(self) -> threading.Event: DEPRECATED_PROFILES = {"main_thread_only": "thread"} +def effective_profile(name: str) -> str: + """Validate a profile name and map deprecated spellings, silently. + + For the places that *consume* a profile (worker config, strategy + lookup). :func:`resolve_profile` is the one that warns, and is called + once where the value enters. + """ + replacement = DEPRECATED_PROFILES.get(name) + if replacement is not None: + return replacement + if name not in WORKER_PROFILES: + raise ValueError(f"unknown profile {name!r} (known: {list(WORKER_PROFILES)})") + return name + + def resolve_profile(name: str) -> str: - """Validate a ``profile=`` value, mapping deprecated spellings.""" + """Validate a ``profile=`` value, warning about deprecated spellings. + + Callers that hold a caller-supplied :class:`~execnet._xspec.XSpec` must + *not* write the result back onto it: pytest-xdist reuses a spec object + across gateways and re-reads ``spec.execmodel`` to decide whether it + still needs prefixing, so normalizing the value it set makes the second + use build a spec with a duplicate key. Validate here, and map with + :func:`effective_profile` where the value is used. + """ replacement = DEPRECATED_PROFILES.get(name) if replacement is not None: warnings.warn( diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py index 70559bfa..a2b36505 100644 --- a/src/execnet/_multi.py +++ b/src/execnet/_multi.py @@ -229,9 +229,11 @@ def makegateway(self, spec: XSpec | str | None = None) -> Gateway: spec = XSpec(spec) self.allocate_id(spec) if spec.profile is None: + # filling in a missing value is idempotent; rewriting one the + # caller set is not -- see resolve_profile's docstring spec.profile = self._profile else: - spec.profile = resolve_profile(spec.profile) + resolve_profile(spec.profile) from . import _trio_host if not (spec.socket or spec.via or spec.ssh or spec.vagrant_ssh or spec.popen): diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index 67b02d17..b1892dcf 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -164,15 +164,21 @@ def worker_cli_arg(spec: Any) -> str: """ import execnet + from ._execmodel import effective_profile + + # the spec keeps whatever the caller spelled; the worker gets what it + # actually has a strategy for. A spec that never went through a Group + # has no profile at all -- the default applies, as it would there. + profile = effective_profile(spec.profile or "thread") config: dict[str, Any] = { "id": f"{spec.id}-worker", - "profile": spec.profile, + "profile": profile, # pre-3.0 spelling, same value: a worker from an older execnet # reads this one. Drop with the XSpec alias. - "execmodel": spec.profile, + "execmodel": profile, # derived, not configurable: the gevent profile parks its greenlets # on gevent wakeners, every other profile is thread-shaped. - "wait": "gevent" if spec.profile == "gevent" else "thread", + "wait": "gevent" if profile == "gevent" else "thread", "coordinator_version": execnet.__version__, } # Startup setup applied by the worker before serving (never through diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 735ee842..9f3f06fe 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -915,7 +915,7 @@ async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: # exec'd async sources as tasks. spec.profile = "thread" else: - spec.profile = resolve_profile(spec.profile) + resolve_profile(spec.profile) if spec.id is None: spec.id = "gw%d" % len(self._gateways) process: trio.Process | None = None diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 9f824412..b20be7b6 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -14,6 +14,7 @@ from ._boundary import Mailbox from ._boundary import WaitBackend from ._errors import geterrortext +from ._execmodel import effective_profile from ._execmodel import get_execmodel from ._gateway_base import WorkerGateway from ._serialize import loads_internal @@ -439,7 +440,7 @@ def _build_worker_gateway( gateway._wait_backend = wait try: - strategy_factory = WORKER_EXEC_STRATEGIES[model.backend] + strategy_factory = WORKER_EXEC_STRATEGIES[effective_profile(model.backend)] except KeyError: raise ValueError( f"profile {model.backend!r} has no worker exec strategy " @@ -652,6 +653,7 @@ def _main() -> None: # coordinator still connects. profile = config.get("profile") or config["execmodel"] wait: WaitBackend = config.get("wait", "thread") + profile = effective_profile(profile) if profile == "trio": # pure-async profile: one thread, the loop owns the main thread if ns.socket_fd is not None: diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py index 3f624d16..8fd121d8 100644 --- a/testing/test_execmodel_trio.py +++ b/testing/test_execmodel_trio.py @@ -153,3 +153,31 @@ async def main() -> None: assert await channel.receive() is True trio_lib.run(main) + + +def test_makegateway_does_not_rewrite_the_callers_profile( + makegateway: Callable[[str], Gateway], +) -> None: + # pytest-xdist reuses one XSpec across gateways and re-reads + # spec.execmodel to decide whether it still needs its + # "execmodel=main_thread_only//" prefix. Normalizing the value it set + # made the second use build a spec with a duplicate key, which broke + # crashed-worker replacement. Filling in a *missing* value is fine; + # rewriting one the caller set is not. + spec = execnet.XSpec("execmodel=main_thread_only//popen") + with pytest.warns(DeprecationWarning, match="main_thread_only"): + gw = makegateway(spec) # type: ignore[arg-type] + assert spec.execmodel == "main_thread_only" + assert spec.profile == "main_thread_only" + # ... while the worker still gets a profile it has a strategy for + assert gw.remote_status().profile == "thread" + + +def test_makegateway_fills_in_a_missing_profile() -> None: + group = execnet.Group() + try: + spec = execnet.XSpec("popen") + group.makegateway(spec) + assert spec.profile == "thread" + finally: + group.terminate(timeout=5.0) diff --git a/testing/test_namespaces.py b/testing/test_namespaces.py index 856268ea..63a707b3 100644 --- a/testing/test_namespaces.py +++ b/testing/test_namespaces.py @@ -109,24 +109,22 @@ def test_dumps_is_a_temporary_xdist_shim() -> None: from execnet import _serialize assert execnet._XDIST_COMPAT == ("dumps",) - execnet._xdist_compat_warned.discard("dumps") - with pytest.warns(DeprecationWarning, match="temporary pytest-xdist"): - assert execnet.dumps is _serialize.dumps + assert execnet.dumps is _serialize.dumps # reachable, but never advertised assert "dumps" not in execnet.__all__ assert "dumps" not in dir(execnet) -def test_dumps_shim_warns_only_once() -> None: - # xdist reaches this from serialize_warning_message, i.e. from inside - # pytest's warning-recording hook: warning every time makes recording - # one warning record another, ad infinitum, and the worker wedges. - execnet._xdist_compat_warned.discard("dumps") - with pytest.warns(DeprecationWarning, match="temporary pytest-xdist"): - execnet.dumps # noqa: B018 +def test_dumps_shim_does_not_warn() -> None: + # xdist reaches this from serialize_warning_message -- once per warning + # a *user's* test raises, from inside pytest's warning-recording hook. + # A warning there is attributed to that test, which cannot act on it, + # and warning on every access made recording one warning record + # another, unbounded, wedging the run. with warnings.catch_warnings(): - warnings.simplefilter("error", DeprecationWarning) - execnet.dumps # noqa: B018 + warnings.simplefilter("error") + for _ in range(3): + execnet.dumps # noqa: B018 def test_boundary_kit_is_private() -> None: From 17e7c7a95791700793e4aad93da27e5fda872b69 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 30 Jul 2026 07:35:55 +0200 Subject: [PATCH 66/91] ci: split the xdist job into a pinned release and a floating tip The single floating job could go red for reasons that are xdist's to fix, which is not a reason to block an execnet PR -- but a floating job is also the only thing that catches drift early. So run both. The blocking target is pinned: xdist v3.8.0 with pytest<9, a combination checked to be green against *released* execnet 2.1.2 (196 passed, 0 failed). The pytest pin is load-bearing -- xdist 3.8.0 fails two of its own tests under pytest 9 regardless of which execnet is installed, so without it the job would have started life red and meant nothing. Bump the ref and the pin together, after re-checking the new pair against released execnet. The floating target is continue-on-error and leaves `ref` empty so it follows xdist's default branch rather than hardcoding a name -- which also fixes the previous job: that branch is `master`, not `main`, so the checkout would have failed outright. Both verified locally against this execnet: release 195 passed + 1 deselected, default branch 219 passed + 1 deselected. Co-Authored-By: Claude Opus 5 --- .github/workflows/test.yml | 46 +++++++++++++++++++++++++------- .github/xdist-known-failures.txt | 10 +++++-- handoff-phase-c-worker-axes.md | 30 ++++++++++++++++----- 3 files changed, 67 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b600032e..c28d57ba 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,11 +11,6 @@ on: - "main" workflow_dispatch: - inputs: - xdist-ref: - description: "pytest-xdist ref to test against" - default: "main" - type: string # Cancel running jobs for the same workflow and branch. concurrency: @@ -78,10 +73,24 @@ jobs: # which does not exercise any of that -- so run xdist's own test suite # against the execnet built from this branch. # + # Two targets: + # + # * `release` is a pinned combination known to be green against the last + # *released* execnet, so a failure there means we broke something. It + # blocks. Bump `ref`/`pytest` together, and only after checking the new + # pair is green against released execnet -- otherwise the signal is + # gone. (xdist 3.8.0 needs pytest<9: with pytest 9 two of its tests + # fail against released execnet too.) + # * `default-branch` is early warning for drift on both sides. It is + # allowed to fail, because it can go red for reasons that are xdist's + # to fix. An empty `ref` follows whatever xdist's default branch is + # (`master` today) rather than hardcoding a name that could change. + # # To reproduce locally: - # git clone --depth 1 https://github.com/pytest-dev/pytest-xdist + # git clone https://github.com/pytest-dev/pytest-xdist + # git -C pytest-xdist checkout v3.8.0 # or stay on the default branch # python -m venv .xdist-venv - # .xdist-venv/bin/pip install ./pytest-xdist[testing] pytest . + # .xdist-venv/bin/pip install ./pytest-xdist[testing] "pytest<9" . # cd pytest-xdist && ../.xdist-venv/bin/python -m pytest xdist: @@ -89,6 +98,23 @@ jobs: runs-on: ubuntu-latest + name: xdist (${{ matrix.name }}) + + continue-on-error: ${{ matrix.experimental }} + + strategy: + fail-fast: false + matrix: + include: + - name: release + ref: "v3.8.0" + pytest: "pytest<9" + experimental: false + - name: default-branch + ref: "" + pytest: "pytest" + experimental: true + steps: - uses: actions/checkout@v7 with: @@ -97,7 +123,7 @@ jobs: - uses: actions/checkout@v7 with: repository: pytest-dev/pytest-xdist - ref: ${{ inputs.xdist-ref || 'main' }} + ref: ${{ matrix.ref }} path: pytest-xdist - name: Download Package @@ -115,10 +141,10 @@ jobs: shell: bash run: | python -m pip install --upgrade pip - python -m pip install "./pytest-xdist[testing]" pytest + python -m pip install "./pytest-xdist[testing]" "${{ matrix.pytest }}" # after xdist, so its execnet>=2.1 requirement does not win python -m pip install --force-reinstall `find dist/*.tar.gz` - python -c "import execnet; print('execnet', execnet.__version__)" + python -c "import execnet, xdist; print('execnet', execnet.__version__)" - name: Run the pytest-xdist test suite shell: bash diff --git a/.github/xdist-known-failures.txt b/.github/xdist-known-failures.txt index 6dd34d86..c3ded559 100644 --- a/.github/xdist-known-failures.txt +++ b/.github/xdist-known-failures.txt @@ -1,9 +1,15 @@ # pytest-xdist tests that are expected to fail against this execnet, with # the reason. Everything not listed here must pass -- the whole point of -# the `xdist` CI job is that this list stays short and justified. +# the `xdist (release)` CI job is that this list stays short and justified. # # Blank lines and `#` comments are ignored; every other line is one pytest -# node id, passed to `pytest --deselect`. +# node id, passed to `pytest --deselect`. The same list is used for both +# the pinned release and the `main` target; a node id that does not exist +# in one of them is silently ignored, so an entry may name a test that +# only one version has. +# +# Do not add an entry to make CI green. Add one only for a test that is +# asserting behaviour execnet deliberately changed, and say so. # Asserts `sys.argv == ["-c"]`, and its own docstring says it is # documenting "the behavior due to execnet using `python -c`". execnet no diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index ca036254..98df1166 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -72,11 +72,25 @@ suite is ~540 items rather than ~765. ### D.3 done: the xdist suite runs in CI (2026-07-30) The `xdist` job in `.github/workflows/test.yml` runs pytest-xdist's own -test suite against the execnet built from the branch. **Doing this for -the first time found 16 real regressions** that our suite could not see -(it runs xdist as a *tool*, which exercises none of the crash-replacement -or report-serialization paths). Baseline: released execnet 2.1.2 gives -220 passed / 0 failed. Two root causes, both fixed: +test suite against the execnet built from the branch, in two variants: + +- **`release`** — pinned `v3.8.0` + `pytest<9`, a combination verified + green against *released* execnet 2.1.2 (196 passed / 0 failed). This + one blocks: a failure means we broke something. Bump the ref and the + pytest pin together, and only after re-checking the new pair against + released execnet — xdist 3.8.0 with pytest 9 fails two of its own tests + regardless of execnet, which is why the pin exists. +- **`default-branch`** — floating, `continue-on-error`. Early warning for + drift on either side, without letting an xdist-side breakage block + execnet PRs. The `ref` is left empty so it follows xdist's default + branch, which is `master`, not `main` — worth knowing before hardcoding + a name. + +**Doing this for the first time found 16 real regressions** that our suite +could not see (it runs xdist as a *tool*, which exercises none of the +crash-replacement or report-serialization paths). Baseline for xdist's +default branch: released execnet 2.1.2 gives 220 passed / 0 failed. Two +root causes, both fixed: 1. `makegateway` wrote the *normalized* profile back onto the caller's XSpec. xdist reuses one spec object and re-reads `spec.execmodel` to @@ -91,10 +105,12 @@ or report-serialization paths). Baseline: released execnet 2.1.2 gives 2. the `execnet.dumps` deprecation warning (see above). Remaining known failure, deselected via `.github/xdist-known-failures.txt` -(the job is green otherwise): `test_remote_inner_argv` asserts +(both jobs are green otherwise): `test_remote_inner_argv` asserts `sys.argv == ["-c"]` and documents "the behavior due to execnet using `python -c`" -- which the no-source-shipping worker launch deliberately -changed. That one needs an xdist PR. +changed. That one needs an xdist PR. The deselect list is shared by both +targets; pytest ignores a `--deselect` that matches nothing, so an entry +may name a test only one xdist version has. Still open from the review, deliberately not done: the async surfaces have no `remote_status()`, no `MultiChannel`, no group iteration, and no From 5710eed7e2d9274c6832146c03e794dbf59a7d7b Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 30 Jul 2026 10:33:32 +0200 Subject: [PATCH 67/91] feat!: add the execnet CLI and stop swallowing worker stdout The protocol used to *be* the worker's stdin/stdout, so the worker had to steal fd 0/1 back and point them at the null device -- which meant a remote print() went nowhere at all, and left no way to say "run the protocol somewhere else" from a command line. `execnet worker` names the transport instead: --protocol-stdio (default), --protocol-fd (an inherited socket, or a read,write pipe pair), --protocol-connect and --protocol-listen (unix:/path or host:port). The config gains --config-fd/--config-file next to --config, so a launcher can keep it out of argv. `execnet server` is the socketserver and `execnet info` reports version and capabilities as JSON, which provisioning will use in place of an `import execnet, trio` probe. The CLI is the launch contract: `python -m execnet worker ...` for a direct interpreter launch, `execnet worker ...` under `uv run`. execnet-socketserver stays as a deprecated alias. Behaviour change: a worker's stdio is now the user's. With a transport of its own the worker leaves fd 0/1/2 alone entirely; on the stdio transport it closes stdin and folds stdout onto stderr rather than discarding both. Remote print() output that used to vanish now reaches the coordinator, and --stdin/--stdout/--stderr override any of it. Co-Authored-By: Claude Opus 5 --- pyproject.toml | 4 +- src/execnet/__main__.py | 11 ++ src/execnet/_cli.py | 255 +++++++++++++++++++++++++++ src/execnet/_provision.py | 47 +++-- src/execnet/_socketserver.py | 38 ++-- src/execnet/_trio_gateway.py | 18 +- src/execnet/_trio_host.py | 11 +- src/execnet/_trio_worker.py | 331 ++++++++++++++++++++++++----------- testing/test_basics.py | 30 +++- testing/test_gateway.py | 2 +- testing/test_provision.py | 10 +- testing/test_xspec.py | 2 +- 12 files changed, 593 insertions(+), 166 deletions(-) create mode 100644 src/execnet/__main__.py create mode 100644 src/execnet/_cli.py diff --git a/pyproject.toml b/pyproject.toml index 68029e90..e321346c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,9 @@ classifiers = [ ] [project.scripts] -execnet-socketserver = "execnet._socketserver:main" +execnet = "execnet._cli:main" +# deprecated alias for `execnet server` +execnet-socketserver = "execnet._cli:socketserver_main" [project.optional-dependencies] gevent = [ diff --git a/src/execnet/__main__.py b/src/execnet/__main__.py new file mode 100644 index 00000000..83640e8d --- /dev/null +++ b/src/execnet/__main__.py @@ -0,0 +1,11 @@ +"""``python -m execnet`` -- the same CLI as the ``execnet`` console script. + +Provisioning emits this form for a direct interpreter launch, where the +console script's location on the target is not knowable; under ``uv run`` +the bare ``execnet`` command resolves inside the provisioned environment. +""" + +from ._cli import main + +if __name__ == "__main__": + main() diff --git a/src/execnet/_cli.py b/src/execnet/_cli.py new file mode 100644 index 00000000..ebd86bbe --- /dev/null +++ b/src/execnet/_cli.py @@ -0,0 +1,255 @@ +"""The ``execnet`` command line. + +Three subcommands, reachable both as the ``execnet`` console script and as +``python -m execnet`` (the latter is what provisioning emits for a direct +interpreter launch, where the script's location is not knowable):: + + execnet worker ... # serve one gateway over a chosen transport + execnet server ... # accept gateway connections on a socket + execnet info # what this interpreter's execnet can do, as JSON + +``execnet worker`` is the launch contract between a coordinator and the +process it starts. Naming the protocol transport explicitly is what makes +ssh socket redirects and trampoline processes expressible: the protocol no +longer has to be the process's stdin/stdout, so a worker's stdio can belong +to the code it runs. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from typing import Any + +__all__ = ["main"] + +#: what the worker may do with each standard fd once the protocol has a +#: transport of its own. ``stderr`` for stdout points fd 1 at fd 2, which +#: keeps remote output visible without needing a second stream. +STDIN_DISPOSITIONS = ("inherit", "close", "devnull") +STDOUT_DISPOSITIONS = ("inherit", "devnull", "stderr") +STDERR_DISPOSITIONS = ("inherit", "devnull") + + +def _protocol_fd(value: str) -> tuple[int, ...]: + """``N`` (a bidirectional socket) or ``R,W`` (a pipe pair).""" + try: + fds = tuple(int(part) for part in value.split(",")) + except ValueError: + raise argparse.ArgumentTypeError( + f"expected FD or READFD,WRITEFD, got {value!r}" + ) from None + if len(fds) not in (1, 2): + raise argparse.ArgumentTypeError( + f"expected FD or READFD,WRITEFD, got {value!r}" + ) + return fds + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="execnet", + description="Serve and inspect execnet gateways.", + ) + sub = parser.add_subparsers(dest="command", required=True) + + worker = sub.add_parser( + "worker", + help="serve a single gateway to the coordinator that launched us", + description=( + "Serve one gateway over the given protocol transport. Normally" + " launched by a coordinator, not by hand." + ), + ) + transport = worker.add_mutually_exclusive_group() + transport.add_argument( + "--protocol-stdio", + dest="protocol", + action="store_const", + const=("stdio", None), + help="run the protocol over this process's stdin/stdout (default)", + ) + transport.add_argument( + "--protocol-fd", + metavar="FD[,FD]", + type=_protocol_fd, + help="run the protocol over an inherited socket fd, or pipe read,write pair", + ) + transport.add_argument( + "--protocol-connect", + metavar="ADDR", + help="dial out to ADDR (unix:/path or host:port) and serve there", + ) + transport.add_argument( + "--protocol-listen", + metavar="ADDR", + help="listen on ADDR (unix:/path or host:port) for one connection", + ) + + config = worker.add_mutually_exclusive_group() + config.add_argument("--config", metavar="JSON", help="worker config as JSON") + config.add_argument( + "--config-fd", + metavar="FD", + type=int, + help="read the worker config as JSON from FD until EOF", + ) + config.add_argument( + "--config-file", metavar="PATH", help="read the worker config as JSON from PATH" + ) + + worker.add_argument( + "--stdin", choices=STDIN_DISPOSITIONS, default=None, help="what to do with fd 0" + ) + worker.add_argument( + "--stdout", + choices=STDOUT_DISPOSITIONS, + default=None, + help="what to do with fd 1", + ) + worker.add_argument( + "--stderr", + choices=STDERR_DISPOSITIONS, + default=None, + help="what to do with fd 2", + ) + worker.set_defaults(func=_run_worker) + + server = sub.add_parser( + "server", + help="accept gateway connections on a socket", + description=( + "Listen for coordinator connections and hand each one to a fresh" + " worker subprocess. No code is executed in this process." + ), + ) + server.add_argument( + "hostport", + nargs="?", + default=":8888", + help="address to bind as HOST:PORT or :PORT (default: :8888)", + ) + server.add_argument( + "--once", + action="store_true", + help="serve a single connection and exit instead of looping", + ) + server.set_defaults(func=_run_server) + + info = sub.add_parser( + "info", + help="report this execnet's version and capabilities as JSON", + description=( + "Print a JSON object describing this interpreter's execnet." + " Coordinators use it to decide whether a target interpreter can" + " host a worker directly or has to be provisioned." + ), + ) + info.set_defaults(func=_run_info) + + return parser + + +def _load_config(ns: argparse.Namespace) -> dict[str, Any]: + """The worker config, from whichever source was named. + + ``--config-fd`` is what a coordinator should use for anything sensitive: + a config passed in argv is visible in ``ps`` to every user on the host, + and it carries ``env:`` values. + """ + if ns.config is not None: + raw = ns.config + elif ns.config_fd is not None: + with os.fdopen(ns.config_fd, "r", encoding="utf-8") as stream: + raw = stream.read() + elif ns.config_file is not None: + with open(ns.config_file, encoding="utf-8") as stream: + raw = stream.read() + else: + raise SystemExit("execnet worker: one of --config/--config-fd/--config-file") + config: dict[str, Any] = json.loads(raw) + return config + + +def _run_worker(ns: argparse.Namespace) -> None: + from . import _trio_worker + + if ns.protocol_fd is not None: + transport: _trio_worker.Transport = _trio_worker.FdTransport(ns.protocol_fd) + elif ns.protocol_connect is not None: + transport = _trio_worker.ConnectTransport(ns.protocol_connect) + elif ns.protocol_listen is not None: + transport = _trio_worker.ListenTransport(ns.protocol_listen) + else: + transport = _trio_worker.StdioTransport() + # The stdio transport owns fd 0/1, so it has to claim them before the + # config is read (--config-fd 0 would be the very fd we are moving). + transport.prepare() + config = _load_config(ns) + _trio_worker.serve_worker( + config, + transport, + stdin=ns.stdin, + stdout=ns.stdout, + stderr=ns.stderr, + ) + + +def _run_server(ns: argparse.Namespace) -> None: + import trio + + from . import _socketserver + + trio.run(_socketserver.serve, ns.hostport, ns.once) + + +def _run_info(ns: argparse.Namespace) -> None: + json.dump(interpreter_info(), sys.stdout) + sys.stdout.write("\n") + + +def interpreter_info() -> dict[str, Any]: + """What a coordinator needs to know before launching a worker here. + + Distinct from ``_message.gateway_info``, which answers the in-protocol + ``GATEWAY_INFO`` request on an established gateway; this one is what a + coordinator can learn *before* connecting. + """ + from ._version import version + + return { + "execnet": version, + "python": ".".join(str(part) for part in sys.version_info[:3]), + "executable": sys.executable, + "platform": sys.platform, + "protocols": _supported_protocols(), + } + + +def _supported_protocols() -> list[str]: + """Transports this platform can actually serve.""" + protocols = ["stdio", "listen", "connect"] + if hasattr(os, "dup"): + protocols.append("fd") + return sorted(protocols) + + +def main(argv: list[str] | None = None) -> None: + """Console entry point (``execnet``) and ``python -m execnet``.""" + parser = _build_parser() + ns = parser.parse_args(argv) + ns.func(ns) + + +def socketserver_main(argv: list[str] | None = None) -> None: + """Deprecated ``execnet-socketserver`` entry point; use ``execnet server``.""" + import warnings + + warnings.warn( + "execnet-socketserver is deprecated; use `execnet server` instead.", + DeprecationWarning, + stacklevel=2, + ) + main(["server", *(sys.argv[1:] if argv is None else argv)]) diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index b1892dcf..1e21a996 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -2,8 +2,9 @@ Non-same-interpreter Trio workers (foreign-python popen, later ssh/socket) are launched inside an environment that ``uv`` provisions with a matching execnet + -trio. The worker itself is always ``python -m execnet._trio_worker`` and does -the usual ``b"1"`` stdio handshake; only the launch prefix differs. +trio. The worker itself is always ``python -m execnet worker`` and does the +usual ``b"1"`` handshake on whichever protocol transport it was given; only +the launch prefix differs. Delivery of execnet into that environment is version-aware: @@ -158,8 +159,8 @@ def coordinator_requirement() -> str: def worker_cli_arg(spec: Any) -> str: """Single JSON CLI argument carrying the worker config (the whole 'spec thing'). - Passed to ``python -m execnet._trio_worker`` by every launcher (popen, uv, - ssh) so the worker config lives in one place rather than scattered positional + Passed to ``python -m execnet worker`` by every launcher (popen, uv, ssh) + so the worker config lives in one place rather than scattered positional args. """ import execnet @@ -192,18 +193,24 @@ def worker_cli_arg(spec: Any) -> str: return json.dumps(config) -def _worker_tokens(config: str) -> list[str]: - """``python -u -m execnet._trio_worker `` tokens. +def _worker_tokens(config: str | None, *protocol: str) -> list[str]: + """``python -u -m execnet worker`` tokens for a launch. The literal ``python`` token is resolved by uv (inside the provisioned - environment) or the remote shell. + environment) or the remote shell. ``config`` of None means the config + arrives on stdin (``--config-fd 0``), which is what remote launches use: + a config in argv is visible in ``ps`` to every user on that host, and it + carries ``env:`` values. """ - return ["python", "-u", "-m", "execnet._trio_worker", config] + tokens = ["python", "-u", "-m", "execnet", "worker", *protocol] + if config is None: + return [*tokens, "--config-fd", "0"] + return [*tokens, "--config", config] -def worker_module_tokens(spec: Any) -> list[str]: - """``python -u -m execnet._trio_worker `` tokens for ``spec``.""" - return _worker_tokens(worker_cli_arg(spec)) +def worker_module_tokens(spec: Any, *protocol: str) -> list[str]: + """``python -u -m execnet worker`` tokens for ``spec``.""" + return _worker_tokens(worker_cli_arg(spec), *protocol) def _uv_tokens(python: str | None) -> list[str]: @@ -227,7 +234,7 @@ def _extra_with_tokens(config: str) -> list[str]: return [] -def uv_worker_argv(spec: Any) -> list[str]: +def uv_worker_argv(spec: Any, *protocol: str) -> list[str]: """``uv run`` argv to launch the Trio worker locally (wheel path is local).""" config = worker_cli_arg(spec) return [ @@ -235,7 +242,7 @@ def uv_worker_argv(spec: Any) -> list[str]: "--with", coordinator_requirement(), *_extra_with_tokens(config), - *_worker_tokens(config), + *_worker_tokens(config, *protocol), ] @@ -398,8 +405,8 @@ def sub_spawn_argv(request: dict[str, Any]) -> tuple[list[str], bytes]: if python: assert isinstance(python, str) if target_has_execnet(python): - argv = [*shell_split_path(python), "-u", "-m", "execnet._trio_worker"] - return [*argv, config], b"" + argv = [*shell_split_path(python), "-u", "-m", "execnet", "worker"] + return [*argv, "--config", config], b"" requirement, _ = _requested_requirement(request) if requirement is None or not uv_available(): raise RuntimeError( @@ -412,4 +419,12 @@ def sub_spawn_argv(request: dict[str, Any]) -> tuple[list[str], bytes]: requirement, *_worker_tokens(config), ], b"" - return [sys.executable, "-u", "-m", "execnet._trio_worker", config], b"" + return [ + sys.executable, + "-u", + "-m", + "execnet", + "worker", + "--config", + config, + ], b"" diff --git a/src/execnet/_socketserver.py b/src/execnet/_socketserver.py index c27609f6..ee6c4191 100644 --- a/src/execnet/_socketserver.py +++ b/src/execnet/_socketserver.py @@ -1,18 +1,20 @@ """Trio socket server for execnet gateways. Listens on a TCP port and hands each accepted connection (by fd) to a fresh -``python -m execnet._trio_worker`` subprocess that serves the gateway over it. -No code is executed inline. +worker subprocess that serves the gateway over it. No code is executed +inline. -This module implements the ``execnet-socketserver`` console command, which is -the supported entry point -- run it on the target host, or install-free with -``uvx --from execnet execnet-socketserver``. +The supported entry point is ``execnet server`` (see :mod:`execnet._cli`) -- +run it on the target host, or install-free with +``uvx --from execnet execnet server``. The old ``execnet-socketserver`` +console command still works and forwards here with a DeprecationWarning. """ from __future__ import annotations -async def _trio_serve(hostport: str, once: bool) -> None: +async def serve(hostport: str, once: bool) -> None: + """Bind ``hostport`` and hand accepted connections to worker processes.""" import trio from execnet import _trio_host @@ -38,28 +40,10 @@ async def handler(stream: trio.SocketStream) -> None: def main(argv: list[str] | None = None) -> None: - """Console entry point (``execnet-socketserver``).""" - import argparse + """Deprecated ``execnet-socketserver`` console entry point.""" + from ._cli import socketserver_main - import trio - - parser = argparse.ArgumentParser( - prog="execnet-socketserver", - description="Serve execnet gateway connections over a socket.", - ) - parser.add_argument( - "hostport", - nargs="?", - default=":8888", - help="address to bind as HOST:PORT or :PORT (default: :8888)", - ) - parser.add_argument( - "--once", - action="store_true", - help="serve a single connection and exit instead of looping", - ) - args = parser.parse_args(argv) - trio.run(_trio_serve, args.hostport, args.once) + socketserver_main(argv) if __name__ == "__main__": diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 9f3f06fe..ae141e0f 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -101,8 +101,10 @@ async def open_popen_process(args: list[str]) -> trio.Process: ) -def popen_module_args(spec: Any) -> list[str]: - """Launch the Trio worker as a module: ``python -m execnet._trio_worker``. +def popen_module_args( + spec: Any, *protocol: str, config_on_stdin: bool = False +) -> list[str]: + """Launch the Trio worker as a module: ``python -m execnet worker``. No source is sent over the wire; the worker imports the installed execnet + trio. Used for same-interpreter popen and for a ``python=`` interpreter that @@ -118,11 +120,15 @@ def popen_module_args(spec: Any) -> list[str]: args = [*interpreter, "-u"] if getattr(spec, "dont_write_bytecode", False): args.append("-B") - args += ["-m", "execnet._trio_worker", _provision.worker_cli_arg(spec)] + args += ["-m", "execnet", "worker", *protocol] + if config_on_stdin: + args += ["--config-fd", "0"] + else: + args += ["--config", _provision.worker_cli_arg(spec)] return args -def popen_worker_argv(spec: Any) -> list[str]: +def popen_worker_argv(spec: Any, *protocol: str) -> list[str]: """Argv for a popen worker: direct module launch, or uv-provisioned. A bare ``python=`` interpreter without execnet gets execnet + trio @@ -131,8 +137,8 @@ def popen_worker_argv(spec: Any) -> list[str]: from . import _provision if spec.python and not _provision.target_has_execnet(spec.python): - return _provision.uv_worker_argv(spec) - return popen_module_args(spec) + return _provision.uv_worker_argv(spec, *protocol) + return popen_module_args(spec, *protocol) class RawChannel: diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 567851b0..206f8081 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -807,7 +807,16 @@ def _spawn_socket_worker(fd: int) -> subprocess.Popen[bytes]: } ) return subprocess.Popen( - [sys.executable, "-m", "execnet._trio_worker", config, "--socket-fd", str(fd)], + [ + sys.executable, + "-m", + "execnet", + "worker", + "--protocol-fd", + str(fd), + "--config", + config, + ], pass_fds=[fd], ) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index b20be7b6..0550dfc0 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -2,12 +2,17 @@ from __future__ import annotations +import json import os +import stat import sys import threading from collections.abc import Callable +from collections.abc import Sequence +from contextlib import suppress from typing import TYPE_CHECKING from typing import Any +from typing import Protocol import trio @@ -369,37 +374,66 @@ def waitall(self, timeout: float | None = None) -> bool: return self._idle.wait(timeout) -def _prepare_protocol_fds() -> tuple[int, int]: - """Dup protocol pipes off stdin/stdout and redirect stdio to /dev/null. +def _devnull() -> str: + try: + return os.devnull + except AttributeError: # pragma: no cover - defensive + return "NUL" if os.name == "nt" else "/dev/null" + + +def _dup_protocol_fds() -> tuple[int, int]: + """Move the protocol off stdin/stdout, leaving fd 0/1 free to redirect. - Returns ``(read_fd, write_fd)`` for the Message protocol (child reads - coordinator stdin writes; child writes go to coordinator stdout reads). + Returns ``(read_fd, write_fd)`` for the Message protocol (the worker + reads what the coordinator writes to our stdin, and writes what the + coordinator reads from our stdout). What happens to fd 0/1 afterwards + is the caller's choice -- see :func:`apply_stdio`. """ if not hasattr(os, "dup"): # pragma: no cover - jython legacy - raise RuntimeError("Trio worker requires os.dup") + raise RuntimeError("the execnet worker requires os.dup") + return os.dup(0), os.dup(1) - try: - devnull = os.devnull - except AttributeError: - devnull = "NUL" if os.name == "nt" else "/dev/null" - read_fd = os.dup(0) - fd = os.open(devnull, os.O_RDONLY) - os.dup2(fd, 0) - os.close(fd) +def apply_stdio( + stdin: str = "inherit", stdout: str = "inherit", stderr: str = "inherit" +) -> None: + """Point the worker's standard fds where the launcher asked. - write_fd = os.dup(1) - fd = os.open(devnull, os.O_WRONLY) - os.dup2(fd, 1) + ``inherit`` leaves an fd alone, which is the default once the protocol + has a transport of its own: a worker's output is then the user's, not + something execnet has to swallow to protect the wire. - if os.name == "nt": - sys.stderr = os.fdopen(os.dup(2), "w", 1) + ``close`` (stdin only) reopens fd 0 on the null device *and* closes + ``sys.stdin``. Reads through Python raise, while the fd itself stays + reserved -- genuinely closing it would let the next ``os.open`` land on + fd 0, where anything writing to "stdin" would corrupt an unrelated file. + """ + if stdin in ("close", "devnull"): + fd = os.open(_devnull(), os.O_RDONLY) + os.dup2(fd, 0) + os.close(fd) + if stdin == "close": + with suppress(Exception): + sys.stdin.close() + sys.stdin = os.fdopen(0, "r", closefd=False) + sys.stdin.close() + else: + sys.stdin = os.fdopen(0, "r", closefd=False) + + if stdout == "stderr": + os.dup2(2, 1) + sys.stdout = os.fdopen(1, "w", buffering=1, closefd=False) + elif stdout == "devnull": + fd = os.open(_devnull(), os.O_WRONLY) + os.dup2(fd, 1) + os.close(fd) + sys.stdout = os.fdopen(1, "w", buffering=1, closefd=False) + + if stderr == "devnull": + fd = os.open(_devnull(), os.O_WRONLY) os.dup2(fd, 2) - os.close(fd) - - sys.stdin = os.fdopen(0, "r", 1, closefd=False) - sys.stdout = os.fdopen(1, "w", 1, closefd=False) - return read_fd, write_fd + os.close(fd) + sys.stderr = os.fdopen(2, "w", buffering=1, closefd=False) class _WorkerIOStub: @@ -511,67 +545,193 @@ async def _serve_async_worker(stream: Any, id: str) -> None: nursery.cancel_scope.cancel() -def serve_popen_async(id: str) -> None: - """Serve the pure-async worker over the stdio pipes (profile=trio).""" - from ._trio_gateway import staple_fd_stream +class Transport(Protocol): + """How a worker's protocol stream comes into being. - read_fd, write_fd = _prepare_protocol_fds() - os.write(write_fd, b"1") + Split in two because the stdio transport has to claim fd 0/1 *before* + anything else touches them (including reading ``--config-fd 0``), while + opening the stream itself needs a running trio loop. + """ - async def main() -> None: - await _serve_async_worker(staple_fd_stream(read_fd, write_fd), id) + #: default stdio disposition once this transport is serving + stdio_defaults: tuple[str, str, str] - trio.run(main) - os._exit(0) + def prepare(self) -> None: + """Synchronous fd bookkeeping, before the config is read.""" + async def open(self) -> Any: + """The protocol ByteStream, ready for the Message protocol.""" -def serve_socket_async(id: str, socket_fd: int) -> None: - """Serve the pure-async worker over an inherited socket fd.""" - from . import _trio_host - async def main() -> None: - stream = await _trio_host.adopt_socket(socket_fd) - await _serve_async_worker(stream, id) +async def _send_ready(stream: Any) -> Any: + """Write the single handshake byte the coordinator waits for.""" + await stream.send_all(b"1") + return stream - trio.run(main) - os._exit(0) +class StdioTransport: + """The protocol *is* this process's stdin/stdout (the classic shape). -def serve_popen_trio( - id: str, profile: str = "thread", wait: WaitBackend = "thread" -) -> None: - """Serve a WorkerGateway over the stdio pipes (popen / ssh worker).""" - from . import _trio_host + Nothing else can use fd 0/1 afterwards, so the default disposition + closes stdin and folds stdout onto stderr -- remote ``print()`` stays + visible on the coordinator instead of going to the null device, and it + cannot corrupt the wire because the wire is no longer fd 1. + """ - model = get_execmodel(profile) - read_fd, write_fd = _prepare_protocol_fds() - # Bootstrap handshake: the coordinator waits for this byte on our stdout - # before starting the Message protocol. We are launched as a plain module - # (``python -m execnet._trio_worker``), so nothing was sent to bootstrap us. - os.write(write_fd, b"1") + stdio_defaults = ("close", "stderr", "inherit") + + def __init__(self) -> None: + self._fds: tuple[int, int] | None = None + + def prepare(self) -> None: + self._fds = _dup_protocol_fds() + + async def open(self) -> Any: + from ._trio_gateway import staple_fd_stream + + assert self._fds is not None, "prepare() first" + read_fd, write_fd = self._fds + return await _send_ready(staple_fd_stream(read_fd, write_fd)) - host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") - host.start() - io = host.call(_make_fd_io, read_fd, write_fd) - _run_worker(host, io, id, model, wait) +class FdTransport: + """The protocol runs over inherited fds: one socket, or a pipe pair.""" + + stdio_defaults = ("inherit", "inherit", "inherit") + + def __init__(self, fds: Sequence[int]) -> None: + self.fds = tuple(fds) + + def prepare(self) -> None: + pass + + async def open(self) -> Any: + from . import _trio_host + from ._trio_gateway import staple_fd_stream + + if len(self.fds) == 2: + read_fd, write_fd = self.fds + return await _send_ready(staple_fd_stream(read_fd, write_fd)) + (fd,) = self.fds + if not stat.S_ISSOCK(os.fstat(fd).st_mode): + raise ValueError( + f"--protocol-fd {fd} is not a socket; a single fd must be" + " bidirectional, use --protocol-fd READFD,WRITEFD for a pipe pair" + ) + # adopt_socket sends the handshake itself + return await _trio_host.adopt_socket(fd) + + +def parse_address(address: str) -> tuple[str, Any]: + """``unix:/path`` or ``host:port`` -> ``("unix", path)`` / ``("tcp", (h, p))``.""" + if address.startswith("unix:"): + return "unix", address[len("unix:") :] + host, sep, port = address.rpartition(":") + if not sep or not port.isdigit(): + raise ValueError(f"expected unix:/path or host:port, got {address!r}") + return "tcp", (host or "localhost", int(port)) + + +class ConnectTransport: + """The worker dials out to the coordinator and serves on that connection. + + This is what lets ssh carry the protocol without owning our stdio: the + coordinator listens on a local unix socket, ``ssh -R`` forwards it, and + we connect to the remote end of the forward. + """ + + stdio_defaults = ("inherit", "inherit", "inherit") + + def __init__(self, address: str) -> None: + self.kind, self.target = parse_address(address) + + def prepare(self) -> None: + pass + + async def open(self) -> Any: + if self.kind == "unix": + stream = await trio.open_unix_socket(self.target) + else: + stream = await trio.open_tcp_stream(*self.target) + return await _send_ready(stream) -def serve_socket_trio( - id: str, profile: str, socket_fd: int, wait: WaitBackend = "thread" + +class ListenTransport: + """The worker listens and serves the first coordinator that connects. + + The bound address is printed to stdout as JSON before serving, so a + launcher that asked for an ephemeral port can learn which one it got. + """ + + stdio_defaults = ("inherit", "inherit", "inherit") + + def __init__(self, address: str) -> None: + self.kind, self.target = parse_address(address) + + def prepare(self) -> None: + pass + + async def open(self) -> Any: + if self.kind == "unix": + sock = trio.socket.socket(trio.socket.AF_UNIX, trio.socket.SOCK_STREAM) + await sock.bind(self.target) + sock.listen(1) + listeners = [trio.SocketListener(sock)] + bound: Any = self.target + else: + host, port = self.target + listeners = await trio.open_tcp_listeners(port, host=host) + bound = listeners[0].socket.getsockname()[:2] + print(json.dumps({"listening": bound}), flush=True) + stream = await listeners[0].accept() + for listener in listeners: + await listener.aclose() + return await _send_ready(stream) + + +def serve_worker( + config: dict[str, Any], + transport: Transport, + *, + stdin: str | None = None, + stdout: str | None = None, + stderr: str | None = None, ) -> None: - """Serve a WorkerGateway over an inherited socket fd. + """Serve one gateway for ``config`` over ``transport``, then exit. - Used for the socketserver (an accepted TCP connection) and, in future, a - popen socketpair. The socket is adopted inside Trio and the handshake is - written on the host loop; config comes from the CLI. + ``transport.prepare()`` must already have run (the CLI does it before + reading the config, so ``--config-fd 0`` still works under the stdio + transport). """ + _check_version(config["coordinator_version"]) + _apply_worker_setup(config) + defaults = transport.stdio_defaults + apply_stdio( + stdin=stdin if stdin is not None else defaults[0], + stdout=stdout if stdout is not None else defaults[1], + stderr=stderr if stderr is not None else defaults[2], + ) + + id = config["id"] + # "execmodel" is the pre-3.0 spelling; accept both so a version-skewed + # coordinator still connects. + profile = effective_profile(config.get("profile") or config["execmodel"]) + wait: WaitBackend = config.get("wait", "thread") + + if profile == "trio": + # pure-async profile: one thread, the loop owns the main thread + async def main() -> None: + await _serve_async_worker(await transport.open(), id) + + trio.run(main) + os._exit(0) + from . import _trio_host - model = get_execmodel(profile) host = _trio_host.TrioHost(name=f"execnet-trio-worker-{id}") host.start() - io = host.call(_trio_host.adopt_socket, socket_fd) - _run_worker(host, io, id, model, wait) + io = host.call(transport.open) + _run_worker(host, io, id, get_execmodel(profile), wait) def _rough_version(version: str) -> tuple[int, ...]: @@ -625,46 +785,3 @@ def _check_version(coordinator_version: str) -> None: % (coordinator_version, execnet.__version__) ) sys.stderr.flush() - - -def _main() -> None: - """Entry point for ``python -m execnet._trio_worker [--socket-fd N]``. - - ```` is the coordinator's ``_provision.worker_cli_arg`` payload: - ``{"id", "profile", "wait", "coordinator_version"}``. With ``--socket-fd`` the - worker serves over that inherited socket (socketserver); otherwise over the - stdio pipes (popen / ssh). The worker imports execnet + trio from the - environment; no source is sent over the wire to bootstrap it. - """ - import argparse - import json - - parser = argparse.ArgumentParser(prog="execnet._trio_worker") - parser.add_argument("config", help="JSON worker config") - # Protocol transport: an inherited socket fd (socketserver, or a future popen - # socketpair); without it the worker serves over the stdio pipes (ssh). - parser.add_argument("--socket-fd", type=int, default=None) - ns = parser.parse_args() - - config = json.loads(ns.config) - _check_version(config["coordinator_version"]) - _apply_worker_setup(config) - # "execmodel" is the pre-3.0 spelling; accept both so a version-skewed - # coordinator still connects. - profile = config.get("profile") or config["execmodel"] - wait: WaitBackend = config.get("wait", "thread") - profile = effective_profile(profile) - if profile == "trio": - # pure-async profile: one thread, the loop owns the main thread - if ns.socket_fd is not None: - serve_socket_async(config["id"], ns.socket_fd) - else: - serve_popen_async(config["id"]) - elif ns.socket_fd is not None: - serve_socket_trio(config["id"], profile, ns.socket_fd, wait=wait) - else: - serve_popen_trio(id=config["id"], profile=profile, wait=wait) - - -if __name__ == "__main__": - _main() diff --git a/testing/test_basics.py b/testing/test_basics.py index 0afcdd0d..4cebf7ef 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -185,10 +185,9 @@ class Arg(Exception): @pytest.mark.skipif("not hasattr(os, 'dup')") -def test_stdouterrin_setnull(capfd: pytest.CaptureFixture[str]) -> None: - # _prepare_protocol_fds dups the stdio fds for the Message protocol and - # points fd 0/1 at devnull; writes/reads on the original fds must go - # nowhere. Back up and restore the real fds around the call. +def test_stdio_disposition_devnull(capfd: pytest.CaptureFixture[str]) -> None: + # apply_stdio(devnull) points fd 0/1 at the null device: writes and + # reads on them must go nowhere. Back up and restore the real fds. from execnet import _trio_worker orig_stdin = sys.stdin @@ -196,9 +195,10 @@ def test_stdouterrin_setnull(capfd: pytest.CaptureFixture[str]) -> None: orig_fd0 = os.dup(0) orig_fd1 = os.dup(1) try: - read_fd, write_fd = _trio_worker._prepare_protocol_fds() + read_fd, write_fd = _trio_worker._dup_protocol_fds() os.close(read_fd) os.close(write_fd) + _trio_worker.apply_stdio(stdin="devnull", stdout="devnull") os.write(1, b"hello") os.read(0, 1) out, err = capfd.readouterr() @@ -213,6 +213,26 @@ def test_stdouterrin_setnull(capfd: pytest.CaptureFixture[str]) -> None: os.close(orig_fd1) +@pytest.mark.skipif("not hasattr(os, 'dup')") +def test_stdio_disposition_stdout_to_stderr(capfd: pytest.CaptureFixture[str]) -> None: + # The stdio transport's default: remote output lands on stderr rather + # than the null device, so it stays visible without touching the wire. + from execnet import _trio_worker + + orig_stdout = sys.stdout + orig_fd1 = os.dup(1) + try: + _trio_worker.apply_stdio(stdout="stderr") + os.write(1, b"to-stderr") + out, err = capfd.readouterr() + assert not out + assert "to-stderr" in err + finally: + sys.stdout = orig_stdout + os.dup2(orig_fd1, 1) + os.close(orig_fd1) + + class PseudoChannel: class gateway: class _channelfactory: diff --git a/testing/test_gateway.py b/testing/test_gateway.py index 3c6f2fe1..86ee74f3 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -578,7 +578,7 @@ def test_popen_args(spec: str, expected_args: list[str]) -> None: args = _trio_gateway.popen_module_args(execnet.XSpec(spec + "//id=gw0")) assert args[: len(expected_args)] == expected_args - assert args[len(expected_args) :][:3] == ["-u", "-m", "execnet._trio_worker"] + assert args[len(expected_args) :][:4] == ["-u", "-m", "execnet", "worker"] def test_sequential_remote_exec_claims_the_main_thread( diff --git a/testing/test_provision.py b/testing/test_provision.py index 84e660e5..d7f01796 100644 --- a/testing/test_provision.py +++ b/testing/test_provision.py @@ -60,7 +60,15 @@ def test_sub_spawn_argv_plain_popen() -> None: import sys argv, preamble = _provision.sub_spawn_argv({"config": "{}"}) - assert argv == [sys.executable, "-u", "-m", "execnet._trio_worker", "{}"] + assert argv == [ + sys.executable, + "-u", + "-m", + "execnet", + "worker", + "--config", + "{}", + ] assert preamble == b"" diff --git a/testing/test_xspec.py b/testing/test_xspec.py index 891b6a1a..f81afaf5 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -80,7 +80,7 @@ def test_popen_with_sudo_python(self) -> None: spec = XSpec("popen//python=sudo python3//id=gw0") args = _trio_gateway.popen_module_args(spec) - assert args[:5] == ["sudo", "python3", "-u", "-m", "execnet._trio_worker"] + assert args[:6] == ["sudo", "python3", "-u", "-m", "execnet", "worker"] def test_env(self) -> None: xspec = XSpec("popen//env:NAME=value1") From a84380f222c2aa963223ff653735950b838489a1 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 30 Jul 2026 11:48:50 +0200 Subject: [PATCH 68/91] feat!: dial back over ssh -R, run popen on a socketpair Completes getting the protocol off the worker's stdio. transport= is a new spec key (socket|stdio) defaulting to socket on POSIX and stdio on Windows, where subprocess rejects pass_fds and ssh -R cannot forward a unix socket. popen workers get an inherited socketpair. ssh workers dial back: the coordinator binds a private unix socket, `ssh -R` forwards it, and the worker connects to the remote end -- which frees ssh's stdin for the config. That closes a real exposure: the config carries `env:` values, and it used to ride in the remote argv where `ps` shows it to every user on that host. HostNotFound had to be reworked with it, since a dial-back has no shared pipe to observe EOF on; the accept now races process exit and still maps ssh's 255. A dev-coordinator wheel is delivered over its own ssh connection before the launch and cached remotely by name, so the launch command drops the `head -c ` framing, the mktemp prelude and the `exec`. Two traps worth knowing: shlex.quote("~/...") creates a directory literally named "~", and the cached-skip branch must still drain stdin or the coordinator gets EPIPE. `execnet info` replaces the `import execnet, trio` probe, so provisioning learns the target's version and transports before connecting. Also fixes a claim race the -n 12 run exposed: HybridExec released the main-thread claim after the channel close that lets a coordinator send the next request, so a sequential remote_exec could be admitted first and overflow. The release now happens on the main thread before the done signal. The window shrinks but does not close -- unlike the serialized main_thread_only it replaces, only the *first* request is guaranteed the main thread, which is now documented rather than asserted. The asyncssh test harness gained a server_factory accepting `-R` unix forwards, which asyncssh refuses by default. Co-Authored-By: Claude Opus 5 --- CHANGELOG.rst | 52 +++++ handoff-phase-c-worker-axes.md | 53 +++++ src/execnet/_cli.py | 13 +- src/execnet/_provision.py | 235 +++++++++++++++----- src/execnet/_trio_gateway.py | 245 +++++++++++++++++--- src/execnet/_trio_host.py | 27 ++- src/execnet/_trio_worker.py | 25 ++- src/execnet/_xspec.py | 4 + testing/test_cli.py | 395 +++++++++++++++++++++++++++++++++ testing/test_gateway.py | 19 +- testing/test_provision.py | 44 +++- testing/test_ssh_local.py | 14 ++ 12 files changed, 1018 insertions(+), 108 deletions(-) create mode 100644 testing/test_cli.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 49dd647b..8431327b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,58 @@ 2.2.0 (UNRELEASED) ------------------ +* New ``execnet`` command line, and it is now the launch contract between a + coordinator and the worker process it starts:: + + execnet worker --protocol-stdio | --protocol-fd FD[,FD] + | --protocol-connect ADDR | --protocol-listen ADDR + --config JSON | --config-fd FD | --config-file PATH + --stdin/--stdout/--stderr DISPOSITION + execnet server [HOST:PORT] [--once] + execnet info + + ``ADDR`` is ``unix:/path`` or ``host:port``. Provisioning emits + ``python -m execnet worker ...`` for a direct interpreter launch and + ``execnet worker ...`` under ``uv run``; both are the same CLI. + ``execnet-socketserver`` still works and forwards to ``execnet server`` + with a ``DeprecationWarning``. ``execnet info`` reports version, trio + availability and supported transports as JSON, and replaces the + ``import execnet, trio`` probe used to decide whether a ``python=`` + interpreter can host a worker directly. +* The protocol no longer has to be the worker's stdin/stdout. A new + ``transport=socket|stdio`` spec key selects; it defaults to ``socket`` on + POSIX and ``stdio`` on Windows, where neither ``pass_fds`` nor ``ssh -R`` + unix-socket forwarding is available. ``socket`` means an inherited + socketpair for ``popen`` and an ``ssh -R``-forwarded unix socket that the + worker dials back on for ``ssh=``/``vagrant_ssh=``. +* **A worker's stdio now belongs to the code it runs.** It used to be + redirected to the null device so it could not corrupt the protocol, which + meant a remote ``print()`` went nowhere at all. With the socket transport + the worker leaves fd 0/1/2 alone entirely; with the stdio transport it + closes stdin and folds stdout onto stderr rather than discarding both. + Remote output that used to vanish now reaches the coordinator -- and a + worker can now also *read* the coordinator's stdin. New ``stdin=``, + ``stdout=`` and ``stderr=`` spec keys (``inherit``/``devnull``, plus + ``close`` for stdin and ``stderr`` for stdout) override any of it, e.g. + ``popen//stdin=devnull``. +* **The worker config no longer travels in the remote command line** for + ssh gateways, which closes an exposure: the config carries ``env:`` + values, and a remote argv is readable by every user on that host through + ``ps``. The socket transport frees ssh's stdin, so the config goes there + (``--config-fd 0``) instead. +* A shipped wheel (dev-version coordinator) is delivered to an ssh remote + over its own connection before the worker is launched, and cached there + by name. The launch command no longer needs ``head -c `` byte + accounting, a ``mktemp`` prelude, or ``exec`` to keep an fd alive, and the + protocol stream never carries a payload. +* ``main_thread_only`` used to *serialize*, so every sequential + ``remote_exec`` was guaranteed the worker's main thread. The ``thread`` + profile it now maps to releases its claim as an exec finishes, a moment + after the channel close that lets the coordinator send the next request -- + so a coordinator that immediately re-execs can rarely land on a pool + thread instead. The *first* request always gets the main thread; use the + ``trio`` or ``gevent`` profile where placement must never race. + * One namespace per concurrency library you drive execnet from: ``execnet.sync`` (plain threads; the top-level ``execnet.*`` aliases), ``execnet.trio``, ``execnet.aio`` and the new ``execnet.gevent``, whose blocking diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md index 98df1166..4e91d32e 100644 --- a/handoff-phase-c-worker-axes.md +++ b/handoff-phase-c-worker-axes.md @@ -117,6 +117,59 @@ no `remote_status()`, no `MultiChannel`, no group iteration, and no `RSync`. `AsyncGroup.makegateway` defaults workers to the `thread` profile (the coordinator's shape does not dictate the worker's). +## CLI + transports — LANDED 2026-07-30 + +The protocol used to *be* the worker's stdin/stdout. `execnet worker` now +names the transport, and the CLI is the launch contract (every launcher +emits `python -m execnet worker ...`, or `execnet worker ...` under +`uv run`). `_trio_worker` keeps the serving/profile layer; argument +parsing lives in `_cli`. + +| transport | who uses it | how | +|---|---|---| +| `--protocol-fd N` / `R,W` | popen (`transport=socket`), socketserver, `installvia` | inherited socketpair via `pass_fds`; a single fd must be a socket (checked with `S_ISSOCK`), a pipe needs the pair form | +| `--protocol-connect ADDR` | ssh / vagrant_ssh (`transport=socket`) | coordinator binds a private unix socket, `ssh -R` forwards it, the worker dials back | +| `--protocol-listen ADDR` | nothing yet — for trampolines | binds, prints `{"listening": ...}`, serves the first connection | +| `--protocol-stdio` | Windows, `transport=stdio`, and the `via` tunnel (which relays over the sub's stdio by construction) | the classic pipe pair | + +`transport=socket|stdio` is a spec key defaulting per platform +(`_provision.resolve_transport`); Windows stays on stdio because +`subprocess` rejects `pass_fds` there and `ssh -R` cannot forward a unix +socket. **Still unverified: whether the pre-existing socket/`installvia` +path (which already used `pass_fds`) works on the Windows CI job at all.** + +Things this fixed, each verified while doing it: + +- remote `print()` used to go to `/dev/null`; a worker's stdio is now the + user's, with `stdin=`/`stdout=`/`stderr=` spec keys to override. On the + stdio transport the fallback is close stdin + stdout→stderr, so output + is visible without a second stream. +- the ssh worker config carried `env:` values **in the remote argv**, + readable by every user on that host via `ps`. The socket transport + frees ssh's stdin, so it goes there (`--config-fd 0`). +- the dev-coordinator wheel was framed in-band on the protocol pipe with + `head -c `; it now travels on its own ssh connection before the + launch, cached remotely under `"$HOME"/.cache/execnet/wheels`. Two + traps found writing that: `shlex.quote("~/...")` creates a directory + literally named `~` (use `"$HOME"`), and the cached-skip branch must + still drain stdin or the coordinator gets EPIPE. + +`HostNotFound` needed rework for the dial-back: with no shared pipe there +is no EOF to observe, so `_accept_or_diagnose` races the accept against +`process.wait()` and still maps ssh's exit 255. + +The asyncssh harness in `testing/test_ssh_local.py` needed a +`server_factory` whose `unix_server_requested` returns True — asyncssh +refuses `-R` unix forwards otherwise, which is what a dial-back needs. + +Known semantic loss, documented on `HybridExec` and in the changelog: +`main_thread_only` serialized, so *every* sequential `remote_exec` got the +main thread. The `thread` profile releases its claim just after the +channel close that lets the coordinator send the next request, so an +immediate re-exec can rarely land on a pool thread. The first request is +still deterministic (FIFO admission). This surfaced as a `-n 12` flake, +not by reasoning — worth remembering that the hybrid claim is best-effort. + ## Where the repo stands (2026-07-25, after `a69b844`) One protocol engine: `AsyncGateway` (`_trio_gateway.py`). The sync API diff --git a/src/execnet/_cli.py b/src/execnet/_cli.py index ebd86bbe..0b4e86cf 100644 --- a/src/execnet/_cli.py +++ b/src/execnet/_cli.py @@ -162,7 +162,10 @@ def _load_config(ns: argparse.Namespace) -> dict[str, Any]: if ns.config is not None: raw = ns.config elif ns.config_fd is not None: - with os.fdopen(ns.config_fd, "r", encoding="utf-8") as stream: + # read a dup, so ``--config-fd 0`` leaves fd 0 itself open (at EOF). + # Closing it would free the slot for the next os.open, and anything + # then writing to "stdin" would land in an unrelated file. + with os.fdopen(os.dup(ns.config_fd), "r", encoding="utf-8") as stream: raw = stream.read() elif ns.config_file is not None: with open(ns.config_file, encoding="utf-8") as stream: @@ -219,8 +222,16 @@ def interpreter_info() -> dict[str, Any]: """ from ._version import version + try: + import trio + + trio_version: str | None = trio.__version__ + except Exception: + trio_version = None + return { "execnet": version, + "trio": trio_version, "python": ".".join(str(part) for part in sys.version_info[:3]), "executable": sys.executable, "platform": sys.platform, diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index 1e21a996..22427b1a 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -31,6 +31,35 @@ _RELEASED_RE = re.compile(r"^\d+\.\d+\.\d+$") +#: ``transport=`` values: where a worker's protocol stream lives. +#: +#: ``socket`` keeps the protocol off the worker's stdio -- an inherited +#: socketpair for popen, an ``ssh -R``-forwarded unix socket for ssh -- so +#: the worker's stdin/stdout/stderr belong to the code it runs. ``stdio`` +#: is the classic shape, and the only one available where the machinery +#: ``socket`` needs is missing. +TRANSPORTS = ("socket", "stdio") + + +def socket_transport_available() -> bool: + """Whether this coordinator can drive the socket transport. + + ``subprocess`` refuses ``pass_fds`` on Windows and ``ssh -R`` cannot + forward a unix socket there either, so Windows coordinators stay on + stdio unless someone asks for otherwise and accepts the consequences. + """ + return not sys.platform.startswith("win") + + +def resolve_transport(spec: Any) -> str: + """The transport for ``spec``: explicit if given, else the platform default.""" + requested: str | None = getattr(spec, "transport", None) + if requested is None: + return "socket" if socket_transport_available() else "stdio" + if requested not in TRANSPORTS: + raise ValueError(f"unknown transport {requested!r} (known: {list(TRANSPORTS)})") + return requested + def uv_available() -> bool: """Whether the ``uv`` launcher is on PATH.""" @@ -49,18 +78,37 @@ def shell_split_path(path: str) -> list[str]: @cache -def target_has_execnet(python: str) -> bool: - """Whether interpreter ``python`` can already import execnet + trio. - - When true the worker can be launched directly on that interpreter - (preserving ``sys.executable``); otherwise it must be uv-provisioned. +def target_info(python: str) -> dict[str, Any] | None: + """``execnet info`` from interpreter ``python``, or None if it cannot run. + + One probe answers everything provisioning wants to know before it + connects: whether execnet is importable at all, which version it is, + whether trio is there, and which transports it can serve. An execnet + too old to have the CLI fails the probe and gets uv-provisioned, which + is the right outcome. """ - argv = [*shell_split_path(python), "-c", "import execnet, trio"] + argv = [*shell_split_path(python), "-m", "execnet", "info"] try: completed = subprocess.run(argv, capture_output=True, timeout=30, check=False) - return completed.returncode == 0 except (OSError, subprocess.SubprocessError): - return False + return None + if completed.returncode != 0: + return None + try: + info: dict[str, Any] = json.loads(completed.stdout) + except ValueError: + return None + return info + + +def target_has_execnet(python: str) -> bool: + """Whether ``python`` can host a worker directly (execnet + trio present). + + When true the worker is launched on that interpreter as-is (preserving + ``sys.executable``); otherwise it must be uv-provisioned. + """ + info = target_info(python) + return info is not None and info.get("trio") is not None def _version_slug(version: str) -> str: @@ -193,6 +241,22 @@ def worker_cli_arg(spec: Any) -> str: return json.dumps(config) +def stdio_tokens(spec: Any) -> list[str]: + """``--stdin/--stdout/--stderr`` tokens for whatever ``spec`` asked for. + + A worker inherits the coordinator's stdio by default now that the + protocol has a transport of its own, which also means remote code can + *consume* the coordinator's stdin. These keys are how a caller says + otherwise, e.g. ``popen//stdin=devnull``. + """ + tokens = [] + for name in ("stdin", "stdout", "stderr"): + value = getattr(spec, name, None) + if value: + tokens += [f"--{name}", value] + return tokens + + def _worker_tokens(config: str | None, *protocol: str) -> list[str]: """``python -u -m execnet worker`` tokens for a launch. @@ -246,68 +310,114 @@ def uv_worker_argv(spec: Any, *protocol: str) -> list[str]: ] +#: where a shipped wheel lands on the remote, keyed by name (which carries +#: the version) so repeated gateways to one host reuse it. A *shell +#: fragment*, not a path: ``$HOME`` is expanded remotely, and quoting it as +#: a literal would create a directory actually called ``~``. +REMOTE_WHEEL_DIR = '"$HOME"/.cache/execnet/wheels' + + +def remote_wheel_path(wheel: Path) -> str: + """Shell fragment for the remote path a shipped wheel is delivered to.""" + return f"{REMOTE_WHEEL_DIR}/{shlex.quote(wheel.name)}" + + +def wheel_delivery_command(wheel: Path) -> str: + """Remote sh command that receives ``wheel`` on stdin, unless already there. + + Out of band: run over its *own* ssh connection before the worker + launch, so the protocol stream never has to carry a payload and the + launch command needs neither ``head -c `` byte accounting nor + ``exec`` to keep an fd alive. + + Both branches consume stdin -- the coordinator streams the wheel + unconditionally, and a remote that skipped the write without draining + would hand it an EPIPE. + """ + path = remote_wheel_path(wheel) + return ( + f"mkdir -p {REMOTE_WHEEL_DIR} || exit 1; " + f"if [ -s {path} ]; then cat > /dev/null; else cat > {path}.tmp" + f" && mv {path}.tmp {path}; fi" + ) + + def _remote_shell_command( python: str | None, config: str, - *, + *protocol: str, requirement: str | None = None, wheel: Path | None = None, -) -> tuple[str, bytes]: - """Remote sh command + stdin preamble launching the worker via uv. + config_on_stdin: bool = False, +) -> str: + """Remote sh command launching the worker via uv. - With ``requirement`` the remote installs from an index and no preamble is - needed. With ``wheel`` a POSIX-sh prelude receives the wheel bytes from - stdin (``head -c N``) into a temp dir and ``exec``s uv against it; the - wheel bytes are returned as the preamble to stream before the protocol. + ``requirement`` installs from an index; ``wheel`` uses one already + delivered by :func:`wheel_delivery_command`. """ - worker = _worker_tokens(config) + worker = _worker_tokens(None if config_on_stdin else config, *protocol) uv = [*_uv_tokens(python), *_extra_with_tokens(config)] if wheel is None: assert requirement is not None - return shlex.join([*uv, "--with", requirement, *worker]), b"" - - data = wheel.read_bytes() - # "$d/": expand the temp dir, concatenate the (quoted) wheel filename. - remote_wheel = '"$d/"' + shlex.quote(wheel.name) - uv_run = " ".join(shlex.quote(token) for token in [*uv, "--with"]) - worker_cmd = " ".join(shlex.quote(token) for token in worker) - prelude = ( - f"d=$(mktemp -d) && " - f"head -c {len(data)} > {remote_wheel} && " - f"exec {uv_run} {remote_wheel} {worker_cmd}" + return shlex.join([*uv, "--with", requirement, *worker]) + # the wheel path is remote-side and may contain ~, so it is not quoted + # by shlex.join -- splice it in after quoting the rest + return " ".join( + [shlex.join([*uv, "--with"]), remote_wheel_path(wheel), shlex.join(worker)] ) - return prelude, data -def ssh_remote_command(spec: Any) -> tuple[str, bytes]: - """Remote shell command + stdin preamble to launch the worker over ssh. +def ssh_remote_command(spec: Any, *protocol: str, config_on_stdin: bool = False) -> str: + """Remote shell command launching the worker over ssh. - Released coordinator -> ``uv run --with execnet== …`` with no preamble. - Dev coordinator -> wheel-shipping prelude (see ``_remote_shell_command``). + Released coordinator -> ``uv run --with execnet== …``. Dev + coordinator -> ``uv run --with …``; delivering the + wheel is a separate step (:func:`wheel_delivery_command`). """ import execnet version = execnet.__version__ config = worker_cli_arg(spec) + kwargs: dict[str, Any] = {"config_on_stdin": config_on_stdin} if _RELEASED_RE.match(version): - return _remote_shell_command( - spec.python, config, requirement=f"execnet=={version}" - ) - return _remote_shell_command(spec.python, config, wheel=_build_wheel(version)) + kwargs["requirement"] = f"execnet=={version}" + else: + kwargs["wheel"] = _build_wheel(version) + return _remote_shell_command(spec.python, config, *protocol, **kwargs) + + +def ssh_wheel(spec: Any) -> Path | None: + """The wheel this coordinator must deliver before launching, if any.""" + import execnet + version = execnet.__version__ + if _RELEASED_RE.match(version): + return None + return _build_wheel(version) -def ssh_argv(ssh: str, ssh_config: str | None, remote_command: str) -> list[str]: + +def ssh_argv( + ssh: str, + ssh_config: str | None, + remote_command: str, + options: list[str] | None = None, +) -> list[str]: """``ssh`` client argv running ``remote_command`` on host ``ssh``.""" args = ["ssh", "-C"] if ssh_config: args += ["-F", ssh_config] + if options: + args += options args += ssh.split() args.append(remote_command) return args def vagrant_ssh_argv( - machine: str, ssh_config: str | None, remote_command: str + machine: str, + ssh_config: str | None, + remote_command: str, + options: list[str] | None = None, ) -> list[str]: """``vagrant ssh`` argv running ``remote_command`` on the named VM. @@ -317,6 +427,8 @@ def vagrant_ssh_argv( args = ["vagrant", "ssh", machine, "--", "-C"] if ssh_config: args += ["-F", ssh_config] + if options: + args += options args.append(remote_command) return args @@ -375,14 +487,24 @@ def _requested_requirement(request: dict[str, Any]) -> tuple[str | None, Path | return None, None -def sub_spawn_argv(request: dict[str, Any]) -> tuple[list[str], bytes]: - """(argv, stdin preamble) spawning a requested sub-worker on this host. +#: an out-of-band step to run before a sub-worker launch: ``(argv, stdin)`` +DeliveryStep = tuple[list[str], bytes] + + +def sub_spawn_argv( + request: dict[str, Any], +) -> tuple[list[str], DeliveryStep | None]: + """(argv, wheel delivery) spawning a requested sub-worker on this host. Handles a ``GATEWAY_START_SUB`` request on a via master: plain popen runs this interpreter's worker module, a foreign ``python`` runs directly when it already has execnet and is uv-provisioned otherwise, and ``ssh`` wraps - the remote uv command (streaming a shipped wheel as the preamble for dev - versions). + the remote uv command. A shipped wheel is delivered by the returned + step -- its own ssh connection, run before the launch -- rather than + framed into the launch command's stdin. + + The sub's protocol is relayed over its stdio by the master, so it always + gets the stdio transport. """ config = request["config"] assert isinstance(config, str) @@ -393,20 +515,27 @@ def sub_spawn_argv(request: dict[str, Any]) -> tuple[list[str], bytes]: requirement, wheel = _requested_requirement(request) if requirement is None: raise RuntimeError("remote spawn request without provisioning material") - command, preamble = _remote_shell_command( - python, config, requirement=requirement, wheel=wheel - ) ssh_config = request.get("ssh_config") - if ssh: - assert isinstance(ssh, str) - return ssh_argv(ssh, ssh_config, command), preamble - assert isinstance(vagrant, str) - return vagrant_ssh_argv(vagrant, ssh_config, command), preamble + + def wrap(command: str, options: list[str] | None = None) -> list[str]: + if ssh: + assert isinstance(ssh, str) + return ssh_argv(ssh, ssh_config, command, options) + assert isinstance(vagrant, str) + return vagrant_ssh_argv(vagrant, ssh_config, command, options) + + delivery: DeliveryStep | None = None + if wheel is not None: + delivery = (wrap(wheel_delivery_command(wheel)), wheel.read_bytes()) + command = _remote_shell_command(python, config, wheel=wheel) + else: + command = _remote_shell_command(python, config, requirement=requirement) + return wrap(command), delivery if python: assert isinstance(python, str) if target_has_execnet(python): argv = [*shell_split_path(python), "-u", "-m", "execnet", "worker"] - return [*argv, "--config", config], b"" + return [*argv, "--config", config], None requirement, _ = _requested_requirement(request) if requirement is None or not uv_available(): raise RuntimeError( @@ -418,7 +547,7 @@ def sub_spawn_argv(request: dict[str, Any]) -> tuple[list[str], bytes]: "--with", requirement, *_worker_tokens(config), - ], b"" + ], None return [ sys.executable, "-u", @@ -427,4 +556,4 @@ def sub_spawn_argv(request: dict[str, Any]) -> tuple[list[str], bytes]: "worker", "--config", config, - ], b"" + ], None diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index ae141e0f..44b7dbd3 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -22,9 +22,11 @@ from __future__ import annotations import math +import os import subprocess import sys import types +import uuid from collections.abc import AsyncIterator from collections.abc import Callable from contextlib import asynccontextmanager @@ -121,6 +123,7 @@ def popen_module_args( if getattr(spec, "dont_write_bytecode", False): args.append("-B") args += ["-m", "execnet", "worker", *protocol] + args += _provision.stdio_tokens(spec) if config_on_stdin: args += ["--config-fd", "0"] else: @@ -757,29 +760,192 @@ async def serve_gateway( await gateway.aclose() -def ssh_transport_args(spec: Any) -> tuple[list[str], bytes]: - """``(ssh argv, stdin preamble)`` for an ssh worker. +#: how long to wait for an ssh worker to dial back before giving up +SSH_CONNECT_TIMEOUT = 60.0 - The remote runs the uv-provisioned worker; for a dev coordinator the - preamble carries the wheel bytes that the remote command receives. + +def _ssh_argv( + spec: Any, command: str, forward: tuple[str, str] | None = None +) -> list[str]: + """ssh/vagrant argv running ``command``, optionally with a ``-R`` forward. + + ``forward`` is ``(remote_socket, local_socket)``. ``StreamLocalBindUnlink`` + makes sshd replace a stale socket file rather than refuse to bind. """ from . import _provision - remote_command, preamble = _provision.ssh_remote_command(spec) + options: list[str] = [] + if forward is not None: + remote_sock, local_sock = forward + options += ["-R", f"{remote_sock}:{local_sock}"] + options += ["-o", "StreamLocalBindUnlink=yes"] + if spec.ssh is not None: + return _provision.ssh_argv(spec.ssh, spec.ssh_config, command, options) + assert spec.vagrant_ssh is not None + return _provision.vagrant_ssh_argv( + spec.vagrant_ssh, spec.ssh_config, command, options + ) + + +def ssh_transport_args(spec: Any) -> list[str]: + """ssh argv for a stdio-transport ssh worker.""" + from . import _provision + assert spec.ssh is not None - return _provision.ssh_argv(spec.ssh, spec.ssh_config, remote_command), preamble + return _ssh_argv(spec, _provision.ssh_remote_command(spec)) -def vagrant_transport_args(spec: Any) -> tuple[list[str], bytes]: - """``(vagrant ssh argv, stdin preamble)`` for a vagrant_ssh worker.""" +def vagrant_transport_args(spec: Any) -> list[str]: + """vagrant-ssh argv for a stdio-transport vagrant_ssh worker.""" from . import _provision - remote_command, preamble = _provision.ssh_remote_command(spec) assert spec.vagrant_ssh is not None - argv = _provision.vagrant_ssh_argv( - spec.vagrant_ssh, spec.ssh_config, remote_command + return _ssh_argv(spec, _provision.ssh_remote_command(spec)) + + +@asynccontextmanager +async def _dialback_listener() -> AsyncIterator[tuple[str, trio.SocketListener]]: + """A private unix socket the worker will be forwarded back to.""" + import shutil + import tempfile + + directory = tempfile.mkdtemp(prefix="execnet-dialback-") + os.chmod(directory, 0o700) + path = os.path.join(directory, "gw.sock") + sock = trio.socket.socket(trio.socket.AF_UNIX, trio.socket.SOCK_STREAM) + await sock.bind(path) + sock.listen(1) + listener = trio.SocketListener(sock) + try: + yield path, listener + finally: + with trio.CancelScope(shield=True), suppress(Exception): + await listener.aclose() + shutil.rmtree(directory, ignore_errors=True) + + +async def _accept_or_diagnose( + listener: trio.SocketListener, + process: trio.Process, + remoteaddress: str, +) -> trio.SocketStream: + """Await the worker's dial-back, or explain why it never came. + + With the stdio transport a dead ssh shows up as EOF on the protocol + pipe. Here there is no such pipe, so the accept is raced against the + process exiting -- ssh's 255 still means "could not reach or + authenticate the host". + """ + accepted: list[trio.SocketStream] = [] + exited: list[int] = [] + + async def accept(nursery: trio.Nursery) -> None: + accepted.append(await listener.accept()) + nursery.cancel_scope.cancel() + + async def watch(nursery: trio.Nursery) -> None: + exited.append(await process.wait()) + nursery.cancel_scope.cancel() + + with trio.move_on_after(SSH_CONNECT_TIMEOUT): + async with trio.open_nursery() as nursery: + nursery.start_soon(accept, nursery) + nursery.start_soon(watch, nursery) + if accepted: + return accepted[0] + with trio.CancelScope(shield=True), trio.move_on_after(5): + process.kill() + await process.wait() + if exited and exited[0] == 255: + raise HostNotFound(remoteaddress) + if exited: + raise EOFError( + f"ssh worker exited with {exited[0]} before connecting back" + f" to {remoteaddress}" + ) + raise EOFError( + f"ssh worker did not connect back within {SSH_CONNECT_TIMEOUT}s" + f" ({remoteaddress})" ) - return argv, preamble + + +async def connect_ssh_worker(spec: Any) -> tuple[ByteStream, trio.Process]: + """Spawn an ssh/vagrant worker on the transport ``spec`` resolves to. + + ``transport=socket`` forwards a private unix socket to the remote with + ``ssh -R`` and lets the worker dial back on it, which leaves ssh's + stdin free for the config (so it is not in the remote argv, where every + user on that host can read the ``env:`` values) and leaves the worker's + stdout and stderr free for the code it runs. + """ + from . import _provision + + remoteaddress = spec.ssh or spec.vagrant_ssh + assert remoteaddress is not None + wheel = _provision.ssh_wheel(spec) + if wheel is not None: + await deliver_remote_wheel(spec, wheel) + + if _provision.resolve_transport(spec) == "stdio": + args = ( + ssh_transport_args(spec) + if spec.ssh is not None + else vagrant_transport_args(spec) + ) + return await connect_command_worker(args, remoteaddress=remoteaddress) + + async with _dialback_listener() as (local_sock, listener): + remote_sock = f"/tmp/execnet-{uuid.uuid4().hex}.sock" + command = _provision.ssh_remote_command( + spec, + "--protocol-connect", + f"unix:{remote_sock}", + config_on_stdin=True, + ) + argv = _ssh_argv(spec, command, forward=(remote_sock, local_sock)) + # stdout/stderr stay inherited: remote output is the user's now. + process = await trio.lowlevel.open_process(argv, stdin=subprocess.PIPE) + try: + assert process.stdin is not None + await process.stdin.send_all(_provision.worker_cli_arg(spec).encode()) + await process.stdin.aclose() + stream = await _accept_or_diagnose(listener, process, remoteaddress) + await read_handshake_ack(stream, "ssh") + except BaseException: + with trio.CancelScope(shield=True), trio.move_on_after(5): + process.kill() + await process.wait() + raise + return stream, process + + +async def deliver_remote_wheel(spec: Any, wheel: Any) -> None: + """Ship ``wheel`` to the remote over its own connection, before launching. + + Out of band on purpose: the protocol stream never carries a payload, so + the launch command needs no ``head -c `` byte accounting and no + ``exec`` to keep an fd alive. Skipped remote-side when the file is + already cached there. + """ + from . import _provision + + argv = _ssh_argv(spec, _provision.wheel_delivery_command(wheel)) + process = await trio.lowlevel.open_process(argv, stdin=subprocess.PIPE) + try: + assert process.stdin is not None + await process.stdin.send_all(wheel.read_bytes()) + await process.stdin.aclose() + code = await process.wait() + except BaseException: + with trio.CancelScope(shield=True), trio.move_on_after(5): + process.kill() + await process.wait() + raise + if code != 0: + raise HostNotFound( + f"could not deliver the execnet wheel to {spec.ssh or spec.vagrant_ssh}" + f" (exit {code})" + ) async def connect_command_worker( @@ -817,6 +983,44 @@ async def connect_command_worker( return stream, process +async def connect_popen_worker(spec: Any) -> tuple[ByteStream, trio.Process]: + """Spawn a local worker for ``spec`` on its resolved transport. + + With ``transport=socket`` the protocol runs over an inherited + socketpair, so the child's stdin/stdout/stderr stay the user's: remote + ``print()`` reaches the terminal instead of being swallowed to keep the + wire clean. ``transport=stdio`` is the classic pipe pair. + """ + from . import _provision + + if _provision.resolve_transport(spec) == "stdio": + return await connect_command_worker(popen_worker_argv(spec)) + + import socket as _socket + + ours, theirs = _socket.socketpair() + try: + args = popen_worker_argv(spec, "--protocol-fd", str(theirs.fileno())) + process = await trio.lowlevel.open_process(args, pass_fds=(theirs.fileno(),)) + except BaseException: + ours.close() + theirs.close() + raise + # The child holds its own copy now; ours would keep the pair open. + theirs.close() + stream = trio.SocketStream(trio.socket.from_stdlib_socket(ours)) + try: + await read_handshake_ack(stream, "bootstrap") + except BaseException: + with trio.CancelScope(shield=True): + with trio.move_on_after(5): + process.kill() + await process.wait() + await stream.aclose() + raise + return stream, process + + async def connect_socket_worker( address: tuple[str, int], remoteaddress: str ) -> ByteStream: @@ -934,20 +1138,11 @@ async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: elif spec.socket: address, remoteaddress = await self._resolve_socket_address(spec) stream = await connect_socket_worker(address, remoteaddress) - elif spec.ssh: - args, preamble = ssh_transport_args(spec) - remoteaddress = spec.ssh - stream, process = await connect_command_worker( - args, preamble=preamble, remoteaddress=remoteaddress - ) - elif spec.vagrant_ssh: - args, preamble = vagrant_transport_args(spec) - remoteaddress = spec.vagrant_ssh - stream, process = await connect_command_worker( - args, preamble=preamble, remoteaddress=remoteaddress - ) + elif spec.ssh or spec.vagrant_ssh: + remoteaddress = spec.ssh or spec.vagrant_ssh + stream, process = await connect_ssh_worker(spec) elif spec.popen or spec.python: - stream, process = await connect_command_worker(popen_worker_argv(spec)) + stream, process = await connect_popen_worker(spec) else: raise ValueError(f"unsupported spec for AsyncGroup: {spec!r}") gateway = self._make_gateway(stream, spec) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 206f8081..192133f9 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -880,6 +880,23 @@ def start_socketserver_via( return realhost, int(realport) +async def _run_delivery_step(argv: list[str], payload: bytes) -> None: + """Run an out-of-band delivery command, feeding ``payload`` to its stdin.""" + process = await trio.lowlevel.open_process(argv, stdin=subprocess.PIPE) + try: + assert process.stdin is not None + await process.stdin.send_all(payload) + await process.stdin.aclose() + code = await process.wait() + except BaseException: + with trio.CancelScope(shield=True), trio.move_on_after(5): + process.kill() + await process.wait() + raise + if code != 0: + raise RuntimeError(f"delivery step failed with exit {code}: {argv[0]}") + + async def _start_sub_and_relay( gateway: BaseGateway, channelid: int, request: dict[str, Any] ) -> None: @@ -891,8 +908,8 @@ async def _start_sub_and_relay( one whole frame), while the sub's stdout runs through a FrameDecoder so every CHANNEL_DATA sent back carries exactly one frame -- except the initial ready byte, which is forwarded on its own for the handshake. - A stdin preamble (shipped wheel for a dev-version ssh sub) is streamed - before the relayed protocol bytes. + A shipped wheel (dev-version ssh sub) is delivered first, over its own + connection, so the relayed stream carries protocol bytes only. """ from . import _provision @@ -901,7 +918,9 @@ def send_close_error(text: str) -> None: gateway._send(Message.CHANNEL_CLOSE_ERROR, channelid, dumps_internal(text)) try: - args, preamble = _provision.sub_spawn_argv(request) + args, delivery = _provision.sub_spawn_argv(request) + if delivery is not None: + await _run_delivery_step(*delivery) process = await open_popen_process(args) except Exception as exc: send_close_error(f"could not spawn via sub-gateway: {exc}") @@ -912,8 +931,6 @@ def send_close_error(text: str) -> None: async def coordinator_to_sub() -> None: assert process.stdin is not None - if preamble: - await process.stdin.send_all(preamble) with suppress(RemoteError): async for data in raw: await process.stdin.send_all(data) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 0550dfc0..50faccb4 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -90,6 +90,9 @@ async def run(self, channel: Channel, item: ExecItem) -> None: self._primary.put((channel, item, done)) await trio.to_thread.run_sync(done.wait, abandon_on_cancel=True) + def released(self) -> None: + """Hook: the main thread is free again (called on it, before done).""" + def integrate_as_primary_thread(self) -> None: """Block the main thread running exec tasks until shutdown.""" while True: @@ -100,6 +103,9 @@ def integrate_as_primary_thread(self) -> None: try: self.gateway.executetask((channel, item)) finally: + # Release before signalling: the next request should see the + # main thread free as early as we can make it. + self.released() done.set() def trigger_shutdown(self) -> None: @@ -119,6 +125,16 @@ class HybridExec(PrimaryThreadPump): it existed for the main-thread guarantee, which the claim provides, and its extra behaviour -- refusing a second concurrent remote_exec rather than overflowing -- was a deadlock guard, not a feature. + + One difference from that profile is worth knowing: it *serialized*, so + every sequential remote_exec was guaranteed the main thread. Here the + claim is released as the exec finishes, while the channel close that + tells the coordinator it may send the next one is emitted a moment + earlier -- so a coordinator that immediately re-execs can, rarely, be + admitted before the release lands and get a pool thread instead. The + *first* request always gets the main thread; a caller that needs the + guarantee for every request wants the ``trio`` or ``gevent`` profile, + where placement is not a race. """ def __init__(self, gateway: WorkerGateway) -> None: @@ -135,6 +151,10 @@ async def admit(self, channel: Channel, item: ExecItem) -> bool: self._claimed.add(channel.id) return True + def released(self) -> None: + with self._claim_lock: + self._primary_busy = False + async def run(self, channel: Channel, item: ExecItem) -> None: with self._claim_lock: claimed = channel.id in self._claimed @@ -145,8 +165,9 @@ async def run(self, channel: Channel, item: ExecItem) -> None: try: await super().run(channel, item) finally: - with self._claim_lock: - self._primary_busy = False + # released() normally cleared this on the main thread already; + # repeat it so a cancelled or failed run cannot strand the claim + self.released() class GreenletExec: diff --git a/src/execnet/_xspec.py b/src/execnet/_xspec.py index f13d0521..1d2b3af2 100644 --- a/src/execnet/_xspec.py +++ b/src/execnet/_xspec.py @@ -32,6 +32,10 @@ class XSpec: socket: str | None = None ssh: str | None = None ssh_config: str | None = None + stderr: str | None = None + stdin: str | None = None + stdout: str | None = None + transport: str | None = None vagrant_ssh: str | None = None via: str | None = None diff --git a/testing/test_cli.py b/testing/test_cli.py new file mode 100644 index 00000000..0ef91a67 --- /dev/null +++ b/testing/test_cli.py @@ -0,0 +1,395 @@ +"""The ``execnet`` command line, and the transports it exposes. + +``execnet worker`` is the launch contract between a coordinator and the +process it starts. These tests drive it the way a coordinator does -- +including the transports that keep the protocol off the worker's stdio. +""" + +from __future__ import annotations + +import json +import os +import shutil +import socket +import subprocess +import sys +import tempfile +from collections.abc import Iterator + +import pytest + +import execnet +from execnet import _cli +from execnet import _provision + +posix_only = pytest.mark.skipif( + sys.platform.startswith("win"), reason="needs POSIX fd passing / unix sockets" +) + +TESTTIMEOUT = 30.0 + + +def worker_config(**overrides: object) -> str: + config = { + "id": "cli-test-worker", + "profile": "thread", + "execmodel": "thread", + "wait": "thread", + "coordinator_version": execnet.__version__, + } + config.update(overrides) + return json.dumps(config) + + +class TestInfo: + def test_info_reports_what_a_coordinator_needs(self) -> None: + out = subprocess.run( + [sys.executable, "-m", "execnet", "info"], + capture_output=True, + text=True, + check=True, + ) + info = json.loads(out.stdout) + assert info["execnet"] == execnet.__version__ + assert info["trio"] is not None + assert info["executable"] + assert "stdio" in info["protocols"] + + def test_info_matches_the_in_process_view(self) -> None: + assert _cli.interpreter_info()["execnet"] == execnet.__version__ + + def test_probe_uses_info(self) -> None: + _provision.target_info.cache_clear() + info = _provision.target_info(sys.executable) + assert info is not None + assert info["execnet"] == execnet.__version__ + assert _provision.target_has_execnet(sys.executable) + + def test_probe_rejects_an_interpreter_without_execnet(self, tmp_path) -> None: + # a python that cannot import execnet fails the probe, which is what + # sends it down the uv-provisioning path + _provision.target_info.cache_clear() + fake = tmp_path / "fake-python" + fake.write_text("#!/bin/sh\nexit 1\n") + fake.chmod(0o755) + assert _provision.target_info(str(fake)) is None + assert not _provision.target_has_execnet(str(fake)) + + +class TestArgumentGrammar: + def test_protocol_flags_are_mutually_exclusive(self) -> None: + with pytest.raises(SystemExit): + _cli._build_parser().parse_args( + ["worker", "--protocol-fd", "3", "--protocol-connect", "unix:/x"] + ) + + def test_config_sources_are_mutually_exclusive(self) -> None: + with pytest.raises(SystemExit): + _cli._build_parser().parse_args( + ["worker", "--config", "{}", "--config-fd", "0"] + ) + + @pytest.mark.parametrize( + ("value", "expected"), [("3", (3,)), ("4,5", (4, 5))] + ) + def test_protocol_fd_accepts_one_fd_or_a_pair( + self, value: str, expected: tuple[int, ...] + ) -> None: + ns = _cli._build_parser().parse_args(["worker", "--protocol-fd", value]) + assert ns.protocol_fd == expected + + def test_protocol_fd_rejects_nonsense(self) -> None: + with pytest.raises(SystemExit): + _cli._build_parser().parse_args(["worker", "--protocol-fd", "a,b,c"]) + + @pytest.mark.parametrize( + ("address", "expected"), + [ + ("unix:/tmp/x.sock", ("unix", "/tmp/x.sock")), + ("localhost:8888", ("tcp", ("localhost", 8888))), + (":8888", ("tcp", ("localhost", 8888))), + ], + ) + def test_parse_address(self, address: str, expected: tuple) -> None: + from execnet._trio_worker import parse_address + + assert parse_address(address) == expected + + def test_parse_address_rejects_a_bare_path(self) -> None: + from execnet._trio_worker import parse_address + + with pytest.raises(ValueError, match="unix:/path or host:port"): + parse_address("/tmp/x.sock") + + +@posix_only +class TestProtocolFd: + """A worker serving over an inherited socket, driven by hand.""" + + def test_socketpair_roundtrip(self) -> None: + ours, theirs = socket.socketpair() + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "execnet", + "worker", + "--protocol-fd", + str(theirs.fileno()), + "--config", + worker_config(), + ], + pass_fds=(theirs.fileno(),), + ) + theirs.close() + try: + assert ours.recv(1) == b"1" # the ready handshake + finally: + ours.close() + proc.terminate() + proc.wait(timeout=TESTTIMEOUT) + + def test_a_plain_pipe_fd_is_rejected(self) -> None: + # one fd has to be bidirectional; a pipe end needs the read,write form + read_fd, write_fd = os.pipe() + try: + out = subprocess.run( + [ + sys.executable, + "-m", + "execnet", + "worker", + "--protocol-fd", + str(read_fd), + "--config", + worker_config(), + ], + pass_fds=(read_fd,), + capture_output=True, + text=True, + timeout=TESTTIMEOUT, + ) + finally: + os.close(read_fd) + os.close(write_fd) + assert out.returncode != 0 + assert "not a socket" in out.stderr + + +class TestConfigSources: + @posix_only + def test_config_fd_keeps_it_out_of_argv(self) -> None: + ours, theirs = socket.socketpair() + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "execnet", + "worker", + "--protocol-fd", + str(theirs.fileno()), + "--config-fd", + "0", + ], + pass_fds=(theirs.fileno(),), + stdin=subprocess.PIPE, + ) + theirs.close() + try: + assert proc.stdin is not None + proc.stdin.write(worker_config().encode()) + proc.stdin.close() + assert ours.recv(1) == b"1" + finally: + ours.close() + proc.terminate() + proc.wait(timeout=TESTTIMEOUT) + + def test_config_file(self, tmp_path) -> None: + path = tmp_path / "config.json" + path.write_text(worker_config()) + ns = _cli._build_parser().parse_args( + ["worker", "--config-file", str(path)] + ) + assert _cli._load_config(ns)["id"] == "cli-test-worker" + + def test_no_config_source_is_an_error(self) -> None: + ns = _cli._build_parser().parse_args(["worker"]) + with pytest.raises(SystemExit, match="--config"): + _cli._load_config(ns) + + +class TestTransportSelection: + def test_defaults_per_platform(self) -> None: + spec = execnet.XSpec("popen") + expected = "stdio" if sys.platform.startswith("win") else "socket" + assert _provision.resolve_transport(spec) == expected + + def test_explicit_wins(self) -> None: + assert _provision.resolve_transport(execnet.XSpec("popen//transport=stdio")) == ( + "stdio" + ) + + def test_unknown_is_rejected(self) -> None: + with pytest.raises(ValueError, match="unknown transport"): + _provision.resolve_transport(execnet.XSpec("popen//transport=carrier-pigeon")) + + @posix_only + def test_socket_transport_keeps_the_protocol_off_stdio(self) -> None: + group = execnet.Group() + try: + gateway = group.makegateway("popen") + channel = gateway.remote_exec( + "import sys; channel.send(sys.argv)", + ) + argv = channel.receive(TESTTIMEOUT) + assert "--protocol-fd" in argv + finally: + group.terminate(timeout=5.0) + + def test_stdio_transport_still_works(self) -> None: + group = execnet.Group() + try: + gateway = group.makegateway("popen//transport=stdio") + channel = gateway.remote_exec("channel.send(6 * 7)") + assert channel.receive(TESTTIMEOUT) == 42 + finally: + group.terminate(timeout=5.0) + + +class TestWorkerStdio: + """Whose stdio is it? The code the worker runs, unless told otherwise.""" + + @posix_only + def test_socket_transport_inherits_stdio(self, capfd) -> None: + group = execnet.Group() + try: + gateway = group.makegateway("popen") + gateway.remote_exec("print('INHERITED-STDOUT')").waitclose(TESTTIMEOUT) + finally: + group.terminate(timeout=5.0) + out, _ = capfd.readouterr() + assert "INHERITED-STDOUT" in out + + def test_stdio_transport_folds_stdout_onto_stderr(self, capfd) -> None: + # the protocol owns fd 1 here, so remote output cannot go there -- + # but it lands on stderr rather than being discarded + group = execnet.Group() + try: + gateway = group.makegateway("popen//transport=stdio") + gateway.remote_exec("print('FOLDED-ONTO-STDERR')").waitclose(TESTTIMEOUT) + finally: + group.terminate(timeout=5.0) + out, err = capfd.readouterr() + assert "FOLDED-ONTO-STDERR" not in out + assert "FOLDED-ONTO-STDERR" in err + + @posix_only + def test_stdin_can_be_sent_to_devnull(self) -> None: + group = execnet.Group() + try: + gateway = group.makegateway("popen//stdin=devnull") + channel = gateway.remote_exec( + "import os; channel.send(os.readlink('/proc/self/fd/0'))" + ) + assert channel.receive(TESTTIMEOUT) == "/dev/null" + finally: + group.terminate(timeout=5.0) + + @posix_only + def test_stdout_can_be_silenced(self, capfd) -> None: + group = execnet.Group() + try: + gateway = group.makegateway("popen//stdout=devnull") + gateway.remote_exec("print('SILENCED')").waitclose(TESTTIMEOUT) + finally: + group.terminate(timeout=5.0) + out, err = capfd.readouterr() + assert "SILENCED" not in out + assert "SILENCED" not in err + + +@posix_only +class TestListenTransport: + def test_worker_listens_and_reports_its_address(self) -> None: + directory = tempfile.mkdtemp(prefix="execnet-cli-test-") + path = os.path.join(directory, "gw.sock") + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "execnet", + "worker", + "--protocol-listen", + f"unix:{path}", + "--config", + worker_config(), + ], + stdout=subprocess.PIPE, + ) + try: + assert proc.stdout is not None + announced = json.loads(proc.stdout.readline()) + assert announced == {"listening": path} + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client.settimeout(TESTTIMEOUT) + client.connect(path) + try: + assert client.recv(1) == b"1" + finally: + client.close() + finally: + proc.terminate() + proc.wait(timeout=TESTTIMEOUT) + shutil.rmtree(directory, ignore_errors=True) + + +class TestServerCommand: + def test_server_is_the_socketserver(self) -> None: + parser = _cli._build_parser() + ns = parser.parse_args(["server", "127.0.0.1:0", "--once"]) + assert ns.hostport == "127.0.0.1:0" + assert ns.once is True + + def test_socketserver_alias_warns(self, monkeypatch: pytest.MonkeyPatch) -> None: + called: list[list[str]] = [] + monkeypatch.setattr(_cli, "main", lambda argv: called.append(argv)) + with pytest.warns(DeprecationWarning, match="execnet server"): + _cli.socketserver_main([":0", "--once"]) + assert called == [["server", ":0", "--once"]] + + +class TestRemoteCommand: + """What ends up on the remote host's command line.""" + + def test_config_is_not_in_the_remote_argv(self) -> None: + # env: values are secrets often enough; the remote argv is readable + # by every user on that host via ps + spec = execnet.XSpec("ssh=host//id=gw0//env:TOKEN=s3cr3t") + spec.profile = "thread" + command = _provision.ssh_remote_command( + spec, "--protocol-connect", "unix:/tmp/x.sock", config_on_stdin=True + ) + assert "s3cr3t" not in command + assert "--config-fd 0" in command + + def test_launch_command_frames_nothing_in_band(self) -> None: + spec = execnet.XSpec("ssh=host//id=gw0") + spec.profile = "thread" + command = _provision.ssh_remote_command(spec) + # the wheel travels on its own connection now + assert "head -c" not in command + assert "mktemp -d" not in command + + def test_dialback_argv_forwards_a_unix_socket(self) -> None: + from execnet import _trio_gateway + + spec = execnet.XSpec("ssh=host//id=gw0") + spec.profile = "thread" + argv = _trio_gateway._ssh_argv( + spec, "worker-cmd", forward=("/tmp/remote.sock", "/tmp/local.sock") + ) + assert "-R" in argv + assert argv[argv.index("-R") + 1] == "/tmp/remote.sock:/tmp/local.sock" + # a stale remote socket must not block the bind + assert "StreamLocalBindUnlink=yes" in argv diff --git a/testing/test_gateway.py b/testing/test_gateway.py index 86ee74f3..ac845b38 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -432,15 +432,12 @@ def test_ssh_trio_args_include_config( from execnet import _trio_host monkeypatch.setattr( - _provision, "ssh_remote_command", lambda spec: ("worker-cmd", b"") - ) - args, preamble = _trio_host.ssh_trio_args( - execnet.XSpec("ssh=xyz//ssh_config=qwe") + _provision, "ssh_remote_command", lambda spec, *a, **kw: "worker-cmd" ) + args = _trio_host.ssh_trio_args(execnet.XSpec("ssh=xyz//ssh_config=qwe")) assert args[args.index("-F") + 1] == "qwe" assert "xyz" in args assert args[-1] == "worker-cmd" - assert preamble == b"" def test_sshaddress(self, gw: Gateway, specssh: execnet.XSpec) -> None: assert gw.remoteaddress == specssh.ssh @@ -581,15 +578,17 @@ def test_popen_args(spec: str, expected_args: list[str]) -> None: assert args[len(expected_args) :][:4] == ["-u", "-m", "execnet", "worker"] -def test_sequential_remote_exec_claims_the_main_thread( +def test_first_remote_exec_claims_the_main_thread( makegateway: Callable[[str], Gateway], ) -> None: - # The `thread` profile hands each request the worker main thread while - # it is idle -- the GUI/signal-safety property `main_thread_only` - # existed for. Sequential remote_execs therefore all land there. + # The `thread` profile hands the first request the worker main thread + # -- the GUI/signal-safety property `main_thread_only` existed for. + # FIFO admission makes that one deterministic; a sequential *re*-exec + # races the claim release (see HybridExec), so only the first is + # asserted here. gw = makegateway("profile=thread//popen") try: - for _ in range(10): + for _ in range(1): ch = gw.remote_exec( """ import time, threading diff --git a/testing/test_provision.py b/testing/test_provision.py index d7f01796..0d708b93 100644 --- a/testing/test_provision.py +++ b/testing/test_provision.py @@ -24,20 +24,40 @@ def test_worker_cli_arg_carries_config() -> None: def test_ssh_remote_command_released(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(execnet, "__version__", "9.9.9") spec = execnet.XSpec("ssh=host//id=gw0//execmodel=thread") - command, preamble = _provision.ssh_remote_command(spec) + command = _provision.ssh_remote_command(spec) assert "execnet==9.9.9" in command - assert "head -c" not in command # no shipping - assert preamble == b"" + assert "head -c" not in command # nothing is framed into the launch @pytest.mark.skipif(released, reason="released execnet resolves from an index") @pytest.mark.skipif(not _provision.uv_available(), reason="uv required to build wheel") -def test_ssh_remote_command_dev_ships_wheel() -> None: +def test_ssh_remote_command_dev_uses_a_delivered_wheel() -> None: + # The wheel travels out of band now (its own connection, before the + # launch), so the launch command just points uv at where it landed -- + # no byte accounting, no `exec` to keep an fd alive. spec = execnet.XSpec("ssh=host//id=gw0//execmodel=thread") - command, preamble = _provision.ssh_remote_command(spec) - assert "mktemp -d" in command - assert f"head -c {len(preamble)}" in command - assert preamble[:2] == b"PK" # a wheel is a zip archive + command = _provision.ssh_remote_command(spec) + wheel = _provision.ssh_wheel(spec) + assert wheel is not None + assert wheel.read_bytes()[:2] == b"PK" # a wheel is a zip archive + assert _provision.remote_wheel_path(wheel) in command + assert "head -c" not in command + assert "mktemp -d" not in command + + +@pytest.mark.skipif(released, reason="released execnet resolves from an index") +@pytest.mark.skipif(not _provision.uv_available(), reason="uv required to build wheel") +def test_wheel_delivery_command_expands_home_and_drains_stdin() -> None: + wheel = _provision.ssh_wheel(execnet.XSpec("ssh=host//id=gw0")) + assert wheel is not None + command = _provision.wheel_delivery_command(wheel) + # $HOME must stay expandable: quoting it as a literal would create a + # directory actually named "~" + assert '"$HOME"' in command + assert "~" not in command + # the coordinator always streams the wheel, so the cached branch has + # to consume stdin too or it hands the coordinator an EPIPE + assert "cat > /dev/null" in command def test_vagrant_ssh_argv() -> None: @@ -59,7 +79,7 @@ def test_vagrant_ssh_argv() -> None: def test_sub_spawn_argv_plain_popen() -> None: import sys - argv, preamble = _provision.sub_spawn_argv({"config": "{}"}) + argv, delivery = _provision.sub_spawn_argv({"config": "{}"}) assert argv == [ sys.executable, "-u", @@ -69,7 +89,7 @@ def test_sub_spawn_argv_plain_popen() -> None: "--config", "{}", ] - assert preamble == b"" + assert delivery is None def test_sub_spawn_argv_vagrant_released() -> None: @@ -78,7 +98,7 @@ def test_sub_spawn_argv_vagrant_released() -> None: "vagrant_ssh": "default", "requirement": "execnet==9.9.9", } - argv, preamble = _provision.sub_spawn_argv(request) + argv, delivery = _provision.sub_spawn_argv(request) assert argv[:5] == ["vagrant", "ssh", "default", "--", "-C"] assert "execnet==9.9.9" in argv[-1] - assert preamble == b"" + assert delivery is None diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py index 443b5da2..764ad12a 100644 --- a/testing/test_ssh_local.py +++ b/testing/test_ssh_local.py @@ -31,6 +31,19 @@ CLIENT_PUBKEY = SSHKEYS / "insecure_client_ed25519.pub" +class _ForwardingSSHServer(asyncssh.SSHServer): # type: ignore[misc] + """Accepts ``-R`` unix-socket forwards, which the socket transport needs. + + execnet's ssh worker dials back to the coordinator over a unix socket + that ``ssh -R`` forwards; asyncssh refuses such requests unless the + server opts in, and returning True asks it to do the standard + forwarding. + """ + + def unix_server_requested(self, listen_path: str) -> bool: + return True + + class SSHServerThread: """asyncssh server on an ephemeral port, driven from its own asyncio thread.""" @@ -64,6 +77,7 @@ async def _serve(self) -> None: 0, server_host_keys=[str(HOST_KEY)], authorized_client_keys=str(CLIENT_PUBKEY), + server_factory=_ForwardingSSHServer, process_factory=self._handle, encoding=None, # binary stdio ) From e43db6324e81f61995fb41c25fecfc8a440e2e1a Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 30 Jul 2026 12:32:36 +0200 Subject: [PATCH 69/91] refactor: one typed endmarker sentinel, collapsing make_receive_queue make_receive_queue branched on its endmarker only to decide between calling setcallback with and without it -- which looks redundant, because setcallback's default *is* NO_ENDMARKER_WANTED. It was not: _multi and _channel each defined their own `object()` under that name, and the consumer task compares against _channel's. Passing _multi's through would have been read as a real endmarker and queued a bare object. So share one sentinel and name its type. It is an enum member rather than a bare object() so it can appear in an annotation: an endmarker may be any object, identity is the only thing separating "none wanted" from a legitimate one, and `Endmarker = object | Literal[NoEndmarker.NOT_WANTED]` now says on every signature which sentinel a caller has to hand back. With one definition the branch collapses to a plain forward. Co-Authored-By: Claude Opus 5 --- src/execnet/_channel.py | 24 ++++++++++++++++++++++-- src/execnet/_gateway_base.py | 3 ++- src/execnet/_multi.py | 12 ++++-------- src/execnet/_trio_host.py | 5 +++-- testing/test_cli.py | 27 +++++++++++++-------------- 5 files changed, 44 insertions(+), 27 deletions(-) diff --git a/src/execnet/_channel.py b/src/execnet/_channel.py index 98c5d862..611dbe2f 100644 --- a/src/execnet/_channel.py +++ b/src/execnet/_channel.py @@ -8,6 +8,7 @@ from __future__ import annotations +import enum import threading import weakref from collections.abc import Callable @@ -30,7 +31,26 @@ if TYPE_CHECKING: from ._gateway_base import BaseGateway -NO_ENDMARKER_WANTED = object() + +class NoEndmarker(enum.Enum): + """Type of the "no endmarker wanted" sentinel. + + An enum rather than a bare ``object()`` so it is nameable in an + annotation: an endmarker may be *any* object, so the only thing that + distinguishes "none wanted" from a legitimate endmarker is identity, + and a second module inventing its own ``object()`` for it would be + silently wrong. ``endmarker: object | Literal[NoEndmarker.NOT_WANTED]`` + says which sentinel a caller has to hand back. + """ + + NOT_WANTED = enum.auto() + + +NO_ENDMARKER_WANTED = NoEndmarker.NOT_WANTED + +#: what an ``endmarker=`` parameter accepts: any object to deliver at the +#: end, or the sentinel meaning "do not deliver one" +Endmarker = object | Literal[NoEndmarker.NOT_WANTED] class Channel: @@ -82,7 +102,7 @@ def _trace(self, *msg: object) -> None: def setcallback( self, callback: Callable[[Any], Any], - endmarker: object = NO_ENDMARKER_WANTED, + endmarker: Endmarker = NO_ENDMARKER_WANTED, ) -> None: """Set a callback function for receiving items. diff --git a/src/execnet/_gateway_base.py b/src/execnet/_gateway_base.py index 11920dae..773ba314 100644 --- a/src/execnet/_gateway_base.py +++ b/src/execnet/_gateway_base.py @@ -27,6 +27,7 @@ from ._boundary import make_wakener from ._channel import Channel from ._channel import ChannelFactory +from ._channel import Endmarker from ._errors import INTERRUPT_TEXT from ._errors import geterrortext from ._errors import sysex @@ -109,7 +110,7 @@ def _start_channel_consumer( self, channel: Channel, callback: Callable[[Any], Any], - endmarker: object, + endmarker: Endmarker, ) -> None: """Attach a receiver callback: hand the channel to a consumer task. diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py index a2b36505..293be4da 100644 --- a/src/execnet/_multi.py +++ b/src/execnet/_multi.py @@ -25,7 +25,9 @@ from typing import overload from ._boundary import WaitBackend +from ._channel import NO_ENDMARKER_WANTED from ._channel import Channel +from ._channel import Endmarker from ._execmodel import ExecModel from ._execmodel import get_execmodel from ._execmodel import resolve_profile @@ -39,9 +41,6 @@ from ._gateway import Gateway -NO_ENDMARKER_WANTED = object() - - class Group: """Gateway Group.""" @@ -379,7 +378,7 @@ def receive_each( l.append(obj) return l - def make_receive_queue(self, endmarker: object = NO_ENDMARKER_WANTED): + def make_receive_queue(self, endmarker: Endmarker = NO_ENDMARKER_WANTED): try: return self._queue # type: ignore[has-type] except AttributeError: @@ -391,10 +390,7 @@ def make_receive_queue(self, endmarker: object = NO_ENDMARKER_WANTED): def putreceived(obj, channel: Channel = ch) -> None: self._queue.put((channel, obj)) # type: ignore[union-attr] - if endmarker is NO_ENDMARKER_WANTED: - ch.setcallback(putreceived) - else: - ch.setcallback(putreceived, endmarker=endmarker) + ch.setcallback(putreceived, endmarker=endmarker) return self._queue def waitclose(self) -> None: diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 192133f9..dc5214b6 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -28,6 +28,7 @@ from ._boundary import Flag from ._channel import ENDMARKER from ._channel import NO_ENDMARKER_WANTED +from ._channel import Endmarker from ._errors import GatewayReceivedTerminate from ._errors import RemoteError from ._execmodel import ExecModel @@ -315,7 +316,7 @@ def attach_consumer( self, channel: Any, callback: Callable[[Any], Any], - endmarker: object, + endmarker: Endmarker, ) -> None: """Switch ``channel`` to callback mode: a loop task drains it. @@ -394,7 +395,7 @@ async def _run_consumer( channel: Any, inbox: trio.MemoryReceiveChannel[bytes], callback: Callable[[Any], Any], - endmarker: object, + endmarker: Endmarker, done: Flag, ) -> None: """Drain ``inbox`` into ``callback`` (each call off the loop thread). diff --git a/testing/test_cli.py b/testing/test_cli.py index 0ef91a67..ba9d279a 100644 --- a/testing/test_cli.py +++ b/testing/test_cli.py @@ -14,7 +14,7 @@ import subprocess import sys import tempfile -from collections.abc import Iterator +from typing import Any import pytest @@ -30,7 +30,7 @@ def worker_config(**overrides: object) -> str: - config = { + config: dict[str, object] = { "id": "cli-test-worker", "profile": "thread", "execmodel": "thread", @@ -89,9 +89,7 @@ def test_config_sources_are_mutually_exclusive(self) -> None: ["worker", "--config", "{}", "--config-fd", "0"] ) - @pytest.mark.parametrize( - ("value", "expected"), [("3", (3,)), ("4,5", (4, 5))] - ) + @pytest.mark.parametrize(("value", "expected"), [("3", (3,)), ("4,5", (4, 5))]) def test_protocol_fd_accepts_one_fd_or_a_pair( self, value: str, expected: tuple[int, ...] ) -> None: @@ -110,7 +108,7 @@ def test_protocol_fd_rejects_nonsense(self) -> None: (":8888", ("tcp", ("localhost", 8888))), ], ) - def test_parse_address(self, address: str, expected: tuple) -> None: + def test_parse_address(self, address: str, expected: tuple[str, Any]) -> None: from execnet._trio_worker import parse_address assert parse_address(address) == expected @@ -168,6 +166,7 @@ def test_a_plain_pipe_fd_is_rejected(self) -> None: capture_output=True, text=True, timeout=TESTTIMEOUT, + check=False, ) finally: os.close(read_fd) @@ -208,9 +207,7 @@ def test_config_fd_keeps_it_out_of_argv(self) -> None: def test_config_file(self, tmp_path) -> None: path = tmp_path / "config.json" path.write_text(worker_config()) - ns = _cli._build_parser().parse_args( - ["worker", "--config-file", str(path)] - ) + ns = _cli._build_parser().parse_args(["worker", "--config-file", str(path)]) assert _cli._load_config(ns)["id"] == "cli-test-worker" def test_no_config_source_is_an_error(self) -> None: @@ -226,13 +223,15 @@ def test_defaults_per_platform(self) -> None: assert _provision.resolve_transport(spec) == expected def test_explicit_wins(self) -> None: - assert _provision.resolve_transport(execnet.XSpec("popen//transport=stdio")) == ( - "stdio" - ) + assert _provision.resolve_transport( + execnet.XSpec("popen//transport=stdio") + ) == ("stdio") def test_unknown_is_rejected(self) -> None: with pytest.raises(ValueError, match="unknown transport"): - _provision.resolve_transport(execnet.XSpec("popen//transport=carrier-pigeon")) + _provision.resolve_transport( + execnet.XSpec("popen//transport=carrier-pigeon") + ) @posix_only def test_socket_transport_keeps_the_protocol_off_stdio(self) -> None: @@ -353,7 +352,7 @@ def test_server_is_the_socketserver(self) -> None: def test_socketserver_alias_warns(self, monkeypatch: pytest.MonkeyPatch) -> None: called: list[list[str]] = [] - monkeypatch.setattr(_cli, "main", lambda argv: called.append(argv)) + monkeypatch.setattr(_cli, "main", called.append) with pytest.warns(DeprecationWarning, match="execnet server"): _cli.socketserver_main([":0", "--once"]) assert called == [["server", ":0", "--once"]] From 183980082487f6669b56b14beb6cee92476b6f67 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 30 Jul 2026 16:09:51 +0200 Subject: [PATCH 70/91] fix(ci): install the declared test deps, so the suite actually runs CI has been running *zero* tests since the ssh harness landed on 2026-07-26. test_ssh_local.py imports asyncssh at module level, tox's [testenv] deps listed only pytest and pytest-timeout, and a collection error aborts the whole session -- every job reported "2 skipped, 1 error" and exited 2. Two lists of test requirements had drifted: pyproject declares asyncssh and hypothesis in the testing extra, tox re-listed a subset by hand. So tox installs the extra rather than its own copy, and the extra is now just what running the suite needs; the maintainer tooling (pre-commit, tox, hatch) moves to the dev group. dev also absorbs the testing group, and [tool.uv] default-groups goes away -- it named only "testing", which *replaced* uv's default of "dev" and meant a plain `uv sync` never installed pytest-xdist. Turning the suite back on surfaced failures in tests CI had never executed, all from one condition: a dev-version execnet installed from a wheel has no editable source tree to build a provisioning wheel from, which is exactly what `tox --installpkg` produces. That is a real limitation, so those tests now skip on _provision.provisioning_available() rather than fail. test_ssh_local additionally skips on Windows, where its harness runs POSIX shell commands, and guards its asyncssh import. test_close_initiating_remote_no_error asserts execnet's teardown prints nothing; it now drops PYTHONWARNDEFAULTENCODING for the child, because that variable makes trio's subprocess module emit an EncodingWarning we cannot fix and which is not what the test guards. Verified with `tox run -e py`, the CI command: 510 passed, 76 skipped. Co-Authored-By: Claude Opus 5 --- pyproject.toml | 20 +++++++++++--------- src/execnet/_provision.py | 17 +++++++++++++++++ testing/test_cli.py | 9 +++++++++ testing/test_multi.py | 5 +++++ testing/test_provision.py | 8 ++++++++ testing/test_ssh_local.py | 30 +++++++++++++++++++++++++----- testing/test_termination.py | 11 ++++++++++- tox.ini | 6 +++--- uv.lock | 22 +++++++++++----------- 9 files changed, 99 insertions(+), 29 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e321346c..9c35cd16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,27 +46,31 @@ execnet-socketserver = "execnet._cli:socketserver_main" gevent = [ "gevent", ] +# what running the test suite needs -- tox installs exactly this, so there +# is one list rather than two that drift (asyncssh and hypothesis were +# missing from tox's own list, which failed collection in CI) testing = [ - "pre-commit", "pytest>8.0", "pytest-timeout", "hypothesis", - "tox", - "hatch", - "uv", "asyncssh", + "uv", ] [dependency-groups] +# the default group: everything a working checkout wants. Keeping the +# test requirements here rather than in a separate non-default group means +# a plain `uv sync` can run the suite. dev = [ + "execnet[testing]", "pytest-xdist>=3.8.0", + "pre-commit", + "tox", + "hatch", ] gevent = [ "gevent>=26.7.0", ] -testing = [ - "execnet[testing]", -] [project.urls] Homepage = "https://execnet.readthedocs.io/en/latest/" @@ -125,8 +129,6 @@ include = [ "/testing", "tox.ini", ] -[tool.uv] -default-groups = ["testing"] [tool.mypy] python_version = "3.10" mypy_path = ["src"] diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index 22427b1a..ff9966b2 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -194,6 +194,23 @@ def _build_wheel(version: str) -> Path: return wheel +def provisioning_available() -> bool: + """Whether this coordinator can produce material to provision a remote. + + A released version resolves from an index. A dev version has to build + a wheel, which needs the editable checkout it came from -- so a dev + version installed *from a wheel* (what ``tox --installpkg`` produces, + and what a CI artifact test runs against) can do neither. Callers that + need a remote worker should check this rather than let + :func:`coordinator_requirement` raise. + """ + import execnet + + if _RELEASED_RE.match(execnet.__version__): + return True + return _editable_source_root() is not None + + def coordinator_requirement() -> str: """A ``uv --with`` requirement that installs this coordinator's execnet.""" import execnet diff --git a/testing/test_cli.py b/testing/test_cli.py index ba9d279a..3ab45008 100644 --- a/testing/test_cli.py +++ b/testing/test_cli.py @@ -358,9 +358,18 @@ def test_socketserver_alias_warns(self, monkeypatch: pytest.MonkeyPatch) -> None assert called == [["server", ":0", "--once"]] +needs_provisioning = pytest.mark.skipif( + not _provision.provisioning_available(), + reason="a dev execnet installed without its source tree cannot build a" + " wheel to provision a remote with", +) + + class TestRemoteCommand: """What ends up on the remote host's command line.""" + pytestmark = needs_provisioning + def test_config_is_not_in_the_remote_argv(self) -> None: # env: values are secrets often enough; the remote argv is readable # by every user on that host via ps diff --git a/testing/test_multi.py b/testing/test_multi.py index 156879b3..54eb858b 100644 --- a/testing/test_multi.py +++ b/testing/test_multi.py @@ -15,6 +15,7 @@ from execnet import Gateway from execnet import Group from execnet import XSpec +from execnet import _provision from execnet._channel import Channel from execnet._execmodel import ExecModel from execnet._multi import safe_terminate @@ -232,6 +233,10 @@ def test_terminate_with_proxying(self) -> None: group.makegateway("popen//via=master//id=worker") group.terminate(1.0) + @pytest.mark.skipif( + not _provision.provisioning_available(), + reason="a via sub-spec ships provisioning material eagerly", + ) def test_via_foreign_python(self) -> None: # A python= sub-spec through a via master: the master resolves the # interpreter locally (this interpreter has execnet, so the sub runs diff --git a/testing/test_provision.py b/testing/test_provision.py index 0d708b93..008aae29 100644 --- a/testing/test_provision.py +++ b/testing/test_provision.py @@ -31,6 +31,10 @@ def test_ssh_remote_command_released(monkeypatch: pytest.MonkeyPatch) -> None: @pytest.mark.skipif(released, reason="released execnet resolves from an index") @pytest.mark.skipif(not _provision.uv_available(), reason="uv required to build wheel") +@pytest.mark.skipif( + not _provision.provisioning_available(), + reason="a dev execnet installed without its source tree cannot build a wheel", +) def test_ssh_remote_command_dev_uses_a_delivered_wheel() -> None: # The wheel travels out of band now (its own connection, before the # launch), so the launch command just points uv at where it landed -- @@ -47,6 +51,10 @@ def test_ssh_remote_command_dev_uses_a_delivered_wheel() -> None: @pytest.mark.skipif(released, reason="released execnet resolves from an index") @pytest.mark.skipif(not _provision.uv_available(), reason="uv required to build wheel") +@pytest.mark.skipif( + not _provision.provisioning_available(), + reason="a dev execnet installed without its source tree cannot build a wheel", +) def test_wheel_delivery_command_expands_home_and_drains_stdin() -> None: wheel = _provision.ssh_wheel(execnet.XSpec("ssh=host//id=gw0")) assert wheel is not None diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py index 764ad12a..cc300530 100644 --- a/testing/test_ssh_local.py +++ b/testing/test_ssh_local.py @@ -14,15 +14,35 @@ import threading from collections.abc import Iterator from pathlib import Path +from typing import TYPE_CHECKING -import asyncssh import pytest import execnet - -pytestmark = pytest.mark.skipif( - shutil.which("ssh") is None, reason="system ssh client required" -) +from execnet import _provision + +if TYPE_CHECKING: + import asyncssh +else: + # a module-level `import asyncssh` would be a collection *error* in an + # environment without it, not a skip + asyncssh = pytest.importorskip("asyncssh") + +pytestmark = [ + pytest.mark.skipif( + shutil.which("ssh") is None, reason="system ssh client required" + ), + # the harness runs each remote command through a POSIX shell, and the + # commands execnet builds are POSIX sh + pytest.mark.skipif( + sys.platform.startswith("win"), reason="POSIX-shell remote harness" + ), + pytest.mark.skipif( + not _provision.provisioning_available(), + reason="a dev execnet installed without its source tree cannot build" + " a wheel to provision the remote with", + ), +] # Committed, intentionally-insecure test keys (see sshkeys/README.md). SSHKEYS = Path(__file__).parent / "sshkeys" diff --git a/testing/test_termination.py b/testing/test_termination.py index 03d6bbb8..d2e1dc82 100644 --- a/testing/test_termination.py +++ b/testing/test_termination.py @@ -99,8 +99,17 @@ def test_close_initiating_remote_no_error( execnet.default_group.terminate() """ ) + # This asserts execnet's own teardown prints nothing. tox sets + # PYTHONWARNDEFAULTENCODING to catch *our* encoding bugs, but it also + # makes trio's subprocess module emit an EncodingWarning we cannot fix + # and that is not what this test guards. + env = dict(os.environ) + env.pop("PYTHONWARNDEFAULTENCODING", None) popen = subprocess.Popen( - [anypython, str(p), str(execnetdir)], stdout=None, stderr=subprocess.PIPE + [anypython, str(p), str(execnetdir)], + stdout=None, + stderr=subprocess.PIPE, + env=env, ) _out, err = popen.communicate() print(err) diff --git a/tox.ini b/tox.ini index 5880696d..84334570 100644 --- a/tox.ini +++ b/tox.ini @@ -6,9 +6,9 @@ isolated_build = true usedevelop=true setenv = PYTHONWARNDEFAULTENCODING = 1 -deps= - pytest - pytest-timeout +# the one list, from pyproject: a hand-maintained copy here drifted and +# left asyncssh/hypothesis out, so those modules failed to import +extras = testing passenv = GITHUB_ACTIONS, HOME, USER, XDG_* commands= python -m pytest {posargs:testing} diff --git a/uv.lock b/uv.lock index 311eb0d8..65bd3b34 100644 --- a/uv.lock +++ b/uv.lock @@ -392,45 +392,45 @@ gevent = [ ] testing = [ { name = "asyncssh" }, - { name = "hatch" }, { name = "hypothesis" }, - { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-timeout" }, - { name = "tox" }, { name = "uv" }, ] [package.dev-dependencies] dev = [ + { name = "execnet", extra = ["testing"] }, + { name = "hatch" }, + { name = "pre-commit" }, { name = "pytest-xdist" }, + { name = "tox" }, ] gevent = [ { name = "gevent" }, ] -testing = [ - { name = "execnet", extra = ["testing"] }, -] [package.metadata] requires-dist = [ { name = "asyncssh", marker = "extra == 'testing'" }, { name = "gevent", marker = "extra == 'gevent'" }, - { name = "hatch", marker = "extra == 'testing'" }, { name = "hypothesis", marker = "extra == 'testing'" }, - { name = "pre-commit", marker = "extra == 'testing'" }, { name = "pytest", marker = "extra == 'testing'", specifier = ">8.0" }, { name = "pytest-timeout", marker = "extra == 'testing'" }, - { name = "tox", marker = "extra == 'testing'" }, { name = "trio", specifier = ">=0.32" }, { name = "uv", marker = "extra == 'testing'" }, ] provides-extras = ["gevent", "testing"] [package.metadata.requires-dev] -dev = [{ name = "pytest-xdist", specifier = ">=3.8.0" }] +dev = [ + { name = "execnet", extras = ["testing"] }, + { name = "hatch" }, + { name = "pre-commit" }, + { name = "pytest-xdist", specifier = ">=3.8.0" }, + { name = "tox" }, +] gevent = [{ name = "gevent", specifier = ">=26.7.0" }] -testing = [{ name = "execnet", extras = ["testing"] }] [[package]] name = "filelock" From cd42ce984f61180f5ca259b90fd7f0b669a6e022 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 30 Jul 2026 16:38:46 +0200 Subject: [PATCH 71/91] feat(provision): EXECNET_PROVISION_WHEEL to name the wheel remotes get Provisioning a remote worker needs either a released version to resolve from an index or an editable source tree to build a wheel from. An execnet installed *from* a built distribution has neither -- it carries a dev version with no source tree -- so `tox --installpkg`, which is what CI runs, makes every test needing a provisioned worker skip. That is precisely where provisioning most wants exercising. `EXECNET_PROVISION_WHEEL` hands it one, and the workflow points it at the wheel from the same build the suite is installed from, so those workers run the artifact under test rather than something rebuilt from a checkout that may have moved on. Set-but-wrong raises instead of falling back to a build: the fallback would quietly provision something other than what was asked for. The released/dev/explicit decision now lives in one place, `provisioning_wheel()`, which every launcher routes through -- uv popen, ssh, and via sub-spawn each open-coded the version check before and could have drifted apart on it. Co-Authored-By: Claude Opus 5 --- .github/workflows/test.yml | 6 +++ CHANGELOG.rst | 9 ++++ src/execnet/_provision.py | 90 +++++++++++++++++++++++++++++--------- testing/test_provision.py | 56 ++++++++++++++++++++++++ tox.ini | 2 +- 5 files changed, 141 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c28d57ba..bc6dad53 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,7 +64,13 @@ jobs: - name: Test shell: bash + # The suite is installed from the sdist, which leaves a dev version + # with no source tree to build a provisioning wheel from -- so every + # test that needs a provisioned worker (ssh, foreign python, via) + # would skip. Hand it the wheel from the same build instead: the + # workers then run exactly the artifact under test. run: | + export EXECNET_PROVISION_WHEEL="$PWD/$(find dist -name '*.whl' | head -1)" tox run -e py --installpkg `find dist/*.tar.gz` # pytest-xdist drives execnet's local parallel testing: popen gateways, diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8431327b..b9eccf5c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -45,6 +45,15 @@ by name. The launch command no longer needs ``head -c `` byte accounting, a ``mktemp`` prelude, or ``exec`` to keep an fd alive, and the protocol stream never carries a payload. +* New ``EXECNET_PROVISION_WHEEL`` environment variable naming a prebuilt + wheel to provision remote workers from, instead of resolving the + coordinator's version from an index or building one from its source tree. + This is for testing a built artifact: an execnet installed *from* a + distribution has a dev version but no source tree, so it can do neither + and every test needing a provisioned worker would skip. Pointing this at + the wheel from the same build makes those workers run the artifact under + test. Set but not naming an existing ``.whl`` is an error rather than a + silent fallback, which would provision something else. * ``main_thread_only`` used to *serialize*, so every sequential ``remote_exec`` was guaranteed the worker's main thread. The ``thread`` profile it now maps to releases its claim as an exec finishes, a moment diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index ff9966b2..deed0002 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -12,6 +12,8 @@ * dev coordinator (``X.Y.Z.devN+g...``) -> build a wheel from the editable install's source tree, cache it keyed by version, and ``uv run --with `` +``EXECNET_PROVISION_WHEEL`` overrides both: see :data:`PROVISION_WHEEL_ENV`. + trio is pulled transitively as an execnet dependency. """ @@ -194,18 +196,70 @@ def _build_wheel(version: str) -> Path: return wheel +#: names a prebuilt wheel to provision remotes from, bypassing both the +#: index lookup and the build-from-source path. +#: +#: This exists for testing an *artifact*: CI installs execnet from a built +#: distribution, which leaves a dev version with no editable source tree to +#: build from -- so provisioning would be unavailable exactly where it most +#: wants exercising. Pointing this at the wheel from the same build makes +#: the workers run the code under test rather than something rebuilt from a +#: checkout that may have moved on. +PROVISION_WHEEL_ENV = "EXECNET_PROVISION_WHEEL" + + +def _explicit_wheel() -> Path | None: + """The wheel named by :data:`PROVISION_WHEEL_ENV`, if it is set. + + Set-but-wrong is a configuration error, not a reason to quietly fall + back to building: the fallback would provision something *other* than + what the caller asked to test. + """ + raw = os.environ.get(PROVISION_WHEEL_ENV) + if not raw: + return None + wheel = Path(raw) + if not wheel.is_file(): + raise RuntimeError(f"{PROVISION_WHEEL_ENV}={raw!r} is not an existing file") + if wheel.suffix != ".whl": + raise RuntimeError(f"{PROVISION_WHEEL_ENV}={raw!r} is not a wheel") + return wheel.resolve() + + +def provisioning_wheel() -> Path | None: + """The wheel to provision remotes from, or None to resolve from an index. + + The one place the "released vs dev vs explicitly given" decision is + made; every launcher (uv popen, ssh, via sub-spawn) routes through it so + they cannot disagree about what a remote ends up running. + """ + explicit = _explicit_wheel() + if explicit is not None: + return explicit + + import execnet + + version = execnet.__version__ + if _RELEASED_RE.match(version): + return None + return _build_wheel(version) + + def provisioning_available() -> bool: """Whether this coordinator can produce material to provision a remote. A released version resolves from an index. A dev version has to build a wheel, which needs the editable checkout it came from -- so a dev version installed *from a wheel* (what ``tox --installpkg`` produces, - and what a CI artifact test runs against) can do neither. Callers that - need a remote worker should check this rather than let - :func:`coordinator_requirement` raise. + and what a CI artifact test runs against) can do neither, unless + :data:`PROVISION_WHEEL_ENV` hands it one. Callers that need a remote + worker should check this rather than let :func:`coordinator_requirement` + raise. """ import execnet + if _explicit_wheel() is not None: + return True if _RELEASED_RE.match(execnet.__version__): return True return _editable_source_root() is not None @@ -215,10 +269,10 @@ def coordinator_requirement() -> str: """A ``uv --with`` requirement that installs this coordinator's execnet.""" import execnet - version = execnet.__version__ - if _RELEASED_RE.match(version): - return f"execnet=={version}" - return str(_build_wheel(version)) + wheel = provisioning_wheel() + if wheel is not None: + return str(wheel) + return f"execnet=={execnet.__version__}" def worker_cli_arg(spec: Any) -> str: @@ -393,24 +447,19 @@ def ssh_remote_command(spec: Any, *protocol: str, config_on_stdin: bool = False) """ import execnet - version = execnet.__version__ config = worker_cli_arg(spec) kwargs: dict[str, Any] = {"config_on_stdin": config_on_stdin} - if _RELEASED_RE.match(version): - kwargs["requirement"] = f"execnet=={version}" + wheel = provisioning_wheel() + if wheel is None: + kwargs["requirement"] = f"execnet=={execnet.__version__}" else: - kwargs["wheel"] = _build_wheel(version) + kwargs["wheel"] = wheel return _remote_shell_command(spec.python, config, *protocol, **kwargs) def ssh_wheel(spec: Any) -> Path | None: """The wheel this coordinator must deliver before launching, if any.""" - import execnet - - version = execnet.__version__ - if _RELEASED_RE.match(version): - return None - return _build_wheel(version) + return provisioning_wheel() def ssh_argv( @@ -472,11 +521,10 @@ def spawn_request(spec: Any) -> dict[str, Any]: "ssh_config": spec.ssh_config or None, } if spec.ssh or spec.vagrant_ssh or spec.python: - version = execnet.__version__ - if _RELEASED_RE.match(version): - request["requirement"] = f"execnet=={version}" + wheel = provisioning_wheel() + if wheel is None: + request["requirement"] = f"execnet=={execnet.__version__}" else: - wheel = _build_wheel(version) request["wheel"] = (wheel.name, wheel.read_bytes()) return request diff --git a/testing/test_provision.py b/testing/test_provision.py index 008aae29..97874b34 100644 --- a/testing/test_provision.py +++ b/testing/test_provision.py @@ -22,6 +22,8 @@ def test_worker_cli_arg_carries_config() -> None: def test_ssh_remote_command_released(monkeypatch: pytest.MonkeyPatch) -> None: + # the index path specifically -- an explicit wheel would override it + monkeypatch.delenv(_provision.PROVISION_WHEEL_ENV, raising=False) monkeypatch.setattr(execnet, "__version__", "9.9.9") spec = execnet.XSpec("ssh=host//id=gw0//execmodel=thread") command = _provision.ssh_remote_command(spec) @@ -100,6 +102,60 @@ def test_sub_spawn_argv_plain_popen() -> None: assert delivery is None +class TestExplicitWheel: + """``EXECNET_PROVISION_WHEEL`` names the wheel remotes are provisioned from.""" + + @pytest.fixture + def wheel(self, tmp_path, monkeypatch: pytest.MonkeyPatch): + wheel = tmp_path / "execnet-9.9.9-py3-none-any.whl" + wheel.write_bytes(b"PK\x03\x04not-really-a-wheel") + monkeypatch.setenv(_provision.PROVISION_WHEEL_ENV, str(wheel)) + return wheel + + def test_wins_over_the_index_for_a_released_version( + self, wheel, monkeypatch: pytest.MonkeyPatch + ) -> None: + # a released coordinator would otherwise resolve execnet==X.Y.Z, which + # is the wrong artifact when the point is to test *this* build + monkeypatch.setattr(execnet, "__version__", "9.9.9") + assert _provision.provisioning_wheel() == wheel + assert _provision.coordinator_requirement() == str(wheel) + command = _provision.ssh_remote_command(execnet.XSpec("ssh=host//id=gw0")) + assert "execnet==9.9.9" not in command + assert _provision.remote_wheel_path(wheel) in command + + def test_makes_provisioning_available_without_a_source_tree( + self, wheel, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(execnet, "__version__", "9.9.9.dev1+gdeadbee") + monkeypatch.setattr(_provision, "_editable_source_root", lambda: None) + assert _provision.provisioning_available() + + def test_ships_those_bytes_to_a_via_master( + self, wheel, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(execnet, "__version__", "9.9.9") + request = _provision.spawn_request(execnet.XSpec("ssh=host//id=gw0")) + assert request["wheel"] == (wheel.name, wheel.read_bytes()) + assert "requirement" not in request + + @pytest.mark.parametrize( + ("name", "exists"), + [("execnet-9.9.9-py3-none-any.whl", False), ("execnet.tar.gz", True)], + ) + def test_rejects_what_it_cannot_provision_from( + self, name: str, exists: bool, tmp_path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # falling back to a build here would provision something other than + # what the caller asked for -- silently testing the wrong thing + path = tmp_path / name + if exists: + path.touch() + monkeypatch.setenv(_provision.PROVISION_WHEEL_ENV, str(path)) + with pytest.raises(RuntimeError, match=_provision.PROVISION_WHEEL_ENV): + _provision.provisioning_wheel() + + def test_sub_spawn_argv_vagrant_released() -> None: request = { "config": "{}", diff --git a/tox.ini b/tox.ini index 84334570..c938f81d 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ setenv = # the one list, from pyproject: a hand-maintained copy here drifted and # left asyncssh/hypothesis out, so those modules failed to import extras = testing -passenv = GITHUB_ACTIONS, HOME, USER, XDG_* +passenv = GITHUB_ACTIONS, HOME, USER, XDG_*, EXECNET_PROVISION_WHEEL commands= python -m pytest {posargs:testing} From 9413a2e2a3879031d4731a3780b8332504d58b4c Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 30 Jul 2026 19:35:55 +0200 Subject: [PATCH 72/91] fix: start Windows workers again, over a threaded stdio stream Every Windows worker died at startup with AttributeError: module 'trio.lowlevel' has no attribute 'FdStream' `FdStream` is POSIX-only. Windows resolves to the stdio transport (no `pass_fds`, no `ssh -R` unix sockets), so the worker adopts its own fd 0/1 -- and that went straight through the missing class. 68 failures and 184 errors on every Windows job, invisible until the suite started running again. Trio does have Windows pipe streams, but they require handles opened in OVERLAPPED mode and registered with an IOCP, and inherited stdio is an ordinary synchronous pipe. So the Windows path reads and writes in the thread pool instead. Reads are abandoned on cancellation -- nothing can interrupt a blocking pipe read short of the peer closing -- while writes are not, since a torn write would desynchronize the framing. Verified on Linux by forcing both halves of the Windows configuration (stdio transport plus the threaded stream) and running the full suite: only the three tests that assert POSIX picks the socket transport fail, and those are guarded by `sys.platform` on the real thing. `test_single_thread_on_main` asserted the trio profile uses exactly one thread, which is a property of the transport as much as the profile -- now asserted only where the transport is genuinely threadless. Also drops PYTHONWARNDEFAULTENCODING for the child in test_terminate_implicit_does_trykill, which asserts execnet's teardown is silent and was counting an EncodingWarning from trio's own subprocess module. That was the only Linux failure (3.10, 3.11, pypy). Co-Authored-By: Claude Opus 5 --- CHANGELOG.rst | 7 ++++ src/execnet/_trio_gateway.py | 51 +++++++++++++++++++++++ testing/test_execmodel_trio.py | 8 +++- testing/test_termination.py | 7 +++- testing/test_trio_gateway.py | 75 ++++++++++++++++++++++++++++++++++ 5 files changed, 146 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b9eccf5c..7f54b253 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -45,6 +45,13 @@ by name. The launch command no longer needs ``head -c `` byte accounting, a ``mktemp`` prelude, or ``exec`` to keep an fd alive, and the protocol stream never carries a payload. +* Fixed Windows workers, which could not start at all: adopting the + inherited stdio pipes went through ``trio.lowlevel.FdStream``, which is + POSIX-only. Windows has no async equivalent -- trio's Windows pipe streams + need OVERLAPPED handles registered with an IOCP, and the stdio a process + inherits is an ordinary synchronous pipe -- so those reads and writes now + run in the thread pool. The socket transport, where it is available, still + needs no threads. * New ``EXECNET_PROVISION_WHEEL`` environment variable naming a prebuilt wheel to provision remote workers from, instead of resolving the coordinator's version from an index or building one from its source tree. diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 44b7dbd3..c4b8e3f1 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -81,8 +81,59 @@ def staple_process_stream(process: trio.Process) -> ByteStream: return trio.StapledStream(process.stdin, process.stdout) +class ThreadedFdStream: + """A :class:`ByteStream` over blocking fds, doing its IO in worker threads. + + The Windows stand-in for ``trio.lowlevel.FdStream``, which is POSIX-only. + Trio *does* have Windows pipe streams, but they require handles opened in + OVERLAPPED mode and register them with an IOCP -- and the stdio a process + inherits from its parent is an ordinary synchronous pipe, so a worker + cannot adopt its own fd 0/1 that way. + + Blocking reads and writes therefore go to the thread pool. A pending + read is abandoned on cancellation, since nothing can interrupt it short + of the peer closing; a write is not, because a torn write would leave a + half-message on the wire and desynchronize the framing. + """ + + def __init__(self, read_fd: int, write_fd: int) -> None: + self._read_fd: int | None = read_fd + self._write_fd: int | None = write_fd + + async def receive_some(self, max_bytes: int | None = None) -> bytes: + fd = self._read_fd + if fd is None: + raise trio.ClosedResourceError("stream closed") + return await trio.to_thread.run_sync( + os.read, fd, max_bytes or 65536, abandon_on_cancel=True + ) + + async def send_all(self, data: bytes) -> None: + fd = self._write_fd + if fd is None: + raise trio.ClosedResourceError("stream closed") + view = memoryview(data) + while view: + written = await trio.to_thread.run_sync(os.write, fd, view) + view = view[written:] + + async def send_eof(self) -> None: + fd, self._write_fd = self._write_fd, None + if fd is not None: + os.close(fd) + + async def aclose(self) -> None: + await self.send_eof() + fd, self._read_fd = self._read_fd, None + if fd is not None: + os.close(fd) + await trio.lowlevel.checkpoint() + + def staple_fd_stream(read_fd: int, write_fd: int) -> ByteStream: """One bidirectional stream over OS pipe fds (worker stdio pipes).""" + if not hasattr(trio.lowlevel, "FdStream"): # Windows + return ThreadedFdStream(read_fd, write_fd) return trio.StapledStream( trio.lowlevel.FdStream(write_fd), trio.lowlevel.FdStream(read_fd) ) diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py index 8fd121d8..eeff80f2 100644 --- a/testing/test_execmodel_trio.py +++ b/testing/test_execmodel_trio.py @@ -15,6 +15,7 @@ import execnet import execnet.trio from execnet import Gateway +from execnet import _provision TESTTIMEOUT = 10.0 @@ -62,7 +63,12 @@ def test_single_thread_on_main(self, trio_gw: Gateway) -> None: ) active, on_main = channel.receive(TESTTIMEOUT) assert on_main - assert active == 1 + # The profile itself needs no threads -- exec'd sources are tasks on + # the loop. The *transport* may: adopting inherited stdio has no + # async form on Windows, so those reads and writes run in the thread + # pool. Only the socket transport is genuinely single-threaded. + if _provision.resolve_transport(trio_gw.spec) == "socket": + assert active == 1 def test_concurrent_execs_cooperate(self, trio_gw: Gateway) -> None: # two execs run as tasks on one loop: the first parks in receive diff --git a/testing/test_termination.py b/testing/test_termination.py index d2e1dc82..4cb1406b 100644 --- a/testing/test_termination.py +++ b/testing/test_termination.py @@ -152,7 +152,12 @@ def flush(self): """ % str(execnetdir) ) - popen = subprocess.Popen([str(anypython), str(p)], stdout=subprocess.PIPE) + # as above: this asserts execnet's teardown is silent, and tox's + # PYTHONWARNDEFAULTENCODING makes trio's own subprocess module emit an + # EncodingWarning that would be counted as our noise. + env = dict(os.environ) + env.pop("PYTHONWARNDEFAULTENCODING", None) + popen = subprocess.Popen([str(anypython), str(p)], stdout=subprocess.PIPE, env=env) # sync with start-up assert popen.stdout is not None popen.stdout.readline() diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index 9473df89..fa4cf72d 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -8,6 +8,7 @@ from __future__ import annotations +import os from collections.abc import AsyncIterator from contextlib import asynccontextmanager @@ -22,6 +23,7 @@ from execnet._serialize import loads_internal from execnet._trio_gateway import AsyncGateway from execnet._trio_gateway import AsyncGroup +from execnet._trio_gateway import ThreadedFdStream from execnet._trio_gateway import open_gateway @@ -403,3 +405,76 @@ async def main() -> None: await group.makegateway("id=notype") trio.run(main) + + +class TestThreadedFdStream: + """The Windows stand-in for ``trio.lowlevel.FdStream``. + + Exercised on every platform, since Windows is the only place it is + *used* and the least convenient place to find out it is broken. + """ + + def test_roundtrip_through_a_pipe_pair(self) -> None: + async def main() -> None: + their_read, our_write = os.pipe() + our_read, their_write = os.pipe() + stream = ThreadedFdStream(our_read, our_write) + try: + await stream.send_all(b"hello ") + await stream.send_all(b"world") + assert os.read(their_read, 11) == b"hello world" + + os.write(their_write, b"back") + assert await stream.receive_some(4) == b"back" + finally: + await stream.aclose() + os.close(their_read) + os.close(their_write) + + trio.run(main) + + def test_receive_reports_eof_as_empty(self) -> None: + async def main() -> None: + our_read, their_write = os.pipe() + stream = ThreadedFdStream(our_read, os.open(os.devnull, os.O_WRONLY)) + os.close(their_write) + try: + assert await stream.receive_some(4) == b"" + finally: + await stream.aclose() + + trio.run(main) + + def test_send_eof_lets_the_peer_see_the_end(self) -> None: + async def main() -> None: + their_read, our_write = os.pipe() + stream = ThreadedFdStream(os.open(os.devnull, os.O_RDONLY), our_write) + try: + await stream.send_all(b"tail") + await stream.send_eof() + assert os.read(their_read, 4) == b"tail" + assert os.read(their_read, 4) == b"" # write end is gone + with pytest.raises(trio.ClosedResourceError): + await stream.send_all(b"more") + finally: + await stream.aclose() + os.close(their_read) + + trio.run(main) + + def test_a_cancelled_read_is_abandoned_not_awaited(self) -> None: + # a blocking read on a pipe cannot be interrupted, so cancellation + # must not wait for it -- otherwise shutdown hangs until the peer + # happens to write. + async def main() -> None: + our_read, their_write = os.pipe() + stream = ThreadedFdStream(our_read, os.open(os.devnull, os.O_WRONLY)) + try: + with trio.move_on_after(0.2) as scope: + await stream.receive_some(4) + assert scope.cancelled_caught + finally: + os.close(their_write) + await stream.aclose() + + trio.run(main) From a5fee03d1e623c61753ff992dda3a8821f991c4e Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Thu, 30 Jul 2026 22:47:30 +0200 Subject: [PATCH 73/91] feat: hand workers a socket on Windows, and stop hanging when we cannot Two Windows gaps and one real defect, found by the socket gateway that wedged the whole Windows suite behind a 20-second timeout. The defect: a `socket=`/`installvia=` worker is spawned by the *server*, so when that spawn failed the exception died there while the coordinator sat waiting for a handshake byte nobody would send. One unsupported gateway took the entire session with it. Now a host that cannot hand over a socket refuses before it replies with an address -- the only moment a reason can still reach the coordinator, since afterwards it is already connecting and a socket can say nothing but EOF -- and a spawn that fails anyway closes the connection so the wait ends. The gap: `subprocess` refuses `pass_fds` on Windows. `socket.share()` (WSADuplicateSocket) duplicates a socket into a named pid instead, which needs the pid, which only exists after spawning -- so `--protocol-share` goes in argv and the blob follows in the config on stdin. It is bound to that one pid, so unlike an inheritable handle it is inert to anything else, and unlike `close_fds=False` it leaks nothing to grandchildren. Windows keeps `transport=stdio` as its default. Sharing is new and has no CI behind it yet, while stdio has years; a test asks for `transport=socket` explicitly so the share path is exercised without every Windows gateway riding on it. Flip `default_spawn_transport` once that test is green. `socket_transport_available` split into `socket_handoff_available` (a worker we spawn) and `ssh_dialback_available` (one that must reach back to us) -- they were never the same question, and conflating them is why `transport=socket` on Windows ssh would have hung rather than said no. Asking for a transport that cannot work is now an error at makegateway time. Verified by simulating Windows on Linux: with the share path forced but `share()` absent, every socket gateway fails immediately with EOFError where it previously hung. The handoff wiring -- flag in argv, config on stdin, blob inside it, accepted socket surviving the share view -- is tested against a fake process; WSADuplicateSocket itself is CI-only. Co-Authored-By: Claude Opus 5 --- CHANGELOG.rst | 24 ++++ src/execnet/_cli.py | 14 ++ src/execnet/_provision.py | 101 ++++++++++--- src/execnet/_trio_gateway.py | 75 +++++++++- src/execnet/_trio_host.py | 94 ++++++++++--- src/execnet/_trio_worker.py | 45 ++++++ testing/test_cli.py | 266 ++++++++++++++++++++++++++++++++++- 7 files changed, 572 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7f54b253..e6d9bd00 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -45,6 +45,30 @@ by name. The launch command no longer needs ``head -c `` byte accounting, a ``mktemp`` prelude, or ``exec`` to keep an fd alive, and the protocol stream never carries a payload. +* A worker that cannot be handed its socket now fails instead of hanging. + A ``socket=``/``installvia=`` worker is spawned by the *server*, so a + failure there used to leave the coordinator waiting forever on a + handshake byte nobody was going to send -- one unsupported gateway could + wedge a whole session. A host that cannot hand over a socket at all now + refuses the request before replying with an address, which surfaces as a + remote error rather than an unexplained wait; and a spawn that fails + anyway closes the connection, so the coordinator sees EOF. +* New ``share`` protocol transport (``execnet worker --protocol-share``) + carrying a socket to a worker on Windows, where ``subprocess`` refuses + ``pass_fds``. The socket is duplicated into the child with + ``socket.share()`` (``WSADuplicateSocket``); because that needs the + child's pid, the flag travels in argv and the blob follows in the config + on stdin. The blob is bound to that one pid, so it is inert to anything + else. This makes ``transport=socket`` work on Windows for ``popen`` and + fixes ``socket=``/``installvia=`` gateways served from a Windows host. + Windows still *defaults* to ``transport=stdio``; ask for + ``transport=socket`` to opt in. +* ``transport=socket`` on a gateway that cannot provide it is now an error + at ``makegateway`` time naming the platform, instead of a gateway that + waits for a worker which was never able to reach back. ssh dial-back + needs ``AF_UNIX`` (which CPython does not expose on Windows) and + ``StreamLocal`` forwarding (which Win32-OpenSSH does not implement), so + ssh gateways there stay on stdio. * Fixed Windows workers, which could not start at all: adopting the inherited stdio pipes went through ``trio.lowlevel.FdStream``, which is POSIX-only. Windows has no async equivalent -- trio's Windows pipe streams diff --git a/src/execnet/_cli.py b/src/execnet/_cli.py index 0b4e86cf..009e57fc 100644 --- a/src/execnet/_cli.py +++ b/src/execnet/_cli.py @@ -87,6 +87,14 @@ def _build_parser() -> argparse.ArgumentParser: metavar="ADDR", help="listen on ADDR (unix:/path or host:port) for one connection", ) + transport.add_argument( + "--protocol-share", + action="store_true", + help=( + "adopt a socket duplicated into us with socket.share(); the blob" + " travels in the config (Windows, where fds cannot be inherited)" + ), + ) config = worker.add_mutually_exclusive_group() config.add_argument("--config", metavar="JSON", help="worker config as JSON") @@ -185,12 +193,18 @@ def _run_worker(ns: argparse.Namespace) -> None: transport = _trio_worker.ConnectTransport(ns.protocol_connect) elif ns.protocol_listen is not None: transport = _trio_worker.ListenTransport(ns.protocol_listen) + elif ns.protocol_share: + transport = _trio_worker.ShareTransport() else: transport = _trio_worker.StdioTransport() # The stdio transport owns fd 0/1, so it has to claim them before the # config is read (--config-fd 0 would be the very fd we are moving). transport.prepare() config = _load_config(ns) + if isinstance(transport, _trio_worker.ShareTransport): + # the only transport that cannot exist before the config is read: + # its socket was duplicated to our pid, so it travels inside it + transport.adopt(config) _trio_worker.serve_worker( config, transport, diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index deed0002..2da64eb7 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -43,23 +43,82 @@ TRANSPORTS = ("socket", "stdio") -def socket_transport_available() -> bool: - """Whether this coordinator can drive the socket transport. +def socket_handoff_available() -> bool: + """Whether we can hand a socket to a worker process we spawn ourselves. + + Two different mechanisms, one question. POSIX passes the fd through + ``pass_fds``. Windows refuses that outright, but ``socket.share()`` + (``WSADuplicateSocket``) duplicates the socket into a named pid, and the + resulting blob rides in the worker config -- see the ``share`` transport + in ``_trio_worker``. + """ + import socket as _socket + + if sys.platform.startswith("win"): + return hasattr(_socket.socket, "share") + return True + + +def socket_share_required() -> bool: + """Whether handing a socket to a child needs ``share()`` and not ``pass_fds``. + + A function rather than a ``sys.platform`` test at each call site: those + read as dead code to a type checker running with the other platform's + assumptions, and this is the one question being asked anyway. + """ + return sys.platform.startswith("win") + + +def ssh_dialback_available() -> bool: + """Whether an ssh worker can dial back to us over a forwarded unix socket. + + Needs ``AF_UNIX`` here -- CPython has never exposed it on Windows -- and + ``StreamLocal`` forwarding in the ssh client and server, which + Win32-OpenSSH does not implement. Neither is a coordinator-side choice, + so ssh gateways there stay on the stdio transport. + """ + import socket as _socket - ``subprocess`` refuses ``pass_fds`` on Windows and ``ssh -R`` cannot - forward a unix socket there either, so Windows coordinators stay on - stdio unless someone asks for otherwise and accepts the consequences. + return hasattr(_socket, "AF_UNIX") and not sys.platform.startswith("win") + + +def default_spawn_transport() -> str: + """What a worker we spawn gets when the spec does not say. + + Windows can *serve* the socket transport (see :func:`socket_share_required`), + but stdio is what has years of exercise there while sharing is new, so it + stays the default until the share path has CI behind it. + ``transport=socket`` opts in. """ - return not sys.platform.startswith("win") + if socket_share_required(): + return "stdio" + return "socket" if socket_handoff_available() else "stdio" -def resolve_transport(spec: Any) -> str: - """The transport for ``spec``: explicit if given, else the platform default.""" +def resolve_transport( + spec: Any, *, available: bool = True, default: str | None = None +) -> str: + """The transport for ``spec``: explicit if given, else the default. + + ``available`` is the caller's capability for *its* kind of gateway -- + :func:`socket_handoff_available` for a worker we spawn, + :func:`ssh_dialback_available` for one that has to reach back to us. + Asking for a transport that cannot work is an error at makegateway time, + rather than a hang once nobody connects. ``default`` decouples "can + work" from "is what you get by default". + """ requested: str | None = getattr(spec, "transport", None) if requested is None: - return "socket" if socket_transport_available() else "stdio" + if default is not None: + return default + return "socket" if available else "stdio" if requested not in TRANSPORTS: raise ValueError(f"unknown transport {requested!r} (known: {list(TRANSPORTS)})") + if requested == "socket" and not available: + raise ValueError( + "transport=socket is not available for this gateway on " + f"{sys.platform}; use transport=stdio" + ) return requested @@ -275,12 +334,13 @@ def coordinator_requirement() -> str: return f"execnet=={execnet.__version__}" -def worker_cli_arg(spec: Any) -> str: - """Single JSON CLI argument carrying the worker config (the whole 'spec thing'). +def worker_config(spec: Any) -> dict[str, Any]: + """The worker config for ``spec`` (the whole 'spec thing'), as a dict. - Passed to ``python -m execnet worker`` by every launcher (popen, uv, ssh) - so the worker config lives in one place rather than scattered positional - args. + Every launcher (popen, uv, ssh) sends this so the worker config lives in + one place rather than scattered positional args. A launcher may add to + it before serializing -- the ``share`` transport puts its duplicated + socket here, since that cannot exist until the worker has been spawned. """ import execnet @@ -309,7 +369,12 @@ def worker_cli_arg(spec: Any) -> str: config["nice"] = int(spec.nice) if spec.env: config["env"] = spec.env - return json.dumps(config) + return config + + +def worker_cli_arg(spec: Any) -> str: + """:func:`worker_config` serialized for ``--config``/``--config-fd``.""" + return json.dumps(worker_config(spec)) def stdio_tokens(spec: Any) -> list[str]: @@ -369,7 +434,9 @@ def _extra_with_tokens(config: str) -> list[str]: return [] -def uv_worker_argv(spec: Any, *protocol: str) -> list[str]: +def uv_worker_argv( + spec: Any, *protocol: str, config_on_stdin: bool = False +) -> list[str]: """``uv run`` argv to launch the Trio worker locally (wheel path is local).""" config = worker_cli_arg(spec) return [ @@ -377,7 +444,7 @@ def uv_worker_argv(spec: Any, *protocol: str) -> list[str]: "--with", coordinator_requirement(), *_extra_with_tokens(config), - *_worker_tokens(config, *protocol), + *_worker_tokens(None if config_on_stdin else config, *protocol), ] diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index c4b8e3f1..4429bddf 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -182,7 +182,9 @@ def popen_module_args( return args -def popen_worker_argv(spec: Any, *protocol: str) -> list[str]: +def popen_worker_argv( + spec: Any, *protocol: str, config_on_stdin: bool = False +) -> list[str]: """Argv for a popen worker: direct module launch, or uv-provisioned. A bare ``python=`` interpreter without execnet gets execnet + trio @@ -191,8 +193,10 @@ def popen_worker_argv(spec: Any, *protocol: str) -> list[str]: from . import _provision if spec.python and not _provision.target_has_execnet(spec.python): - return _provision.uv_worker_argv(spec, *protocol) - return popen_module_args(spec, *protocol) + return _provision.uv_worker_argv( + spec, *protocol, config_on_stdin=config_on_stdin + ) + return popen_module_args(spec, *protocol, config_on_stdin=config_on_stdin) class RawChannel: @@ -937,7 +941,10 @@ async def connect_ssh_worker(spec: Any) -> tuple[ByteStream, trio.Process]: if wheel is not None: await deliver_remote_wheel(spec, wheel) - if _provision.resolve_transport(spec) == "stdio": + dialback = _provision.resolve_transport( + spec, available=_provision.ssh_dialback_available() + ) + if dialback == "stdio": args = ( ssh_transport_args(spec) if spec.ssh is not None @@ -1034,6 +1041,56 @@ async def connect_command_worker( return stream, process +#: config key carrying a ``socket.share()`` blob to a ``--protocol-share`` +#: worker, base64 encoded because the config is JSON. +SHARE_KEY = "protocol_share" + + +def share_socket(sock: Any, pid: int) -> str: + """``socket.share(pid)`` as a base64 string for the worker config.""" + import base64 + + return base64.b64encode(sock.share(pid)).decode("ascii") + + +async def _spawn_with_socket(spec: Any, theirs: Any) -> trio.Process: + """Spawn a worker owning ``theirs``, by whichever handoff this OS has. + + POSIX passes the fd itself. Windows cannot -- ``subprocess`` refuses + ``pass_fds`` there -- so the socket is duplicated into the child with + ``WSADuplicateSocket``. That needs the child's pid, so it can only + happen once the child exists: the flag goes in argv, the blob follows in + the config on stdin. + """ + from . import _provision + + if not _provision.socket_share_required(): + args = popen_worker_argv(spec, "--protocol-fd", str(theirs.fileno())) + return await trio.lowlevel.open_process(args, pass_fds=(theirs.fileno(),)) + + args = popen_worker_argv(spec, "--protocol-share", config_on_stdin=True) + process = await trio.lowlevel.open_process(args, stdin=subprocess.PIPE) + try: + assert process.stdin is not None + config = _provision.worker_config(spec) + config[SHARE_KEY] = share_socket(theirs, process.pid) + await process.stdin.send_all(dumps_config(config)) + await process.stdin.aclose() + except BaseException: + with trio.CancelScope(shield=True), suppress(Exception): + process.kill() + await process.wait() + raise + return process + + +def dumps_config(config: dict[str, Any]) -> bytes: + """The worker config as the bytes a ``--config-fd`` worker reads.""" + import json + + return json.dumps(config).encode("utf-8") + + async def connect_popen_worker(spec: Any) -> tuple[ByteStream, trio.Process]: """Spawn a local worker for ``spec`` on its resolved transport. @@ -1044,15 +1101,19 @@ async def connect_popen_worker(spec: Any) -> tuple[ByteStream, trio.Process]: """ from . import _provision - if _provision.resolve_transport(spec) == "stdio": + transport = _provision.resolve_transport( + spec, + available=_provision.socket_handoff_available(), + default=_provision.default_spawn_transport(), + ) + if transport == "stdio": return await connect_command_worker(popen_worker_argv(spec)) import socket as _socket ours, theirs = _socket.socketpair() try: - args = popen_worker_argv(spec, "--protocol-fd", str(theirs.fileno())) - process = await trio.lowlevel.open_process(args, pass_fds=(theirs.fileno(),)) + process = await _spawn_with_socket(spec, theirs) except BaseException: ours.close() theirs.close() diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index dc5214b6..e114805d 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -797,29 +797,52 @@ def makegateway_trio(group: Group, spec: Any) -> Gateway: def _spawn_socket_worker(fd: int) -> subprocess.Popen[bytes]: - """Spawn a worker subprocess serving over the inherited socket ``fd``.""" + """Spawn a worker subprocess serving the accepted socket ``fd``. + + POSIX hands the fd over with ``pass_fds``. Windows has no such thing, + so the socket is duplicated into the child with ``WSADuplicateSocket`` + and the blob travels in the config -- which is why the config goes to + stdin there: it cannot be built until the child's pid exists. + """ import execnet - config = json.dumps( - { - "id": "socketworker%d" % next(_socket_worker_counter), - "execmodel": "thread", - "coordinator_version": execnet.__version__, - } - ) - return subprocess.Popen( - [ - sys.executable, - "-m", - "execnet", - "worker", - "--protocol-fd", - str(fd), - "--config", - config, - ], - pass_fds=[fd], + from . import _provision + from ._trio_gateway import SHARE_KEY + from ._trio_gateway import dumps_config + from ._trio_gateway import share_socket + + config: dict[str, Any] = { + "id": "socketworker%d" % next(_socket_worker_counter), + "execmodel": "thread", + "coordinator_version": execnet.__version__, + } + argv = [sys.executable, "-m", "execnet", "worker"] + if not _provision.socket_share_required(): + return subprocess.Popen( + [*argv, "--protocol-fd", str(fd), "--config", json.dumps(config)], + pass_fds=[fd], + ) + + import socket as _socket + + process = subprocess.Popen( + [*argv, "--protocol-share", "--config-fd", "0"], stdin=subprocess.PIPE ) + try: + # a view on the accepted socket, so share() can reach it; detach so + # dropping the view does not close the fd we do not own + view = _socket.socket(fileno=fd) + try: + config[SHARE_KEY] = share_socket(view, process.pid) + finally: + view.detach() + assert process.stdin is not None + process.stdin.write(dumps_config(config)) + process.stdin.close() + except BaseException: + process.kill() + raise + return process async def serve_socket_connection(stream: trio.SocketStream, *, reap: bool) -> None: @@ -827,9 +850,18 @@ async def serve_socket_connection(stream: trio.SocketStream, *, reap: bool) -> N ``reap`` waits for the worker (loop server); when false the worker outlives this task (one-shot / installvia). + + A failed spawn closes the connection. The coordinator is already waiting + on the other end for a handshake byte that is never coming, and an EOF is + the only thing that will move it -- without this it waits forever. """ - proc = _spawn_socket_worker(stream.socket.fileno()) - # The child forked with a copy of the fd; release ours. + try: + proc = _spawn_socket_worker(stream.socket.fileno()) + except BaseException: + with trio.CancelScope(shield=True), suppress(Exception): + await stream.aclose() + raise + # The child holds its own copy of the socket now; release ours. await stream.aclose() if reap: await trio.to_thread.run_sync(proc.wait) @@ -842,7 +874,25 @@ async def _start_socket_and_reply( Runs as a task on the worker's Trio host (scheduled from the message handler). The reply travels back on ``channelid`` like a STATUS reply. + + Refusing *before* replying is what makes an unsupported host survivable: + once the address has gone out the coordinator will connect and wait for + a handshake, and there is no longer any way to tell it why nobody is + there. An error close instead surfaces at its ``channel.receive()``. """ + from . import _provision + + if not _provision.socket_handoff_available(): + gateway._send( + Message.CHANNEL_CLOSE_ERROR, + channelid, + dumps_internal( + f"cannot serve a socket gateway on {sys.platform}: this host " + "cannot hand an accepted socket to a worker process" + ), + ) + return + listeners = await trio.open_tcp_listeners(0, host=bind_host) addr = listeners[0].socket.getsockname() gateway._send(Message.CHANNEL_DATA, channelid, dumps_internal((addr[0], addr[1]))) diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 50faccb4..f07e6f85 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -615,6 +615,51 @@ async def open(self) -> Any: return await _send_ready(staple_fd_stream(read_fd, write_fd)) +class ShareTransport: + """The protocol socket was duplicated into this process by its launcher. + + The Windows counterpart of an inherited fd: ``subprocess`` refuses + ``pass_fds`` there, but ``WSADuplicateSocket`` can duplicate a socket + into a named pid. The resulting blob is bound to *us*, so it is inert + to anything else that might read it -- and it arrives in the config + rather than in argv, because sharing needs our pid and so cannot happen + until we have been spawned. + + Like any other socket transport, the worker's stdio stays untouched. + """ + + stdio_defaults = ("inherit", "inherit", "inherit") + + def __init__(self) -> None: + self._blob: bytes | None = None + + def prepare(self) -> None: + pass + + def adopt(self, config: dict[str, Any]) -> None: + """Take the share blob out of the config, once it has been read.""" + import base64 + + from ._trio_gateway import SHARE_KEY + + raw = config.pop(SHARE_KEY, None) + if raw is None: + raise SystemExit( + f"execnet worker: --protocol-share needs {SHARE_KEY!r} in the config" + ) + self._blob = base64.b64decode(raw) + + async def open(self) -> Any: + import socket as _socket + + from . import _trio_host + + assert self._blob is not None, "adopt() first" + sock = _socket.fromshare(self._blob) # type: ignore[attr-defined] # Windows + # adopt_socket takes ownership of the fd and sends the handshake + return await _trio_host.adopt_socket(sock.detach()) + + class FdTransport: """The protocol runs over inherited fds: one socket, or a pipe pair.""" diff --git a/testing/test_cli.py b/testing/test_cli.py index 3ab45008..205201ac 100644 --- a/testing/test_cli.py +++ b/testing/test_cli.py @@ -220,7 +220,12 @@ class TestTransportSelection: def test_defaults_per_platform(self) -> None: spec = execnet.XSpec("popen") expected = "stdio" if sys.platform.startswith("win") else "socket" - assert _provision.resolve_transport(spec) == expected + assert ( + _provision.resolve_transport( + spec, default=_provision.default_spawn_transport() + ) + == expected + ) def test_explicit_wins(self) -> None: assert _provision.resolve_transport( @@ -233,6 +238,47 @@ def test_unknown_is_rejected(self) -> None: execnet.XSpec("popen//transport=carrier-pigeon") ) + def test_unavailable_falls_back_when_unasked(self) -> None: + assert ( + _provision.resolve_transport(execnet.XSpec("popen"), available=False) + == "stdio" + ) + + def test_asking_for_an_impossible_transport_is_an_error(self) -> None: + # the alternative is a gateway that hangs waiting for a worker that + # was never able to reach us -- which is what ssh on Windows did + with pytest.raises(ValueError, match="not available"): + _provision.resolve_transport( + execnet.XSpec("ssh=host//transport=socket"), available=False + ) + + def test_windows_can_serve_the_socket_transport_but_does_not_default_to_it( + self, + ) -> None: + # sharing is new; stdio is what has years of Windows behind it + if _provision.socket_share_required(): + assert _provision.socket_handoff_available() + assert _provision.default_spawn_transport() == "stdio" + else: + assert _provision.default_spawn_transport() == "socket" + + def test_ssh_cannot_dial_back_on_windows(self) -> None: + # no AF_UNIX in CPython there, and Win32-OpenSSH cannot -R a unix socket + assert _provision.ssh_dialback_available() == ( + not _provision.socket_share_required() + ) + + def test_socket_transport_roundtrip(self) -> None: + # explicitly, on every platform: this is the only coverage the + # Windows socket.share() handoff gets + group = execnet.Group() + try: + gateway = group.makegateway("popen//transport=socket") + channel = gateway.remote_exec("channel.send(6 * 7)") + assert channel.receive(TESTTIMEOUT) == 42 + finally: + group.terminate(timeout=5.0) + @posix_only def test_socket_transport_keeps_the_protocol_off_stdio(self) -> None: group = execnet.Group() @@ -401,3 +447,221 @@ def test_dialback_argv_forwards_a_unix_socket(self) -> None: assert argv[argv.index("-R") + 1] == "/tmp/remote.sock:/tmp/local.sock" # a stale remote socket must not block the bind assert "StreamLocalBindUnlink=yes" in argv + + +class TestSocketWorkerSpawnFailure: + """A server that cannot start a worker must not leave a coordinator waiting. + + The coordinator connects and blocks for the ``b"1"`` handshake. Nothing + else will ever move it, so a failed spawn has to close the connection -- + otherwise one unsupported gateway wedges the whole session, which is what + ``socket//installvia=`` did on Windows. + """ + + def test_a_failed_spawn_closes_the_connection( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + import trio + + from execnet import _trio_host + + def boom(fd: int) -> Any: + raise RuntimeError("no worker for you") + + monkeypatch.setattr(_trio_host, "_spawn_socket_worker", boom) + + async def main() -> bytes: + ours, theirs = socket.socketpair() + server = trio.SocketStream(trio.socket.from_stdlib_socket(theirs)) + with pytest.raises(RuntimeError, match="no worker"): + await _trio_host.serve_socket_connection(server, reap=False) + # the coordinator's end: EOF, not silence + client = trio.SocketStream(trio.socket.from_stdlib_socket(ours)) + with trio.fail_after(5): + return await client.receive_some(1) + + assert trio.run(main) == b"" + + def test_a_host_that_cannot_hand_over_a_socket_refuses_up_front( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # refusing before the address is replied is what makes it diagnosable: + # afterwards the coordinator is already connecting, and a closed + # socket can only ever say "EOF". + from execnet import _trio_host + + monkeypatch.setattr(_provision, "socket_handoff_available", lambda: False) + sent: list[tuple[int, int, bytes]] = [] + + class FakeGateway: + def _send(self, code: int, channelid: int = 0, data: bytes = b"") -> None: + sent.append((code, channelid, data)) + + import trio + + from execnet._message import Message + from execnet._serialize import loads_internal + + gateway: Any = FakeGateway() + trio.run(_trio_host._start_socket_and_reply, gateway, 7, "localhost") + + assert len(sent) == 1 + code, channelid, data = sent[0] + assert code == Message.CHANNEL_CLOSE_ERROR + assert channelid == 7 + assert "cannot hand an accepted socket" in loads_internal(data) + + +class TestShareTransport: + """The Windows socket handoff. Only ``adopt`` is testable off Windows.""" + + def test_adopt_decodes_the_blob_out_of_the_config(self) -> None: + import base64 + + from execnet import _trio_worker + from execnet._trio_gateway import SHARE_KEY + + transport = _trio_worker.ShareTransport() + config = {"id": "gw", SHARE_KEY: base64.b64encode(b"blobby").decode("ascii")} + transport.adopt(config) + assert transport._blob == b"blobby" + # consumed, so it cannot be mistaken for worker configuration + assert SHARE_KEY not in config + + def test_adopt_without_a_blob_is_a_clear_error(self) -> None: + from execnet import _trio_worker + + transport = _trio_worker.ShareTransport() + with pytest.raises(SystemExit, match="protocol_share"): + transport.adopt({"id": "gw"}) + + def test_the_cli_accepts_the_flag(self) -> None: + ns = _cli._build_parser().parse_args( + ["worker", "--protocol-share", "--config", "{}"] + ) + assert ns.protocol_share is True + + def test_share_is_exclusive_with_the_other_transports(self) -> None: + with pytest.raises(SystemExit): + _cli._build_parser().parse_args( + ["worker", "--protocol-share", "--protocol-stdio", "--config", "{}"] + ) + + +class TestShareHandoffWiring: + """How the share blob gets from coordinator to worker. + + ``WSADuplicateSocket`` itself only exists on Windows, but everything + around it -- the flag in argv, the config on stdin, the blob inside that + config -- is the part that can be wired up wrong, and that is testable + anywhere. + """ + + def test_popen_spawn_puts_the_flag_in_argv_and_the_blob_in_the_config( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + import base64 + + import trio + + from execnet import _trio_gateway + from execnet._trio_gateway import SHARE_KEY + + written: list[bytes] = [] + seen: dict[str, Any] = {} + + class FakeStdin: + async def send_all(self, data: bytes) -> None: + written.append(data) + + async def aclose(self) -> None: + pass + + class FakeProcess: + pid = 4321 + stdin = FakeStdin() + + async def fake_open_process(args: list[str], **kwargs: Any) -> Any: + seen["args"] = args + seen["kwargs"] = kwargs + return FakeProcess() + + monkeypatch.setattr(_provision, "socket_share_required", lambda: True) + monkeypatch.setattr(trio.lowlevel, "open_process", fake_open_process) + monkeypatch.setattr( + _trio_gateway, + "share_socket", + lambda sock, pid: base64.b64encode(b"dup-for-%d" % pid).decode("ascii"), + ) + + spec = execnet.XSpec("popen//id=gw0") + ours, theirs = socket.socketpair() + try: + trio.run(_trio_gateway._spawn_with_socket, spec, theirs) + finally: + ours.close() + theirs.close() + + args = seen["args"] + assert "--protocol-share" in args + # the config cannot be in argv: the blob is not built until we have + # a pid, which is only true once the process exists + assert "--config-fd" in args + assert "--config" not in args + assert "pass_fds" not in seen["kwargs"] + + config = json.loads(b"".join(written)) + assert base64.b64decode(config[SHARE_KEY]) == b"dup-for-4321" + assert config["id"] == "gw0-worker" + + def test_server_side_spawn_shares_the_accepted_socket( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + import base64 + + from execnet import _trio_gateway + from execnet import _trio_host + from execnet._trio_gateway import SHARE_KEY + + written: list[bytes] = [] + seen: dict[str, Any] = {} + + class FakeStdin: + def write(self, data: bytes) -> None: + written.append(data) + + def close(self) -> None: + pass + + class FakePopen: + pid = 99 + stdin = FakeStdin() + + def fake_popen(args: list[str], **kwargs: Any) -> Any: + seen["args"] = args + seen["kwargs"] = kwargs + return FakePopen() + + monkeypatch.setattr(_provision, "socket_share_required", lambda: True) + monkeypatch.setattr(subprocess, "Popen", fake_popen) + monkeypatch.setattr( + _trio_gateway, + "share_socket", + lambda sock, pid: base64.b64encode(b"accepted-%d" % pid).decode("ascii"), + ) + + ours, theirs = socket.socketpair() + try: + _trio_host._spawn_socket_worker(theirs.fileno()) + # the socket we do not own must survive being viewed for share() + assert theirs.fileno() >= 0 + theirs.send(b"still open") + assert ours.recv(16) == b"still open" + finally: + ours.close() + theirs.close() + + assert "--protocol-share" in seen["args"] + assert "pass_fds" not in seen["kwargs"] + config = json.loads(b"".join(written)) + assert base64.b64decode(config[SHARE_KEY]) == b"accepted-99" From 39251b924da4589794bb2a4132e955ea96a2a300 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 05:32:26 +0200 Subject: [PATCH 74/91] fix: the last four Windows failures, three of them latent everywhere With the hang gone, Windows ran the whole suite for the first time: 522 passed, 4 failed. Three of the four are bugs that were always there and that Linux hid. `execnet server :0` reported a port nothing was listening on. A wildcard bind with an ephemeral port gives each address family its *own* random port -- trio documents this -- and only the first was reported. Which family comes first is platform-dependent, so Linux reported the IPv4 port a `127.0.0.1` client could reach and Windows reported the IPv6 one. All families now share the reported port, and a test pins it rather than relying on the ordering that made this pass here. test_basics wrote its generated check script with `write_text()`, i.e. in the locale encoding, and Python reads source as utf-8 (PEP 3120). A single em-dash in the concatenated `_message` source was enough to make the file unreadable wherever the locale is not utf-8. The fourth was mine: `test_single_thread_on_main` asked `resolve_transport` what the transport was without passing the default, so on Windows it was told "socket" while the worker actually ran on stdio. Co-Authored-By: Claude Opus 5 --- CHANGELOG.rst | 6 ++++++ src/execnet/_socketserver.py | 33 +++++++++++++++++++++++++++++++- testing/test_basics.py | 7 ++++++- testing/test_execmodel_trio.py | 5 ++++- testing/test_socketserver_cli.py | 25 ++++++++++++++++++++++++ 5 files changed, 73 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e6d9bd00..d4222550 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -45,6 +45,12 @@ by name. The launch command no longer needs ``head -c `` byte accounting, a ``mktemp`` prelude, or ``exec`` to keep an fd alive, and the protocol stream never carries a payload. +* ``execnet server :0`` reported a port nothing was listening on. Binding a + wildcard host with an ephemeral port gives *each* address family its own + random port, and only the first was reported -- so a client dialling the + other family found nothing. Which family comes first is platform + dependent, which is why this worked on Linux and not on Windows. All + families now share the reported port. * A worker that cannot be handed its socket now fails instead of hanging. A ``socket=``/``installvia=`` worker is spawned by the *server*, so a failure there used to leave the coordinator waiting forever on a diff --git a/src/execnet/_socketserver.py b/src/execnet/_socketserver.py index ee6c4191..1a950a87 100644 --- a/src/execnet/_socketserver.py +++ b/src/execnet/_socketserver.py @@ -12,6 +12,34 @@ from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from trio import SocketListener + + +async def _one_port( + listeners: list[SocketListener], host: str | None +) -> list[SocketListener]: + """Re-bind ``listeners`` so every address family shares a single port. + + Asking for port 0 on a wildcard host binds each family to its *own* + random port -- trio documents this. We report one address, so the other + family would then be listening somewhere the caller was never told + about, and which family gets reported first is platform-dependent: + IPv4 on Linux, IPv6 on Windows, where a client dialling 127.0.0.1 was + told a port nothing was listening on. + """ + import trio + + ports = {listener.socket.getsockname()[1] for listener in listeners} + if len(ports) < 2: + return listeners + port = listeners[0].socket.getsockname()[1] + for listener in listeners: + await listener.aclose() + return await trio.open_tcp_listeners(port, host=host) + async def serve(hostport: str, once: bool) -> None: """Bind ``hostport`` and hand accepted connections to worker processes.""" @@ -20,7 +48,10 @@ async def serve(hostport: str, once: bool) -> None: from execnet import _trio_host host, _, port_str = hostport.rpartition(":") - listeners = await trio.open_tcp_listeners(int(port_str), host=host or None) + bind_host = host or None + listeners = await trio.open_tcp_listeners(int(port_str), host=bind_host) + if int(port_str) == 0: + listeners = await _one_port(listeners, bind_host) addr = listeners[0].socket.getsockname() # Report the bound address (port may be ephemeral) for callers to read. print("execnet-socketserver listening on %s %s" % (addr[0], addr[1]), flush=True) diff --git a/testing/test_basics.py b/testing/test_basics.py index 4cebf7ef..c9fcae7c 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -143,11 +143,16 @@ def run_check( ) -> subprocess.CompletedProcess[str]: self.idx += 1 check_path = self.path / f"check{self.idx}.py" - check_path.write_text(script) + # utf-8 explicitly: source is utf-8 by default (PEP 3120), so writing + # it in the locale encoding produces a file the interpreter cannot + # read back wherever that is not utf-8 -- Windows, where a single + # em-dash in the concatenated source was enough to break it. + check_path.write_text(script, encoding="utf-8") return subprocess.run( [self.python, os.fspath(check_path), *extra_args], capture_output=True, text=True, + encoding="utf-8", check=True, **process_args, ) diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py index eeff80f2..419a2499 100644 --- a/testing/test_execmodel_trio.py +++ b/testing/test_execmodel_trio.py @@ -67,7 +67,10 @@ def test_single_thread_on_main(self, trio_gw: Gateway) -> None: # the loop. The *transport* may: adopting inherited stdio has no # async form on Windows, so those reads and writes run in the thread # pool. Only the socket transport is genuinely single-threaded. - if _provision.resolve_transport(trio_gw.spec) == "socket": + transport = _provision.resolve_transport( + trio_gw.spec, default=_provision.default_spawn_transport() + ) + if transport == "socket": assert active == 1 def test_concurrent_execs_cooperate(self, trio_gw: Gateway) -> None: diff --git a/testing/test_socketserver_cli.py b/testing/test_socketserver_cli.py index 2b6d60dd..2156ad50 100644 --- a/testing/test_socketserver_cli.py +++ b/testing/test_socketserver_cli.py @@ -56,3 +56,28 @@ def test_socketserver_cli_roundtrip(socketserver_port: int) -> None: assert channel.receive() == 42 finally: group.terminate(timeout=5.0) + + +def test_ephemeral_port_is_the_same_for_every_address_family() -> None: + """One reported port has to be *the* port. + + A wildcard bind with port 0 gives each address family its own random + port, and only the first is reported -- so a client dialling the other + family finds nothing there. Which family comes first is + platform-dependent (IPv4 on Linux, IPv6 on Windows), so this passed by + luck here while failing there. + """ + import trio + + from execnet import _socketserver + + async def main() -> set[int]: + listeners = await trio.open_tcp_listeners(0, host=None) + listeners = await _socketserver._one_port(listeners, None) + try: + return {l.socket.getsockname()[1] for l in listeners} + finally: + for l in listeners: + await l.aclose() + + assert len(trio.run(main)) == 1 From e8ff98dcfd395f5c75de74d187d03d64dade18ec Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 06:14:24 +0200 Subject: [PATCH 75/91] fix: survive a host that cannot hand a socket to a worker (PyPy on Windows) 16 of 17 CI jobs are green; this is the seventeenth. PyPy on Windows has `socket.share` as a name but the call does not work, so the capability check passed and the failure surfaced three steps later as `EOFError: bad socket handshake: b''` -- with the master gateway dead and no explanation anywhere. Three separate defects behind that one symptom: * the capability is now settled by *doing* it once, against our own pid, rather than by `hasattr`. A host that cannot hand over a socket refuses the request up front, where a reason can still reach the coordinator. * a failed socket gateway no longer kills the gateway it was requested through. It runs as a task on that worker's host, so asking for one unsupported sub-gateway cost the master too -- which is why the errors cascaded across 51 tests instead of failing one. * `channel.send`/`receive` check for a foreign event loop before checking whether the channel is closed. Both are caller bugs, but which one you were told about depended on whether the peer had closed yet, so the event-loop diagnostic lost a race on the slower interpreter. The socket transport simply cannot work where neither `pass_fds` nor a working `share()` exists, so those gateways skip there with that reason rather than failing. Removing the limitation means not handing the socket over at all -- spawning the worker as the listener and reporting its address back -- which is a bigger change than this one. Co-Authored-By: Claude Opus 5 --- CHANGELOG.rst | 15 +++++++++++++++ src/execnet/_channel.py | 7 +++++-- src/execnet/_provision.py | 20 ++++++++++++++++++-- src/execnet/_trio_host.py | 9 ++++++++- testing/conftest.py | 6 ++++++ testing/test_socketserver_cli.py | 13 ++++++++++--- testing/test_xspec.py | 4 ++++ 7 files changed, 66 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d4222550..4ac4df6e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -45,6 +45,21 @@ by name. The launch command no longer needs ``head -c `` byte accounting, a ``mktemp`` prelude, or ``exec`` to keep an fd alive, and the protocol stream never carries a payload. +* ``channel.send()`` and ``channel.receive()`` check for a foreign event + loop before they check whether the channel is still open. Calling a + blocking API from inside a loop is a caller bug either way, and which of + the two errors you got depended on whether the peer had closed yet -- + so the more useful message lost a race. +* Whether a socket can be handed to a worker is now settled by *doing* it + once rather than by looking for ``socket.share``. An implementation with + the name but not a working call -- PyPy on Windows -- otherwise passed + the check and failed later, at the point where the only thing left to + tell the coordinator was a closed socket. Such a host now refuses the + request up front, and ``socket=``/``installvia=`` gateways are skipped + there rather than failing. +* A socket gateway that fails to start no longer takes down the gateway it + was requested through. It ran as a task on that worker's host, so an + unsupported sub-gateway used to cost the master as well. * ``execnet server :0`` reported a port nothing was listening on. Binding a wildcard host with an ephemeral port gives *each* address family its own random port, and only the first was reported -- so a client dialling the diff --git a/src/execnet/_channel.py b/src/execnet/_channel.py index 611dbe2f..0745831b 100644 --- a/src/execnet/_channel.py +++ b/src/execnet/_channel.py @@ -328,9 +328,12 @@ def send(self, item: object) -> None: OSError is raised if the write pipe was prematurely closed. """ + # before the state check: calling a blocking API from inside an event + # loop is a bug in the caller either way, and whether the channel has + # closed yet is a race -- the diagnostic should not depend on it + self.gateway._check_event_loop("channel.send()") if self.isclosed(): raise OSError(f"cannot send to {self!r}") - self.gateway._check_event_loop("channel.send()") self.gateway._send(Message.CHANNEL_DATA, self.id, dumps_internal(item)) def receive(self, timeout: float | None = None) -> Any: @@ -344,10 +347,10 @@ def receive(self, timeout: float | None = None) -> Any: reraised as channel.RemoteError exceptions containing a textual representation of the remote traceback. """ + self.gateway._check_event_loop("channel.receive()") mailbox = self._mailbox if mailbox is None: raise OSError("cannot receive(), channel has receiver callback") - self.gateway._check_event_loop("channel.receive()") x = mailbox.get(timeout) if x is ENDMARKER: mailbox.put(x) # for other receivers diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index 2da64eb7..2fa76c8d 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -43,6 +43,7 @@ TRANSPORTS = ("socket", "stdio") +@cache def socket_handoff_available() -> bool: """Whether we can hand a socket to a worker process we spawn ourselves. @@ -51,11 +52,26 @@ def socket_handoff_available() -> bool: (``WSADuplicateSocket``) duplicates the socket into a named pid, and the resulting blob rides in the worker config -- see the ``share`` transport in ``_trio_worker``. + + The Windows answer is settled by *doing* it once, against our own pid, + rather than by looking for the method. An implementation that has the + name but not a working call -- PyPy on Windows -- would otherwise pass + the check and fail at the point where the only thing left to tell the + coordinator is a closed socket. """ import socket as _socket - if sys.platform.startswith("win"): - return hasattr(_socket.socket, "share") + if not socket_share_required(): + return True + try: + left, right = _socket.socketpair() + try: + left.share(os.getpid()) # type: ignore[attr-defined] # Windows + finally: + left.close() + right.close() + except Exception: + return False return True diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index e114805d..3c219f7d 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -901,7 +901,14 @@ async def _start_socket_and_reply( stream = await listeners[0].accept() for listener in listeners: await listener.aclose() - await serve_socket_connection(stream, reap=True) + try: + await serve_socket_connection(stream, reap=True) + except Exception as exc: + # This runs as a task on *this worker's* host: letting it propagate + # tears the whole gateway down, so a coordinator asking for one + # unsupported sub-gateway would lose the master it asked through. + # The connection is already closed, so the coordinator gets its EOF. + trace(f"socket gateway for channel {channelid} failed: {exc!r}") def handle_start_socket(gateway: BaseGateway, channelid: int, data: bytes) -> None: diff --git a/testing/conftest.py b/testing/conftest.py index 29231d27..22c95006 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -12,6 +12,7 @@ import execnet from execnet import Gateway +from execnet import _provision from execnet._execmodel import ExecModel from execnet._execmodel import get_execmodel @@ -188,6 +189,11 @@ def gw( if request.param == "popen": gw = group.makegateway("popen//id=popen//profile=%s" % profile) elif request.param == "socket": + if not _provision.socket_handoff_available(): + # the server accepts the connection and must then give it to + # a worker process; where neither pass_fds nor a working + # socket.share() exists (PyPy on Windows) there is no way to + pytest.skip("this interpreter cannot hand a socket to a worker") pname = "sproxy1" if pname not in group: proxygw = group.makegateway("popen//id=%s" % pname) diff --git a/testing/test_socketserver_cli.py b/testing/test_socketserver_cli.py index 2156ad50..279501b0 100644 --- a/testing/test_socketserver_cli.py +++ b/testing/test_socketserver_cli.py @@ -14,12 +14,19 @@ import pytest import execnet +from execnet import _provision SERVER = shutil.which("execnet-socketserver") -pytestmark = pytest.mark.skipif( - SERVER is None, reason="execnet-socketserver console script not installed" -) +pytestmark = [ + pytest.mark.skipif( + SERVER is None, reason="execnet-socketserver console script not installed" + ), + pytest.mark.skipif( + not _provision.socket_handoff_available(), + reason="the server must hand the accepted socket to a worker process", + ), +] @pytest.fixture diff --git a/testing/test_xspec.py b/testing/test_xspec.py index f81afaf5..a516b4f3 100644 --- a/testing/test_xspec.py +++ b/testing/test_xspec.py @@ -253,6 +253,10 @@ def test_socket_second( assert rinfo.cwd == rinfo2.cwd assert rinfo.version_info == rinfo2.version_info + @pytest.mark.skipif( + not _provision.socket_handoff_available(), + reason="the server must hand the accepted socket to a worker process", + ) def test_socket_installvia(self) -> None: group = execnet.Group() group.makegateway("popen//id=p1") From a45c4d2e93ae5c32313a0e03a7009e9ff00fb50b Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 06:35:20 +0200 Subject: [PATCH 76/91] fix: probe both halves of the socket share, and a racy rinfo test The share probe called `share()` and ignored what came back, so PyPy on Windows passed it and still failed for real: the coordinator's half is `share()`, the worker's is `fromshare()`, and either can be the one that does not work. Sharing to our own pid is what makes testing both cheap -- the blob we produce is one we are entitled to rebuild, so the round trip needs no second process. An empty blob now also counts as failure. test__rinfo raced: `receive()` returns when the *send* arrives and the `os.chdir('..')` runs after it, so the following `_rinfo(update=True)` could observe the old cwd. It waits for the exec to finish now. Not caused by the share work -- it went red on ubuntu PyPy, which had been green, because it was always a race and PyPy is slow enough to lose it. Co-Authored-By: Claude Opus 5 --- src/execnet/_provision.py | 21 +++++++++++++++------ testing/test_gateway.py | 9 +++++++-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index 2fa76c8d..70cddbb3 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -53,11 +53,16 @@ def socket_handoff_available() -> bool: resulting blob rides in the worker config -- see the ``share`` transport in ``_trio_worker``. - The Windows answer is settled by *doing* it once, against our own pid, - rather than by looking for the method. An implementation that has the - name but not a working call -- PyPy on Windows -- would otherwise pass - the check and fail at the point where the only thing left to tell the - coordinator is a closed socket. + The Windows answer is settled by *doing* it once rather than by looking + for the method: an implementation that has the name but not a working + call would otherwise pass the check and fail later, at the point where + the only thing left to tell the coordinator is a closed socket. + + Both halves get exercised, and both matter -- the coordinator calls + ``share()``, the worker calls ``fromshare()``, and either can be the + one that is missing. Sharing to our *own* pid is what makes that + possible in one process: the blob we produce is one we are entitled to + rebuild, so a full round trip needs no second process to test. """ import socket as _socket @@ -66,7 +71,11 @@ def socket_handoff_available() -> bool: try: left, right = _socket.socketpair() try: - left.share(os.getpid()) # type: ignore[attr-defined] # Windows + blob = left.share(os.getpid()) # type: ignore[attr-defined] # Windows + if not blob: + return False + # the worker's half: a blob we cannot rebuild is no use to it + _socket.fromshare(blob).close() # type: ignore[attr-defined] # Windows finally: left.close() right.close() diff --git a/testing/test_gateway.py b/testing/test_gateway.py index ac845b38..57e12979 100644 --- a/testing/test_gateway.py +++ b/testing/test_gateway.py @@ -269,14 +269,19 @@ def test__rinfo(self, gw: Gateway) -> None: assert rinfo.cwd assert rinfo.version_info assert repr(rinfo) - old = gw.remote_exec( + chdir = gw.remote_exec( """ import os.path cwd = os.getcwd() channel.send(os.path.basename(cwd)) os.chdir('..') """ - ).receive() + ) + old = chdir.receive() + # receive() returns when the *send* arrives, and the chdir happens + # after it -- so without waiting for the exec to finish, the _rinfo + # below races it. Slower interpreters lose that race. + chdir.waitclose(TESTTIMEOUT) try: rinfo2 = gw._rinfo() assert rinfo2.cwd == rinfo.cwd From 5365105e0d2c92e87b0ec82ac9b2066b4a9c816b Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 07:40:48 +0200 Subject: [PATCH 77/91] fix: give share() the accepted socket's real family/type/proto Narrowed by what *passed*: `popen//transport=socket` is green on Windows PyPy, so `share()` to a child and `fromshare()` in it both work there. Only the server-side path failed -- and the two differ in exactly one thing. The popen path shares a socket object it already has. The server path had only the accepted fd, and wrapped it with `socket.socket(fileno=fd)`, which makes the constructor *detect* family/type/proto by querying the handle. Hand it what the caller already knows instead: trio's socket carries all three, and skipping the detection removes the only step the working path does not perform. Also explains why no probe caught this -- both halves of the mechanism work in-process on PyPy, so nothing short of the real accepted socket could have shown it. Co-Authored-By: Claude Opus 5 --- src/execnet/_trio_host.py | 15 +++++++++++---- testing/test_cli.py | 4 ++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 3c219f7d..bfcd8e42 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -796,13 +796,19 @@ def makegateway_trio(group: Group, spec: Any) -> Gateway: _socket_worker_counter = itertools.count() -def _spawn_socket_worker(fd: int) -> subprocess.Popen[bytes]: - """Spawn a worker subprocess serving the accepted socket ``fd``. +def _spawn_socket_worker(sock: Any) -> subprocess.Popen[bytes]: + """Spawn a worker subprocess serving the accepted socket ``sock``. POSIX hands the fd over with ``pass_fds``. Windows has no such thing, so the socket is duplicated into the child with ``WSADuplicateSocket`` and the blob travels in the config -- which is why the config goes to stdin there: it cannot be built until the child's pid exists. + + Takes the socket rather than its fd because ``share()`` needs a stdlib + socket object, and building one from a bare fd makes the constructor + *detect* family/type/proto by querying the handle. Passing what the + caller already knows skips that: it is the one difference between this + path and the popen one, and the popen one works where this did not. """ import execnet @@ -817,6 +823,7 @@ def _spawn_socket_worker(fd: int) -> subprocess.Popen[bytes]: "coordinator_version": execnet.__version__, } argv = [sys.executable, "-m", "execnet", "worker"] + fd = sock.fileno() if not _provision.socket_share_required(): return subprocess.Popen( [*argv, "--protocol-fd", str(fd), "--config", json.dumps(config)], @@ -831,7 +838,7 @@ def _spawn_socket_worker(fd: int) -> subprocess.Popen[bytes]: try: # a view on the accepted socket, so share() can reach it; detach so # dropping the view does not close the fd we do not own - view = _socket.socket(fileno=fd) + view = _socket.socket(sock.family, sock.type, sock.proto, fileno=fd) try: config[SHARE_KEY] = share_socket(view, process.pid) finally: @@ -856,7 +863,7 @@ async def serve_socket_connection(stream: trio.SocketStream, *, reap: bool) -> N the only thing that will move it -- without this it waits forever. """ try: - proc = _spawn_socket_worker(stream.socket.fileno()) + proc = _spawn_socket_worker(stream.socket) except BaseException: with trio.CancelScope(shield=True), suppress(Exception): await stream.aclose() diff --git a/testing/test_cli.py b/testing/test_cli.py index 205201ac..e88e3f29 100644 --- a/testing/test_cli.py +++ b/testing/test_cli.py @@ -465,7 +465,7 @@ def test_a_failed_spawn_closes_the_connection( from execnet import _trio_host - def boom(fd: int) -> Any: + def boom(sock: Any) -> Any: raise RuntimeError("no worker for you") monkeypatch.setattr(_trio_host, "_spawn_socket_worker", boom) @@ -652,7 +652,7 @@ def fake_popen(args: list[str], **kwargs: Any) -> Any: ours, theirs = socket.socketpair() try: - _trio_host._spawn_socket_worker(theirs.fileno()) + _trio_host._spawn_socket_worker(theirs) # the socket we do not own must survive being viewed for share() assert theirs.fileno() >= 0 theirs.send(b"still open") From f05f66decc8f373c7616e99415e13df15113b3a8 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 08:12:43 +0200 Subject: [PATCH 78/91] fix: hand adopt_socket the socket, not its fd The previous commit moved the Windows PyPy failure rather than fixing it: the handshake EOFs are gone (52 -> 0) and the real exception finally surfaced, one layer deeper -- in the worker rather than in the coordinator that spawned it. _trio_worker.py:660 in open: adopt_socket(sock.detach()) _trio_host.py:83 in adopt_socket: socket.socket(fileno=socket_fd) OSError: [WinError 10014] The system detected an invalid pointer address in attempting to use a pointer argument in a call Same defect, second site. `ShareTransport.open` had a perfectly good socket from `fromshare()`, detached it to a bare fd, and handed that to `adopt_socket` -- which rebuilds a socket from the handle and so must re-derive family/type/proto by querying it. That derivation is what PyPy on Windows cannot do to a handle that arrived through `WSADuplicateSocket`. `adopt_socket` now takes either, and the share transport passes the socket it already has. Reducing a socket to an integer and asking the next layer to reconstitute it was never buying anything. Co-Authored-By: Claude Opus 5 --- src/execnet/_trio_host.py | 14 +++++++++++--- src/execnet/_trio_worker.py | 6 ++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index bfcd8e42..bf069a13 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -72,15 +72,23 @@ def _run_callback(callback: Callable[[Any], Any], data: bytes, channel: Any) -> ssh_trio_args = ssh_transport_args -async def adopt_socket(socket_fd: int) -> trio.SocketStream: - """Worker side: wrap an inherited socket fd and send the handshake. +async def adopt_socket(sock: int | Any) -> trio.SocketStream: + """Worker side: wrap an inherited socket and send the handshake. + + Takes an fd or an already-built socket. Rebuilding one from its fd + makes the constructor *detect* family/type/proto by querying the + handle, which is not free and not universally reliable -- PyPy on + Windows raises ``WinError 10014`` doing it to a handle that arrived + from ``socket.fromshare()``. A caller holding a real socket should + hand it over rather than reduce it to an integer first. Runs on the Trio host loop. The coordinator waits for ``b"1"`` before starting the Message protocol; the worker config comes from the CLI. """ import socket as _socket - sock = _socket.socket(fileno=socket_fd) + if isinstance(sock, int): + sock = _socket.socket(fileno=sock) stream = trio.SocketStream(trio.socket.from_stdlib_socket(sock)) await stream.send_all(b"1") return stream diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index f07e6f85..47d7a008 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -656,8 +656,10 @@ async def open(self) -> Any: assert self._blob is not None, "adopt() first" sock = _socket.fromshare(self._blob) # type: ignore[attr-defined] # Windows - # adopt_socket takes ownership of the fd and sends the handshake - return await _trio_host.adopt_socket(sock.detach()) + # hand over the socket itself, not its fd: fromshare() already knows + # what this socket is, and making adopt_socket re-derive that from + # the bare handle is what PyPy on Windows cannot do + return await _trio_host.adopt_socket(sock) class FdTransport: From 3a8f182027668f3db21578347f2c822aebddf1d7 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 08:27:59 +0200 Subject: [PATCH 79/91] docs: call the relaying gateway a coordinator, not a master `via=`/`installvia=` gateways spawn and relay for the sub-worker they create, which is what a coordinator does -- the old name said nothing about the role and carried baggage besides. Comments, docstrings, local variables, test gateway ids and the proxy example all follow. Where one sentence covers both parties, only the relaying one is named "coordinator"; the process that requested it is left as "here" or "us", since calling both by the same word is what made the old wording tempting. Also refreshes two doc examples that still expected "thread model" in a gateway repr -- it has said "thread profile" since the `execmodel=` -> `profile=` rename, so those doctests were already stale. The `set_execmodel` section in basics.rst is stale for the same reason and wants a rewrite of its own. Co-Authored-By: Claude Opus 5 --- CHANGELOG.rst | 2 +- doc/basics.rst | 2 +- doc/example/test_group.rst | 4 ++-- doc/example/test_proxy.rst | 16 ++++++++-------- src/execnet/_multi.py | 2 +- src/execnet/_provision.py | 10 +++++----- src/execnet/_trio_gateway.py | 20 +++++++++++--------- src/execnet/_trio_host.py | 22 +++++++++++----------- testing/test_multi.py | 10 +++++----- testing/test_provision.py | 2 +- testing/test_ssh_local.py | 10 +++++----- testing/test_trio_gateway.py | 14 +++++++------- 12 files changed, 58 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4ac4df6e..e04e0b26 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -59,7 +59,7 @@ there rather than failing. * A socket gateway that fails to start no longer takes down the gateway it was requested through. It ran as a task on that worker's host, so an - unsupported sub-gateway used to cost the master as well. + unsupported sub-gateway used to cost that coordinator as well. * ``execnet server :0`` reported a port nothing was listening on. Binding a wildcard host with an ephemeral port gives *each* address family its own random port, and only the first was reported -- so a client dialling the diff --git a/doc/basics.rst b/doc/basics.rst index 8378dbc3..105f1c0f 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -153,7 +153,7 @@ you create any gateways:: You can execute this little test file:: $ python threadmodel.py - + 1 diff --git a/doc/example/test_group.rst b/doc/example/test_group.rst index dd6275b5..0cb10b78 100644 --- a/doc/example/test_group.rst +++ b/doc/example/test_group.rst @@ -14,7 +14,7 @@ multiple gateways:: >>> group >>> list(group) - [, ] + [, ] >>> 'gw0' in group and 'gw1' in group True >>> group['gw0'] == group[0] @@ -37,7 +37,7 @@ Pass an ``id=MYNAME`` part to ``group.makegateway``. Example:: >>> gw = group.makegateway("popen//id=sub1") >>> assert gw.id == "sub1" >>> group['sub1'] - + Getting (auto) IDs before instantiation ------------------------------------------------------ diff --git a/doc/example/test_proxy.rst b/doc/example/test_proxy.rst index f2af992d..e983ef67 100644 --- a/doc/example/test_proxy.rst +++ b/doc/example/test_proxy.rst @@ -5,21 +5,21 @@ Simple proxying ---------------- Using the ``via`` arg of specs we can create a gateway -whose io is created on a remote gateway and proxied to the master. +whose io is created on a remote gateway and proxied to the coordinator. -The simplest use case, is where one creates one master process +The simplest use case, is where one creates one coordinator process and uses it to control new workers and their environment :: >>> import execnet >>> group = execnet.Group() - >>> group.defaultspec = 'popen//via=master' - >>> master = group.makegateway('popen//id=master') - >>> master - + >>> group.defaultspec = 'popen//via=coordinator' + >>> coordinator = group.makegateway('popen//id=coordinator') + >>> coordinator + >>> worker = group.makegateway() >>> worker - + >>> group - + diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py index 293be4da..ac0b86d1 100644 --- a/src/execnet/_multi.py +++ b/src/execnet/_multi.py @@ -294,7 +294,7 @@ def terminate(self, timeout: float | None = None) -> None: if gw.id not in vias: gw.exit() if self._async_group is not None: - # Tunneled (via) gateways terminate before their masters, + # Tunneled (via) gateways terminate before their coordinators, # each with a GATEWAY_TERMINATE + timeout grace, then kill; # bounded at roughly twice the timeout (issues #43 / #221). try: diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index 70cddbb3..a20a9f35 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -592,14 +592,14 @@ def vagrant_ssh_argv( def spawn_request(spec: Any) -> dict[str, Any]: - """Payload for ``GATEWAY_START_SUB``: ask a via master to spawn a sub-worker. + """Payload for ``GATEWAY_START_SUB``: ask a via coordinator to spawn a sub-worker. Carries the sub-spec essentials plus provisioning material when the sub may need it (ssh or foreign python): a released coordinator sends a pip - requirement; a dev coordinator ships its wheel bytes for the master to + requirement; a dev build ships its wheel bytes for that coordinator to materialize into its local wheel cache. - TODO: the wheel is shipped eagerly because only the master can tell + TODO: the wheel is shipped eagerly because only that coordinator can tell whether the target interpreter already has execnet; a wheel-on-demand round-trip would avoid the transfer in the common provisioned case. """ @@ -653,14 +653,14 @@ def sub_spawn_argv( ) -> tuple[list[str], DeliveryStep | None]: """(argv, wheel delivery) spawning a requested sub-worker on this host. - Handles a ``GATEWAY_START_SUB`` request on a via master: plain popen runs + Handles a ``GATEWAY_START_SUB`` request on a via coordinator: plain popen runs this interpreter's worker module, a foreign ``python`` runs directly when it already has execnet and is uv-provisioned otherwise, and ``ssh`` wraps the remote uv command. A shipped wheel is delivered by the returned step -- its own ssh connection, run before the launch -- rather than framed into the launch command's stdin. - The sub's protocol is relayed over its stdio by the master, so it always + The sub's protocol is relayed over its stdio by that coordinator, so it always gets the stdio transport. """ config = request["config"] diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 4429bddf..5c960232 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -361,7 +361,7 @@ class RawChannelStream: """``ByteStream`` over a :class:`RawChannel` -- the frame-native via tunnel. The gateway writer performs one ``send_all`` per frame, so every raw - payload carries exactly one whole sub-protocol frame (the master relay + payload carries exactly one whole sub-protocol frame (the relay keeps that invariant in the other direction). ``receive_some`` buffers payloads and honours ``max_bytes`` for the handshake read. """ @@ -1283,25 +1283,27 @@ async def _resolve_socket_address(self, spec: Any) -> tuple[tuple[str, int], str ``installvia=`` asks that group member to start a one-shot socketserver first. The sync facade overrides this to talk to its - sync master gateway. + sync coordinator gateway. """ if getattr(spec, "installvia", None): - master = self._gateway_by_id(spec.installvia) - realhost, realport = await start_socketserver_via(master) + coordinator = self._gateway_by_id(spec.installvia) + realhost, realport = await start_socketserver_via(coordinator) return (realhost, realport), "%s:%d" % (realhost, realport) assert spec.socket is not None host_str, _, port_str = spec.socket.rpartition(":") return (host_str, int(port_str)), spec.socket async def _open_via_stream(self, spec: Any) -> ByteStream: - """Ask the ``spec.via`` master to spawn a sub-worker; tunnel over a + """Ask the ``spec.via`` coordinator to spawn a sub-worker; tunnel over a raw channel (each payload one whole sub-protocol frame).""" from . import _provision - master = self._gateway_by_id(spec.via) - raw = master.open_raw_channel() + coordinator = self._gateway_by_id(spec.via) + raw = coordinator.open_raw_channel() request = _provision.spawn_request(spec) - await master._send(Message.GATEWAY_START_SUB, raw.id, dumps_internal(request)) + await coordinator._send( + Message.GATEWAY_START_SUB, raw.id, dumps_internal(request) + ) stream = RawChannelStream(raw) await read_handshake_ack(stream, "via") return stream @@ -1316,7 +1318,7 @@ async def terminate(self, timeout: float | None = None) -> None: """Terminate all gateways; never hangs (kill after ``timeout``). Tunneled (``via``) gateways go first so their termination frames - still travel through a live master. + still travel through a live coordinator. """ gateways = list(self._gateways) self._gateways.clear() diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index bf069a13..c18637ee 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -721,7 +721,7 @@ class FacadeAsyncGroup(AsyncGroup): Runs on the group's :class:`TrioHost`. Gateways come out as :class:`SyncBridgeGateway` objects bound to freshly built sync ``Gateway`` facades, and the via / installvia flows go through the sync - master gateway (its dispatch is sync, so async channels cannot be used + sync coordinator gateway (its dispatch is sync, so async channels cannot be on it). """ @@ -744,25 +744,25 @@ def _make_gateway(self, stream: ByteStream, spec: Any) -> AsyncGateway: async def _open_via_stream(self, spec: Any) -> ByteStream: from . import _provision - master = self.group[spec.via] - session = master._trio_session + coordinator = self.group[spec.via] + session = coordinator._trio_session assert isinstance(session, SyncBridgeGateway) - channelid = master._channelfactory.allocate_id() + channelid = coordinator._channelfactory.allocate_id() # Create the raw channel before the request goes out so no relayed # frame can arrive unrouted (we are on the loop: no dispatch races). io = RawChannelStream(session._channel_for(channelid)) request = _provision.spawn_request(spec) - master._send(Message.GATEWAY_START_SUB, channelid, dumps_internal(request)) + coordinator._send(Message.GATEWAY_START_SUB, channelid, dumps_internal(request)) await read_handshake_ack(io, "via") return io async def _resolve_socket_address(self, spec: Any) -> tuple[tuple[str, int], str]: if getattr(spec, "installvia", None): - master = self.group[spec.installvia] - # Blocking sync channel receive on the master: run in a thread - # while this loop keeps dispatching the master's messages. + coordinator = self.group[spec.installvia] + # Blocking sync channel receive on that coordinator: run in a + # thread while this loop keeps dispatching its messages. realhost, realport = await trio.to_thread.run_sync( - start_socketserver_via, master, abandon_on_cancel=True + start_socketserver_via, coordinator, abandon_on_cancel=True ) return (realhost, realport), "%s:%d" % (realhost, realport) assert spec.socket is not None @@ -921,7 +921,7 @@ async def _start_socket_and_reply( except Exception as exc: # This runs as a task on *this worker's* host: letting it propagate # tears the whole gateway down, so a coordinator asking for one - # unsupported sub-gateway would lose the master it asked through. + # unsupported sub-gateway would lose the coordinator it asked through. # The connection is already closed, so the coordinator gets its EOF. trace(f"socket gateway for channel {channelid} failed: {exc!r}") @@ -975,7 +975,7 @@ async def _start_sub_and_relay( ) -> None: """Spawn a requested sub-worker and relay its Message protocol frames. - Runs on the master's Trio host (the ``via`` transport). The tunnel is + Runs on the coordinator's Trio host (the ``via`` transport). The tunnel is frame-native both ways: coordinator payloads arrive verbatim through the session's raw channel and go to the sub's stdin unchanged (each payload one whole frame), while the sub's stdout runs through a FrameDecoder so diff --git a/testing/test_multi.py b/testing/test_multi.py index 54eb858b..d0d01ad8 100644 --- a/testing/test_multi.py +++ b/testing/test_multi.py @@ -229,8 +229,8 @@ def fun(channel, arg) -> None: def test_terminate_with_proxying(self) -> None: group = Group() - group.makegateway("popen//id=master") - group.makegateway("popen//via=master//id=worker") + group.makegateway("popen//id=coordinator") + group.makegateway("popen//via=coordinator//id=worker") group.terminate(1.0) @pytest.mark.skipif( @@ -238,16 +238,16 @@ def test_terminate_with_proxying(self) -> None: reason="a via sub-spec ships provisioning material eagerly", ) def test_via_foreign_python(self) -> None: - # A python= sub-spec through a via master: the master resolves the + # A python= sub-spec through a via coordinator: it resolves the # interpreter locally (this interpreter has execnet, so the sub runs # the worker module directly, no uv provisioning). import sys group = Group() try: - group.makegateway("popen//id=master") + group.makegateway("popen//id=coordinator") gw = group.makegateway( - f"popen//python={sys.executable}//via=master//id=sub" + f"popen//python={sys.executable}//via=coordinator//id=sub" ) channel = gw.remote_exec("channel.send(channel.receive() + 1)") channel.send(41) diff --git a/testing/test_provision.py b/testing/test_provision.py index 97874b34..7f28465c 100644 --- a/testing/test_provision.py +++ b/testing/test_provision.py @@ -131,7 +131,7 @@ def test_makes_provisioning_available_without_a_source_tree( monkeypatch.setattr(_provision, "_editable_source_root", lambda: None) assert _provision.provisioning_available() - def test_ships_those_bytes_to_a_via_master( + def test_ships_those_bytes_to_a_via_coordinator( self, wheel, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(execnet, "__version__", "9.9.9") diff --git a/testing/test_ssh_local.py b/testing/test_ssh_local.py index cc300530..8c37155f 100644 --- a/testing/test_ssh_local.py +++ b/testing/test_ssh_local.py @@ -181,17 +181,17 @@ def test_ssh_roundtrip(ssh_config: str) -> None: def test_ssh_via_roundtrip(ssh_config: str) -> None: - """An ssh sub-gateway spawned by a popen master (GATEWAY_START_SUB relay). + """An ssh sub-gateway spawned by a popen coordinator (GATEWAY_START_SUB relay). - The master runs the ssh client; for a dev coordinator the wheel travels - coordinator -> master (in the spawn request) -> remote (ssh stdin preamble). + That coordinator runs the ssh client; for a dev build the wheel travels + from here into the spawn request, and on to the remote over ssh stdin. """ group = execnet.Group() try: - group.makegateway("popen//id=master") + group.makegateway("popen//id=coordinator") gw = group.makegateway( f"ssh=testhost//ssh_config={ssh_config}//python={sys.executable}" - "//via=master//id=sshvia" + "//via=coordinator//id=sshvia" ) channel = gw.remote_exec("channel.send(channel.receive() + 1)") channel.send(41) diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index fa4cf72d..19c8a9f9 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -377,21 +377,21 @@ async def main() -> None: trio.run(main) - def test_via_gateway_relays_through_master(self) -> None: + def test_via_gateway_relays_through_coordinator(self) -> None: async def main() -> None: async with AsyncGroup() as group: - master = await group.makegateway("popen//id=master") - sub = await group.makegateway("popen//via=master") - master_channel = await master.remote_exec( + coordinator = await group.makegateway("popen//id=coordinator") + sub = await group.makegateway("popen//via=coordinator") + coordinator_channel = await coordinator.remote_exec( "import os; channel.send(os.getpid())" ) sub_channel = await sub.remote_exec( "import os; channel.send(os.getpid())" ) - master_pid = await master_channel.receive() + coordinator_pid = await coordinator_channel.receive() sub_pid = await sub_channel.receive() - # a real second process, reached through the master's relay - assert sub_pid != master_pid + # a real second process, reached through the coordinator's relay + assert sub_pid != coordinator_pid echo = await sub.remote_exec("channel.send(channel.receive() * 2)") await echo.send(21) assert await echo.receive() == 42 From 2fc401304f0ee5feca4474de9439d7e5d9765eb9 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 08:39:30 +0200 Subject: [PATCH 80/91] feat!: default every spawned worker to the socket transport Windows was held on `transport=stdio` until the share path had a green run behind it. It has one now -- all 17 jobs, including `windows-latest, pypy-3.11`, which was the case that took four rounds to pin down. So the platform split goes away: a worker execnet spawns itself gets a socket, by whichever handoff the platform has. That also retires the machinery the split needed. `default=` existed only to say "can, but does not yet", and `default_spawn_transport()` only to compute it; with the answer the same on both platforms they collapse back into `resolve_transport(spec, available=...)`, which is where this started before Windows needed an exception. Two tests lose their `posix_only` marks and now cover Windows: that the protocol really is off fd 0/1 (asserting `--protocol-share` there rather than `--protocol-fd`), and that a worker's stdout reaches the coordinator. The visible consequence on Windows is that one: remote `print()` now goes to the coordinator's console instead of being folded onto stderr, matching POSIX. Co-Authored-By: Claude Opus 5 --- CHANGELOG.rst | 37 ++++++++++++++++++---------------- src/execnet/_provision.py | 24 +++------------------- src/execnet/_trio_gateway.py | 4 +--- testing/test_cli.py | 31 ++++++++++++++++------------ testing/test_execmodel_trio.py | 2 +- 5 files changed, 43 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e04e0b26..d56da718 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,11 +20,12 @@ ``import execnet, trio`` probe used to decide whether a ``python=`` interpreter can host a worker directly. * The protocol no longer has to be the worker's stdin/stdout. A new - ``transport=socket|stdio`` spec key selects; it defaults to ``socket`` on - POSIX and ``stdio`` on Windows, where neither ``pass_fds`` nor ``ssh -R`` - unix-socket forwarding is available. ``socket`` means an inherited - socketpair for ``popen`` and an ``ssh -R``-forwarded unix socket that the - worker dials back on for ``ssh=``/``vagrant_ssh=``. + ``transport=socket|stdio`` spec key selects, and ``socket`` is the default + for every worker execnet spawns itself: an inherited socketpair for + ``popen`` on POSIX, and a socket duplicated with ``socket.share()`` on + Windows. ``ssh=``/``vagrant_ssh=`` gateways use an ``ssh -R``-forwarded + unix socket the worker dials back on, which needs ``AF_UNIX`` and + ``StreamLocal`` forwarding, so those stay on ``stdio`` on Windows. * **A worker's stdio now belongs to the code it runs.** It used to be redirected to the null device so it could not corrupt the protocol, which meant a remote ``print()`` went nowhere at all. With the socket transport @@ -51,12 +52,12 @@ the two errors you got depended on whether the peer had closed yet -- so the more useful message lost a race. * Whether a socket can be handed to a worker is now settled by *doing* it - once rather than by looking for ``socket.share``. An implementation with - the name but not a working call -- PyPy on Windows -- otherwise passed - the check and failed later, at the point where the only thing left to - tell the coordinator was a closed socket. Such a host now refuses the - request up front, and ``socket=``/``installvia=`` gateways are skipped - there rather than failing. + once -- sharing to our own pid and rebuilding the result -- rather than + by looking for ``socket.share``. An implementation with the name but not + a working call would otherwise pass the check and fail later, at the + point where the only thing left to tell the coordinator is a closed + socket. A host that genuinely cannot hand a socket over refuses the + request up front instead. * A socket gateway that fails to start no longer takes down the gateway it was requested through. It ran as a task on that worker's host, so an unsupported sub-gateway used to cost that coordinator as well. @@ -80,10 +81,12 @@ ``socket.share()`` (``WSADuplicateSocket``); because that needs the child's pid, the flag travels in argv and the blob follows in the config on stdin. The blob is bound to that one pid, so it is inert to anything - else. This makes ``transport=socket`` work on Windows for ``popen`` and - fixes ``socket=``/``installvia=`` gateways served from a Windows host. - Windows still *defaults* to ``transport=stdio``; ask for - ``transport=socket`` to opt in. + else. This makes ``transport=socket`` the Windows default too, and fixes + ``socket=``/``installvia=`` gateways served from a Windows host. A socket + is handed over *as a socket* rather than reduced to its handle: rebuilding + one from a bare handle makes the constructor re-derive family/type/proto + by querying it, which PyPy on Windows cannot do to a handle that came + from ``WSADuplicateSocket``. * ``transport=socket`` on a gateway that cannot provide it is now an error at ``makegateway`` time naming the platform, instead of a gateway that waits for a worker which was never able to reach back. ssh dial-back @@ -95,8 +98,8 @@ POSIX-only. Windows has no async equivalent -- trio's Windows pipe streams need OVERLAPPED handles registered with an IOCP, and the stdio a process inherits is an ordinary synchronous pipe -- so those reads and writes now - run in the thread pool. The socket transport, where it is available, still - needs no threads. + run in the thread pool. This only affects ``transport=stdio``; the socket + transport, which is the default, needs no threads. * New ``EXECNET_PROVISION_WHEEL`` environment variable naming a prebuilt wheel to provision remote workers from, instead of resolving the coordinator's version from an index or building one from its source tree. diff --git a/src/execnet/_provision.py b/src/execnet/_provision.py index a20a9f35..cb08de51 100644 --- a/src/execnet/_provision.py +++ b/src/execnet/_provision.py @@ -107,35 +107,17 @@ def ssh_dialback_available() -> bool: return hasattr(_socket, "AF_UNIX") and not sys.platform.startswith("win") -def default_spawn_transport() -> str: - """What a worker we spawn gets when the spec does not say. - - Windows can *serve* the socket transport (see :func:`socket_share_required`), - but stdio is what has years of exercise there while sharing is new, so it - stays the default until the share path has CI behind it. - ``transport=socket`` opts in. - """ - if socket_share_required(): - return "stdio" - return "socket" if socket_handoff_available() else "stdio" - - -def resolve_transport( - spec: Any, *, available: bool = True, default: str | None = None -) -> str: - """The transport for ``spec``: explicit if given, else the default. +def resolve_transport(spec: Any, *, available: bool = True) -> str: + """The transport for ``spec``: explicit if given, else the best available. ``available`` is the caller's capability for *its* kind of gateway -- :func:`socket_handoff_available` for a worker we spawn, :func:`ssh_dialback_available` for one that has to reach back to us. Asking for a transport that cannot work is an error at makegateway time, - rather than a hang once nobody connects. ``default`` decouples "can - work" from "is what you get by default". + rather than a hang once nobody connects. """ requested: str | None = getattr(spec, "transport", None) if requested is None: - if default is not None: - return default return "socket" if available else "stdio" if requested not in TRANSPORTS: raise ValueError(f"unknown transport {requested!r} (known: {list(TRANSPORTS)})") diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 5c960232..64c3509e 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -1102,9 +1102,7 @@ async def connect_popen_worker(spec: Any) -> tuple[ByteStream, trio.Process]: from . import _provision transport = _provision.resolve_transport( - spec, - available=_provision.socket_handoff_available(), - default=_provision.default_spawn_transport(), + spec, available=_provision.socket_handoff_available() ) if transport == "stdio": return await connect_command_worker(popen_worker_argv(spec)) diff --git a/testing/test_cli.py b/testing/test_cli.py index e88e3f29..d2d59154 100644 --- a/testing/test_cli.py +++ b/testing/test_cli.py @@ -217,15 +217,18 @@ def test_no_config_source_is_an_error(self) -> None: class TestTransportSelection: - def test_defaults_per_platform(self) -> None: + def test_a_spawned_worker_defaults_to_the_socket_transport(self) -> None: + # every platform now: POSIX hands over the fd, Windows duplicates the + # socket with share(). Only a host that can do neither gets stdio. spec = execnet.XSpec("popen") - expected = "stdio" if sys.platform.startswith("win") else "socket" + expected = "socket" if _provision.socket_handoff_available() else "stdio" assert ( _provision.resolve_transport( - spec, default=_provision.default_spawn_transport() + spec, available=_provision.socket_handoff_available() ) == expected ) + assert expected == "socket" def test_explicit_wins(self) -> None: assert _provision.resolve_transport( @@ -252,15 +255,12 @@ def test_asking_for_an_impossible_transport_is_an_error(self) -> None: execnet.XSpec("ssh=host//transport=socket"), available=False ) - def test_windows_can_serve_the_socket_transport_but_does_not_default_to_it( - self, - ) -> None: - # sharing is new; stdio is what has years of Windows behind it + def test_windows_hands_a_socket_over_by_duplicating_it(self) -> None: + # `subprocess` refuses pass_fds there, so the capability comes from + # socket.share() instead -- including on PyPy, once the socket is + # handed over as a socket rather than rebuilt from its handle if _provision.socket_share_required(): assert _provision.socket_handoff_available() - assert _provision.default_spawn_transport() == "stdio" - else: - assert _provision.default_spawn_transport() == "socket" def test_ssh_cannot_dial_back_on_windows(self) -> None: # no AF_UNIX in CPython there, and Win32-OpenSSH cannot -R a unix socket @@ -279,8 +279,14 @@ def test_socket_transport_roundtrip(self) -> None: finally: group.terminate(timeout=5.0) - @posix_only def test_socket_transport_keeps_the_protocol_off_stdio(self) -> None: + # whichever handoff this platform has, the point is the same: the + # protocol is named on the command line, so it is not fd 0/1 + expected = ( + "--protocol-share" + if _provision.socket_share_required() + else "--protocol-fd" + ) group = execnet.Group() try: gateway = group.makegateway("popen") @@ -288,7 +294,7 @@ def test_socket_transport_keeps_the_protocol_off_stdio(self) -> None: "import sys; channel.send(sys.argv)", ) argv = channel.receive(TESTTIMEOUT) - assert "--protocol-fd" in argv + assert expected in argv finally: group.terminate(timeout=5.0) @@ -305,7 +311,6 @@ def test_stdio_transport_still_works(self) -> None: class TestWorkerStdio: """Whose stdio is it? The code the worker runs, unless told otherwise.""" - @posix_only def test_socket_transport_inherits_stdio(self, capfd) -> None: group = execnet.Group() try: diff --git a/testing/test_execmodel_trio.py b/testing/test_execmodel_trio.py index 419a2499..285a4ffa 100644 --- a/testing/test_execmodel_trio.py +++ b/testing/test_execmodel_trio.py @@ -68,7 +68,7 @@ def test_single_thread_on_main(self, trio_gw: Gateway) -> None: # async form on Windows, so those reads and writes run in the thread # pool. Only the socket transport is genuinely single-threaded. transport = _provision.resolve_transport( - trio_gw.spec, default=_provision.default_spawn_transport() + trio_gw.spec, available=_provision.socket_handoff_available() ) if transport == "socket": assert active == 1 From f484d144f4125d89862abef182acdcb9529e6307 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 10:13:57 +0200 Subject: [PATCH 81/91] fix: a killed worker is EOF on every transport, not a reset socket Fallout from defaulting Windows to the socket transport: two killed-worker tests went red on all six Windows jobs with BrokenResourceError('socket connection broken: [WinError 10054] An existing connection was forcibly closed by the remote host') where they expected EOFError. A peer that dies abruptly *resets* a socket, while a pipe just reaches EOF -- the same event, reported two ways, and until now the transport decided which. Callers test for EOFError and an endmarker callback has to fire either way, so the reader maps a broken read onto EOFError and keeps the original as __cause__. ClosedResourceError stays as it was: that is us closing, not the peer going away. This also fixes `socket=` gateways on POSIX, where a reset is equally possible over TCP and nothing was making it look like EOF either. Co-Authored-By: Claude Opus 5 --- CHANGELOG.rst | 6 ++++++ src/execnet/_trio_gateway.py | 14 +++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d56da718..9334ba05 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -46,6 +46,12 @@ by name. The launch command no longer needs ``head -c `` byte accounting, a ``mktemp`` prelude, or ``exec`` to keep an fd alive, and the protocol stream never carries a payload. +* A worker that dies abruptly reports ``EOFError`` on every transport. A + killed peer *resets* a socket -- Windows reports ``WSAECONNRESET`` -- + where a pipe would simply reach EOF, so the same event used to surface + as ``trio.BrokenResourceError`` on one transport and ``EOFError`` on the + other. Endmarker callbacks and ``channel._getremoteerror()`` now behave + the same either way. * ``channel.send()`` and ``channel.receive()`` check for a foreign event loop before they check whether the channel is still open. Calling a blocking API from inside a loop is a caller bug either way, and which of diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 64c3509e..5ea1d5cd 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -653,7 +653,19 @@ async def _reader(self) -> None: while True: try: data = await self._stream.receive_some(RECEIVE_CHUNK) - except (trio.BrokenResourceError, trio.ClosedResourceError) as exc: + except trio.BrokenResourceError as exc: + # A peer that died abruptly *resets* a socket -- Windows + # reports WSAECONNRESET -- where a pipe would simply have + # reached EOF. Same event, so report the same thing: + # callers test for EOFError, and an endmarker callback + # must fire either way. + if not self._closed: + error = EOFError(f"connection closed: {exc}") + error.__cause__ = exc + self._error = error + return + except trio.ClosedResourceError as exc: + # we closed it; not the peer going away if not self._closed: self._error = exc return From 0809f98b39733c975d374f5fda93b5dd6599c412 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 10:22:05 +0200 Subject: [PATCH 82/91] docs: hand off the transport and Windows work Covers what a fresh session needs: the CLI as launch contract, the transport matrix and why ssh on Windows cannot leave stdio, the share handoff and the socket-not-fd rule that took four CI rounds to find, the latent bugs that only surfaced once Windows actually ran, and the failure modes worth preserving. Leads with the CI outage, because the most useful thing to know is that every job reported green-looking failures while executing zero tests for four days -- and therefore that any platform CI has not really exercised should be assumed broken. Open work is listed with the reasoning attached, including the two ideas that were evaluated and *not* taken (a trampoline process) or not yet taken (spawning the installvia worker as the listener), so neither gets re-litigated from scratch. Co-Authored-By: Claude Opus 5 --- handoff-windows-and-transports.md | 187 ++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 handoff-windows-and-transports.md diff --git a/handoff-windows-and-transports.md b/handoff-windows-and-transports.md new file mode 100644 index 00000000..8462b785 --- /dev/null +++ b/handoff-windows-and-transports.md @@ -0,0 +1,187 @@ +# Handoff: transports off stdio, the CLI, and Windows + +For a fresh session on branch `feat/trio-host-thread-io` (draft PR +pytest-dev/execnet#422). Continues `handoff-phase-c-worker-axes.md`, +which covers the async core and the worker profile axes and is still +accurate for those; nothing here contradicts it. + +Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` +(never grep-filter pre-commit output). Local state at handoff: **552 +passed, 66 skipped**, pre-commit clean. + +## Read this first: CI was lying + +Until `1839800`, **CI had been executing zero tests since 2026-07-26**. +`testing/test_ssh_local.py` imported `asyncssh` at module level while +`tox.ini` carried its own hand-written `deps` list that had drifted from +the `testing` extra — so collection errored and every job reported +`2 skipped, 1 error` while looking like an ordinary failure. + +Two lessons worth keeping: + +- `tox.ini` now uses `extras = testing`; there must never be a second + list of test requirements. +- `[tool.uv] default-groups = ["testing"]` had been *replacing* uv's + `dev` default, so `uv sync` silently uninstalled pytest-xdist. The + `dev` group now absorbs `execnet[testing]` plus the tooling. + +Turning the suite back on is what surfaced almost everything below. +Assume any platform CI has not actually exercised is broken. + +## What landed this session + +| commit | what | +|---|---| +| `320c89e`, `17e7c7a` | run pytest-xdist's own suite against this execnet: a pinned `release` target that blocks, a floating `default-branch` target that may fail | +| `5710eed` | the `execnet` CLI — it is now the launch contract | +| `a84380f` | ssh dial-back over `ssh -R`, popen over a socketpair | +| `e43db63` | one typed endmarker sentinel | +| `1839800` | the CI fix above | +| `cd42ce9` | `EXECNET_PROVISION_WHEEL` | +| `9413a2e` … `f484d14` | Windows: eight commits, see below | + +## The CLI is the launch contract + +``` +execnet worker --protocol-stdio | --protocol-fd FD[,FD] + | --protocol-connect ADDR | --protocol-listen ADDR + | --protocol-share + --config JSON | --config-fd FD | --config-file PATH + --stdin/--stdout/--stderr DISPOSITION +execnet server [HOST:PORT] [--once] +execnet info +``` + +Everything that starts a worker emits these tokens; there is no second +launch path. `execnet info` (JSON: version, trio, executable, platform, +protocols) replaced the `import execnet, trio` probe, so provisioning +learns the remote version *before* connecting. + +**Config off argv**: `--config-fd 0` exists because the config carries +`env:` values and a remote argv is readable via `ps` by every user on +that host. ssh uses it. Do not regress this to `--config`. + +## Transports + +`transport=socket|stdio`. **`socket` is now the default for every worker +execnet spawns**, on both platforms (`2fc4013`). The worker's stdio is +then untouched, so remote `print()` reaches the coordinator. + +| gateway | handoff | notes | +|---|---|---| +| popen, POSIX | `pass_fds` + `--protocol-fd` | socketpair | +| popen, Windows | `socket.share()` + `--protocol-share` | see below | +| `socket=`/`installvia=` | same two, server-side | server accepts, then hands over | +| `ssh=`/`vagrant_ssh=` | `ssh -R` unix socket, worker dials back | POSIX only | + +ssh on Windows stays on stdio and **cannot** do otherwise: CPython has +never exposed `AF_UNIX` there (cpython#77589) and Win32-OpenSSH does not +implement `StreamLocal` forwarding. `resolve_transport` raises for an +impossible request rather than letting the gateway hang. + +### The share transport (Windows) + +`subprocess` refuses `pass_fds` on Windows. `socket.share(pid)` +(`WSADuplicateSocket`) duplicates the socket into a named pid instead — +but that needs the pid, which does not exist until the child is spawned. +Hence: **the flag goes in argv, the blob follows in the config on +stdin**. The blob is bound to that one pid, so it is inert to anything +else; this beats handle inheritance, which would need `close_fds=False` +and leak every inheritable handle to the child *and its grandchildren*. + +**The rule that took four CI rounds to learn: hand over a socket as a +socket, never as an fd.** Rebuilding one with `socket.socket(fileno=fd)` +makes the constructor re-derive family/type/proto by querying the handle, +and PyPy on Windows raises `WinError 10014` doing that to a handle from +`WSADuplicateSocket`. Both sites were wrong; both are fixed +(`5365105`, `f05f66d`). `adopt_socket` now takes either. + +No in-process probe can catch this: `share()` and `fromshare()` both work +in-process on PyPy. What cracked it was noticing which test *passed* — +`popen//transport=socket` was green while the server path failed, which +isolated the difference to one line. + +## Windows, which had never been tested + +Every Windows worker died at startup on `trio.lowlevel.FdStream`, which +is POSIX-only. Trio has Windows pipe streams but they need OVERLAPPED +handles registered with an IOCP, and inherited stdio is an ordinary +synchronous pipe — so `ThreadedFdStream` (`_trio_gateway.py`) does those +reads and writes in the thread pool. It is now only reachable via +explicit `transport=stdio`. + +Latent bugs that Linux was hiding, all found once Windows ran: + +- **`execnet server :0` reported a port nothing listened on.** A + wildcard bind with an ephemeral port gives *each* address family its + own random port (trio documents this) and only the first was reported. + Which family comes first is platform-dependent — IPv4 on Linux, IPv6 + on Windows. `_socketserver._one_port` re-binds them to one port. +- **`test_basics` wrote generated source in the locale encoding.** + Python reads source as UTF-8 (PEP 3120); one em-dash in `_message` was + enough to break it off UTF-8 locales. +- **A killed worker reported `BrokenResourceError`, not `EOFError`**, on + a socket: a dead peer *resets* a socket where a pipe reaches EOF. The + reader maps it (`f484d14`). Applies to `socket=` on POSIX too. +- **`test__rinfo` raced**: `receive()` returns when the *send* arrives, + and the `os.chdir('..')` after it had not run yet. + +## Failure modes to preserve + +These were all real, and each cost a debugging round: + +- **A socket worker that cannot be spawned must not hang the + coordinator.** It is spawned by the *server*, so the exception dies + there while the coordinator waits for a handshake byte. A host that + cannot hand a socket over refuses *before replying with an address* — + the last moment a reason can reach the coordinator — and a spawn that + fails anyway closes the connection so the wait ends. +- **A failed socket gateway must not kill the gateway it was requested + through.** It runs as a task on that coordinator's host; letting it + propagate cost the coordinator too, which is how one unsupported + gateway became 51 errors. +- **`_check_event_loop` runs before the channel-state check** in + `send`/`receive`. Both are caller bugs, but which one you were told + about depended on whether the peer had closed yet. + +## Naming + +`master` → `coordinator` throughout for the `via=`/`installvia=` gateway +(`3a8f182`): it spawns and relays for the sub-worker, which is what a +coordinator does. Where one sentence covers both parties, only the +relaying one is named; the requesting side is "here"/"us". + +## Open work + +1. **`doc/basics.rst` still documents `set_execmodel` / + `main_thread_only`** as the API, with a whole threading-models + section. Stale since the `execmodel=` → `profile=` rename; needs a + rewrite, not a substitution. Two doctests in `doc/example/` expected + `thread model` in a gateway repr and were already broken — fixed, but + note **docs are not built in CI**, only `tox -e py` runs. Consider + adding `tox -e docs` to the workflow. +2. **`installvia` still needs a socket handoff at all.** It would not, + if the server spawned the worker as the *listener* + (`--protocol-listen 127.0.0.1:0`) and reported its address back: no + `pass_fds`, no `share()`, works on any interpreter, and the spawn + happens *before* the reply so failures are diagnosable by + construction. The open question is how the server learns the port + (worker prints it, or writes it to a path given in the config). + Standalone `execnet server` still needs the handoff — it has already + accepted the connection. +3. **A trampoline process was considered and rejected** — see the + analysis: the current design already frees the worker's stdio + in-process (`_dup_protocol_fds` + `apply_stdio`), and a pump's own + stdio side is still a blocking pipe, so it relocates the thread + rather than removing it, at the cost of a process and two copies per + message. Do not revisit without a new reason. +4. `execnet.anyio` remains deferred (Phase E). + +## Invariants (do not regress) + +- No source shipping, ever. Workers import installed execnet+trio. +- The worker config never travels in a remote argv. +- `import execnet` must not import the trio event loop. +- Hand sockets over as sockets, not fds. +- Keep async-core idioms anyio-portable (neutral `ByteStream`, sans-IO + `FrameDecoder`). From 98eb08d0f62cec68cb29fd2d41932b4ea17b84d7 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 11:00:44 +0200 Subject: [PATCH 83/91] docs: describe execnet as it is now, and run the examples again doc/basics.rst still documented set_execmodel() and a threading-models section that no longer exists. It now covers the namespaces, the worker profiles that replaced execmodels, the full spec-key list including transport= and the stdio dispositions, the shared host thread, and the execnet command line; doc/implnotes.rst is rewritten around the launch contract, transports and provisioning rather than the source-shipping bootstrap it still described. New doc/api.rst is the namespace reference, which also gives the four :mod: targets something to resolve to. The examples said they were automatically tested. They were not: a pytest_plugins line in doc/example/conftest.py has made collecting that directory an error since pytest 7, so nobody noticed the trace sample predating trio, the callback notes describing the old receiver thread, or that basics.rst never imported execnet. They run again -- with the two examples that need a reachable ssh account marked +SKIP -- and tox -e docs runs both them and the -W sphinx build, from a new CI job. The docs had never been built in CI at all. Co-Authored-By: Claude Opus 5 --- .github/workflows/test.yml | 23 +++ CHANGELOG.rst | 9 ++ doc/api.rst | 108 +++++++++++++ doc/basics.rst | 304 +++++++++++++++++++++++++++++++---- doc/conf.py | 12 +- doc/example/conftest.py | 2 - doc/example/test_debug.rst | 57 +++---- doc/example/test_group.rst | 9 +- doc/example/test_info.rst | 54 ++++--- doc/example/test_multi.rst | 5 +- doc/examples.rst | 4 +- doc/implnotes.rst | 244 +++++++++++++++++++--------- doc/index.rst | 12 +- doc/install.rst | 1 + src/execnet/_trio_gateway.py | 4 +- tox.ini | 7 +- 16 files changed, 681 insertions(+), 174 deletions(-) create mode 100644 doc/api.rst diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bc6dad53..b4351ffe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,6 +26,29 @@ jobs: - name: Build and Check Package uses: hynek/build-and-inspect-python-package@v2.18 + # The docs build is `-W`, so a stale reference is an error, and the same + # env runs the doc examples as doctests -- they claim to be tested, and + # for years they silently were not. + docs: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.13" + + - name: Install dependencies + run: pip install tox + + - name: Build the docs and run the doc examples + run: tox run -e docs + test: needs: [package] diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9334ba05..53790bbe 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -185,6 +185,15 @@ * ``Gateway.remote_init_threads()`` raises a ``DeprecationWarning`` instead of printing to stdout. It has been a no-operation since execnet 1.2. +* The documentation describes what execnet does now: worker profiles instead + of threading models, the spec keys (including ``transport=`` and the stdio + dispositions), the namespaces, the host thread, the ``execnet`` command + line, and a namespace reference for the async surfaces. ``tox -e docs`` + now also *runs* the doc examples -- they had claimed to be automatically + tested while a ``pytest_plugins`` line in a non-top-level conftest made + collecting them an error -- and both it and the ``-W`` sphinx build run in + CI, where the docs had never been built at all. + * `#380 `__: Add support for Python 3.13 and 3.14, and drop EOL 3.8 and 3.9. * Trio host-thread Message IO for local ``popen`` + import bootstrap (coordinator and worker). Adds a hard ``trio`` dependency. Disable with ``EXECNET_TRIO_HOST=0``. diff --git a/doc/api.rst b/doc/api.rst new file mode 100644 index 00000000..c70a935a --- /dev/null +++ b/doc/api.rst @@ -0,0 +1,108 @@ +============================================================================== +Namespace reference +============================================================================== + +One namespace per concurrency library you drive execnet *from*. They all +speak the same protocol to the same kind of worker; see :doc:`basics` for +gateway specifications, channels and groups, which are common to all of +them. + +.. _execnet-sync: + +execnet.sync -- blocking +============================================================================== + +.. module:: execnet.sync + +The blocking API for plain threads, and the surface ``import execnet`` +gives you: the top-level ``execnet.*`` names are aliases into this module. +It is what :doc:`basics` documents. + +Calls block the calling thread while a Trio host thread does the protocol +IO, so calling one from inside a running asyncio or trio event loop raises +``RuntimeError`` rather than stalling every task on that loop. Use +:mod:`execnet.aio` or :mod:`execnet.trio` there. + +.. autoclass:: execnet.Host + :members: running, close + + +.. _execnet-trio: + +execnet.trio -- trio-native +============================================================================== + +.. automodule:: execnet.trio + +.. autoclass:: execnet.trio.AsyncGroup + :members: makegateway + +.. autoclass:: execnet.trio.AsyncGateway + :members: remote_exec, terminate + +.. autoclass:: execnet.trio.AsyncChannel + :members: send, receive, send_eof, aclose, wait_closed, isclosed + +.. autofunction:: execnet.trio.open_gateway + + +.. _execnet-aio: + +execnet.aio -- asyncio-native +============================================================================== + +.. automodule:: execnet.aio + +.. autoclass:: execnet.aio.AsyncGroup + :members: start, aclose, makegateway, host + +.. autoclass:: execnet.aio.AsyncGateway + :members: remote_exec, terminate + +.. autoclass:: execnet.aio.AsyncChannel + :members: send, receive, send_eof, aclose, wait_closed, isclosed + +.. autofunction:: execnet.aio.open_gateway + + +.. _execnet-gevent: + +execnet.gevent -- blocking, greenlet-parking +============================================================================== + +.. module:: execnet.gevent + +Identical to :mod:`execnet.sync` except that every blocking wait parks the +calling *greenlet* rather than its OS thread, so a slow ``receive`` no +longer stalls the whole hub:: + + import execnet.gevent + + group = execnet.gevent.Group() + gateway = group.makegateway("popen") + channel = gateway.remote_exec("channel.send(6 * 7)") + print(channel.receive()) # parks this greenlet, not the hub + +Requires ``execnet[gevent]``. Importing it does not monkey-patch anything +-- do that yourself, as early as usual. ``Group``, ``default_group`` and +``makegateway`` are this module's own; the remaining names (``Channel``, +``Gateway``, ``RSync``, the error types) are the ones from +:mod:`execnet.sync`. + +This is about the *caller*. Whether the worker itself runs greenlets is +the independent ``profile=gevent`` spec key -- see +:ref:`worker profiles `. + + +Errors +============================================================================== + +The same types are raised by every namespace, and are re-exported from each +of them. + +.. autoexception:: execnet.RemoteError +.. autoexception:: execnet.TimeoutError +.. autoexception:: execnet.HostNotFound +.. autoexception:: execnet.DataFormatError +.. autoexception:: execnet.DumpError +.. autoexception:: execnet.LoadError diff --git a/doc/basics.rst b/doc/basics.rst index 105f1c0f..232804a2 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -12,6 +12,56 @@ help to manage creation and termination of sub-interpreters. .. currentmodule:: execnet + +Namespaces +=============================================== + +execnet has one namespace per concurrency library you drive it *from*. All +of them speak the same protocol to the same kind of worker; what differs is +what a waiting call does to the caller. + +:mod:`execnet.sync` + The blocking API for plain threads. The top-level ``execnet.*`` names + are aliases into it, so ``import execnet`` is this surface. + +:mod:`execnet.trio` + The trio-native API: ``AsyncGroup``, ``AsyncGateway``, ``AsyncChannel``, + awaited inside your own ``trio.run``. + +:mod:`execnet.aio` + The same three classes for asyncio, awaited inside your own event loop. + +:mod:`execnet.gevent` + The blocking API again, except that every wait parks the calling + *greenlet* rather than its OS thread. Needs ``execnet[gevent]``. + +:mod:`execnet.trio` is the only surface that runs gateways *directly*, as +tasks in your own nursery. The other three run protocol IO on a Trio host +thread (see `The host thread`_) and block the caller until it answers -- +which inside a running event loop would stall every task on it, so those +calls raise ``RuntimeError`` naming the namespace to use instead. + +:: + + import trio + import execnet.trio + + async def main(): + async with execnet.trio.AsyncGroup() as group: + gateway = await group.makegateway("popen") + channel = await gateway.remote_exec("channel.send(6 * 7)") + print(await channel.receive()) + + trio.run(main) + +The rest of this page shows the blocking API. Apart from ``async``/``await`` +and the ``Async`` prefix, the async namespaces mirror it; see +:doc:`the namespace reference ` for what each one offers. + +This is about the *caller*. Where remote code runs inside the worker is an +independent choice -- see `Worker profiles`_. + + Gateways: bootstrapping Python interpreters =================================================== @@ -24,15 +74,24 @@ passing it a gateway specification or URL. Here is an example which instantiates a simple Python subprocess:: + >>> import execnet >>> gateway = execnet.makegateway() Gateways allow to `remote execute code`_ and `exchange data`_ bidirectionally. +Workers are never sent their own source code: a worker imports the +``execnet`` (and ``trio``) that is installed in the environment it runs in. +Where that environment does not have execnet yet, it is provisioned with +uv_ -- so a bare ``python=`` interpreter or an ssh remote needs ``uv`` on +its ``PATH``, not a pre-installed execnet. + +.. _uv: https://docs.astral.sh/uv/ + Examples for valid gateway specifications ------------------------------------------- -* ``ssh=wyvern//python=python3.3//chdir=mycache`` specifies a Python3.3 +* ``ssh=wyvern//python=python3.13//chdir=mycache`` specifies a Python 3.13 interpreter on the host ``wyvern``. The remote process will have ``mycache`` as its current working directory. @@ -57,14 +116,76 @@ Examples for valid gateway specifications same interpreter as the one it is initiated from and additionally remotely sets an environment variable ``NAME`` to ``value``. -* ``socket=192.168.1.4:8888`` specifies a Python Socket server - process that listens on ``192.168.1.4:8888``. Such a server can be - started with the ``execnet-socketserver`` console command, e.g. run - anywhere with ``uvx --from execnet execnet-socketserver :8888``. +* ``socket=192.168.1.4:8888`` specifies a Python server process that + listens on ``192.168.1.4:8888``. Such a server is started with the + ``execnet server`` command, e.g. run anywhere with + ``uvx --from execnet execnet server :8888``; see + :ref:`instantiate gateways through sockets `. + +.. _spec-keys: + +Specification keys +------------------------------------------- + +*Which interpreter to reach, and how* + +``popen`` + A subprocess of this process. The default when no other target is given. + +``python=PATH`` + The interpreter to run, as a path or a ``PATH``-discoverable name. + Combined with ``popen``, ``ssh=`` or ``vagrant_ssh=``. + +``ssh=ARGS`` + Run the worker on a host reachable by the ``ssh`` client binary. The + value is passed to it as arguments, so ``ssh=-p 5000 myhost`` works. + +``ssh_config=PATH`` + An ssh configuration file to pass as ``-F PATH``. + +``vagrant_ssh=NAME`` + Like ``ssh=``, through ``vagrant ssh`` for the named box. + +``socket=HOST:PORT`` + Connect to a running ``execnet server`` and have it spawn the worker. + +``via=GATEWAY-ID`` + Create this gateway's connection *on* another gateway of the same group, + which then relays for it (see :doc:`proxy examples `). + +``installvia=GATEWAY-ID`` + Start a socket server through the named gateway and connect to it. + +*What the worker looks like* + +``profile=thread|trio|gevent`` + Where exec'd code runs inside the worker; see `Worker profiles`_. + ``execmodel=`` is an accepted older spelling of the same key. + +``transport=socket|stdio`` + Which stream carries the protocol; see `Transports`_. + +``stdin=``, ``stdout=``, ``stderr=`` + What the worker does with its standard fds; see `Worker output`_. + +``id=NAME`` + The gateway's id within its group, instead of an allocated ``gwN``. + +``chdir=PATH`` + Working directory of the worker. Defaults to the instantiator's + directory for ``popen``, and to the login home directory for ``ssh=``. + +``nice=N`` + Run the worker at that ``nice`` level (POSIX). -.. versionadded:: 1.5 +``dont_write_bytecode`` + Pass ``-B`` to the worker interpreter. -* ``vagarant_ssh`` opens a python interpreter via the vagarant ssh command +``env:NAME=value`` + Set an environment variable in the worker. May be repeated. + +Keys are separated by ``//``, may not repeat, and a key without ``=value`` +means ``True``. .. _`remote execute code`: @@ -132,38 +253,145 @@ processes then you often want to call ``group.terminate()`` yourself and specify a larger or not timeout. -threading models: thread, main_thread_only +.. _worker-profiles: + +Worker profiles ==================================================================== -.. versionadded:: 1.2 (status: experimental!) +.. versionchanged:: 2.2 + The ``execmodel=`` key is now spelled ``profile=`` and only ever + described the *worker*. The local execution model it was named after + no longer exists: see `Namespaces`_ for the local choice. -execnet supports "thread" and "main_thread_only" as thread models on -each of the two sides. You need to decide which model to use before -you create any gateways:: +A worker's profile says where the code you ``remote_exec`` runs relative to +the worker's own protocol loop. Pass it per gateway:: - # content of threadmodel.py - import execnet - # locally use "thread", remotely use "main_thread_only" model - execnet.set_execmodel("thread", "main_thread_only") - gw = execnet.makegateway() - print (gw) - print (gw.remote_status()) - print (gw.remote_exec("channel.send(1)").receive()) + >>> import execnet + >>> gw = execnet.makegateway("popen//profile=trio") + +``thread`` (the default) + Exec'd code runs on the worker's main thread while that is free, and on + pool threads for anything concurrent with it. The *first* + ``remote_exec`` always gets the real main thread, which is what GUI + loops and signal handlers need. + +``trio`` + Exec'd code runs as a task on the worker's own Trio loop, in the single + thread of that process, and is handed an ``AsyncChannel``. Sources must + be async -- a plain function, or a source string with no top-level + ``await``, is rejected rather than allowed to starve the loop. + +``gevent`` + Exec'd code runs as a greenlet on a gevent hub owning the worker's main + thread, so concurrent execs cooperate on that one thread. Provisioning + adds the ``gevent`` requirement to the worker environment. + +``main_thread_only`` is deprecated and now behaves like ``thread``, whose +main-thread claim is what it existed for. Its other behaviour is gone: a +second concurrent ``remote_exec`` used to fail the channel with +``concurrent remote_exec would cause deadlock``, and now runs on a pool +thread. + +Set the default for a whole group with ``Group(profile=...)`` or +``group.set_profile(...)``; ``execnet.set_profile(...)`` sets it on the +default group. + + +Transports +==================================================================== + +.. versionadded:: 2.2 + +The Message protocol does not have to be the worker's stdin/stdout. The +``transport=`` key selects: + +``socket`` (the default) + The worker gets a socket of its own for the protocol. For ``popen`` it + is an inherited socketpair (a socket duplicated with ``socket.share()`` + on Windows); for ``ssh=``/``vagrant_ssh=`` it is a unix socket forwarded + with ``ssh -R`` that the worker dials back on. + +``stdio`` + The classic transport: the protocol *is* the worker's stdin/stdout. + +Requesting ``transport=socket`` where it cannot work is an error at +``makegateway`` time naming the platform, rather than a gateway that waits +for a worker which was never able to reach back. That case is ssh on +Windows, where CPython does not expose ``AF_UNIX`` and Win32-OpenSSH does +not implement ``StreamLocal`` forwarding. + + +.. _worker-output: + +Worker output +==================================================================== + +.. versionchanged:: 2.2 + A worker's stdio belongs to the code it runs. It used to be redirected + to the null device, so a remote ``print()`` went nowhere at all. + +With the socket transport the worker leaves fd 0/1/2 alone: remote output +reaches your terminal (or your ``capfd``), and remote code can read *your* +stdin. With ``transport=stdio`` the protocol needs those fds, so the worker +closes stdin and folds its stdout onto stderr instead of discarding both. + +Override any of it per gateway: + +=========== ========================================== ================= +key values default +=========== ========================================== ================= +``stdin=`` ``inherit``, ``close``, ``devnull`` transport-defined +``stdout=`` ``inherit``, ``devnull``, ``stderr`` transport-defined +``stderr=`` ``inherit``, ``devnull`` transport-defined +=========== ========================================== ================= -You can execute this little test file:: +For example ``popen//stdin=devnull`` gives remote code an empty stdin while +keeping its output visible. - $ python threadmodel.py - - - 1 -How to execute in the main thread ------------------------------------------------- +The host thread +==================================================================== + +.. versionadded:: 2.2 + +The blocking, asyncio and gevent surfaces have no event loop of their own to +put gateways on, so protocol IO runs on a :class:`Host`: one OS thread +running a Trio loop, shared by every group in the process and stopped at +interpreter exit. You need to know it exists in two cases: it is why a +blocking call from inside a running event loop is an error, and it is what +you pass when you want an isolated loop with deterministic teardown:: -When the remote side of a gateway uses the "thread" model, execution -will preferably run in the main thread. This allows GUI loops -or other code to behave correctly. If you, however, start multiple -executions concurrently, they will run in non-main threads. + with execnet.Host() as host: + group = execnet.Group(host=host) + ... + group.terminate() + # the thread is joined here, rather than at interpreter exit + +Gateways served by a host must be terminated before it closes; closing does +not terminate them for you. :mod:`execnet.trio` uses no host at all. + + +The execnet command line +==================================================================== + +.. versionadded:: 2.2 + +``execnet server [HOST:PORT] [--once]`` + Accept gateway connections on a socket and hand each to a fresh worker + subprocess -- the bootstrapping point for ``socket=`` gateways. See + :ref:`instantiate gateways through sockets `. This + replaces the ``execnet-socketserver`` console script, which still works + and forwards here with a ``DeprecationWarning``. + +``execnet info`` + Print this interpreter's execnet version, trio availability, executable, + platform and supported transports as JSON. A coordinator uses it to + decide whether a ``python=`` interpreter can host a worker directly. + +``execnet worker ...`` + The launch contract between a coordinator and the worker process it + starts. You do not run this by hand; it is documented in + :doc:`implnotes`. remote_status: get low-level execution info @@ -178,7 +406,14 @@ information from the remote side. Calling this method tells you e.g. how many execution tasks are queued, how many are executing and how many -channels are active. +channels are active:: + + >>> import execnet + >>> gw = execnet.makegateway() + >>> gw.remote_status() + + +``execmodel`` repeats ``profile`` under its old name. rsync: synchronise filesystem with remote =============================================================== @@ -206,9 +441,12 @@ Debugging execnet By setting the environment variable ``EXECNET_DEBUG`` you can configure a tracing mechanism: -:EXECNET_DEBUG=1: write per-process trace-files to ``execnet-debug-PID`` +:EXECNET_DEBUG=1: write per-process trace-files to ``execnet-debug-PID`` in the system temp directory :EXECNET_DEBUG=2: perform tracing to stderr (popen-gateway workers will send this to their instantiator) +See :doc:`the debugging example ` for what a trace +looks like. + .. _`dumps/loads`: .. _`dumps/loads API`: diff --git a/doc/conf.py b/doc/conf.py index 3d380529..04627c1d 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -92,10 +92,16 @@ } nitpicky = True +# private types that show up in documented signatures; the modules they +# live in are internal, so there is nothing to link them to nitpick_ignore = [ - ("py:class", "execnet.gateway_base.ChannelFileRead"), - ("py:class", "execnet.gateway_base.ChannelFileWrite"), - ("py:class", "execnet.gateway.Gateway"), + ("py:class", "execnet._channel.ChannelFileRead"), + ("py:class", "execnet._channel.ChannelFileWrite"), + ("py:class", "execnet._gateway.Gateway"), + ("py:class", "execnet._trio_gateway.ByteStream"), + ("py:class", "execnet._trio_gateway.RawChannel"), + ("py:class", "execnet._xspec.XSpec"), + ("py:class", "execnet.aio._HostBridge"), ] # -- Options for HTML output -------------------------------------------------- diff --git a/doc/example/conftest.py b/doc/example/conftest.py index 5044b9eb..0d8b0594 100644 --- a/doc/example/conftest.py +++ b/doc/example/conftest.py @@ -10,5 +10,3 @@ cand = pathlib.Path(__file__).parent if str(cand) not in sys.path: sys.path.insert(0, str(cand)) - -pytest_plugins = ["doctest"] diff --git a/doc/example/test_debug.rst b/doc/example/test_debug.rst index 144a197f..bea6b296 100644 --- a/doc/example/test_debug.rst +++ b/doc/example/test_debug.rst @@ -5,7 +5,7 @@ Debugging execnet / wire messages By setting the environment variable ``EXECNET_DEBUG`` you can configure the execnet tracing mechanism: -:EXECNET_DEBUG=1: write per-process trace-files to ``${TEMPROOT}/execnet-debug-PID`` +:EXECNET_DEBUG=1: write per-process trace-files to ``execnet-debug-PID`` in the system temp directory :EXECNET_DEBUG=2: perform tracing to stderr (popen-gateway workers will send this to their instantiator) Here is a simple example to see what goes on with a simple execution:: @@ -14,30 +14,31 @@ Here is a simple example to see what goes on with a simple execution:: python -c 'import execnet ; execnet.makegateway().remote_exec("42")' -which will show PID-prefixed trace entries:: - - [2326] gw0 starting to receive - [2326] gw0 sent - [2327] creating workergateway on - [2327] gw0-worker starting to receive - [2327] gw0-worker received - [2327] gw0-worker execution starts[1]: '42' - [2327] gw0-worker execution finished - [2327] gw0-worker sent - [2327] gw0-worker 1 sent channel close message - [2326] gw0 received - [2326] gw0 1 channel.__del__ - [2326] === atexit cleanup === - [2326] gw0 gateway.exit() called - [2326] gw0 --> sending GATEWAY_TERMINATE - [2326] gw0 sent - [2326] gw0 joining receiver thread - [2327] gw0-worker received - [2327] gw0-worker putting None to execqueue - [2327] gw0-worker io.close_read() - [2327] gw0-worker leaving - [2327] gw0-worker 1 channel.__del__ - [2327] gw0-worker io.close_write() - [2327] gw0-worker workergateway.serve finished - [2327] gw0-worker gateway.join() called while receiverthread already finished - [2326] gw0 leaving +which will show PID-prefixed trace entries -- the coordinator and its +worker write to the same stream, so their lines interleave:: + + [3451876] creating workergateway on trio id='gw0-worker' + [3451876] integrating as primary thread (trio worker) + [3451876] gw0-worker received + [3451872] gw0 sent + [3451872] gw0 1 channel.__del__ + [3451872] === atexit cleanup === + [3451872] gw0 gateway.exit() called + [3451872] gw0 --> sending GATEWAY_TERMINATE + [3451876] gw0-worker received + [3451872] gw0 sent + [3451872] gw0 --> io.close_write + [3451876] gw0-worker execution starts[1]: '42' + [3451876] gw0-worker execution finished + [3451876] gw0-worker received + [3451876] gw0-worker received GATEWAY_TERMINATE + [3451872] gw0 [trio-bridge] finishing channels + [3451872] gw0 [trio-bridge] terminating execution + [3451876] gw0-worker [trio-bridge] finishing channels + [3451876] gw0-worker shutting down execution pool + [3451876] gw0-worker waiting for receiver to finish + [3451872] gw0 waiting for receiver to finish + +Because the worker leaves its own stderr alone, a remote ``print()`` and a +remote traceback arrive the same way -- see :ref:`worker output +`. diff --git a/doc/example/test_group.rst b/doc/example/test_group.rst index 0cb10b78..ad63adb3 100644 --- a/doc/example/test_group.rst +++ b/doc/example/test_group.rst @@ -93,18 +93,19 @@ Using Groups to manage a certain type of gateway ------------------------------------------------------ Set ``group.defaultspec`` to determine the default gateway -specification used by ``group.makegateway()``: +specification used by ``group.makegateway()`` (this one needs a reachable +ssh account, so it is not run as part of the test suite): >>> import execnet >>> group = execnet.Group() >>> group.defaultspec = "ssh=localhost//chdir=mytmp//nice=20" - >>> gw = group.makegateway() + >>> gw = group.makegateway() # doctest: +SKIP >>> ch = gw.remote_exec(""" ... import os.path ... basename = os.path.basename(os.getcwd()) ... channel.send(basename) - ... """) - >>> ch.receive() + ... """) # doctest: +SKIP + >>> ch.receive() # doctest: +SKIP 'mytmp' This way a Group object becomes kind of a Gateway factory where diff --git a/doc/example/test_info.rst b/doc/example/test_info.rst index 7a5ead4f..4ef3c71b 100644 --- a/doc/example/test_info.rst +++ b/doc/example/test_info.rst @@ -89,19 +89,21 @@ A local subprocess gateway has the same working directory as the instantiatior:: Get information from remote SSH account --------------------------------------- -Use simple execution to obtain information from remote environments:: +Use simple execution to obtain information from remote environments +(this one needs an account you can actually reach, so it is not run as +part of the test suite):: >>> import execnet, os - >>> gw = execnet.makegateway("ssh=codespeak.net") + >>> gw = execnet.makegateway("ssh=wyvern") # doctest: +SKIP >>> channel = gw.remote_exec(""" ... import sys, os ... channel.send((sys.platform, tuple(sys.version_info), os.getpid())) - ... """) - >>> platform, version_info, remote_pid = channel.receive() - >>> platform - 'linux2' - >>> version_info - (2, 6, 6, 'final', 0) + ... """) # doctest: +SKIP + >>> platform, version_info, remote_pid = channel.receive() # doctest: +SKIP + >>> platform # doctest: +SKIP + 'linux' + >>> version_info # doctest: +SKIP + (3, 13, 1, 'final', 0) Use a callback instead of receive() and wait for completion ------------------------------------------------------------- @@ -117,8 +119,10 @@ Set a channel callback to immediately react on incoming data:: >>> l [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, None] -Note that the callback function will execute in the receiver thread -so it should not block on IO or long to execute. +Items reach the callback in order, one at a time per channel, each call on +a thread from a bounded pool -- so a callback that blocks holds up its own +channel and, with enough of them, that pool, but never the protocol loop +that every gateway in the process shares. Sending channels over channels ------------------------------------------------------ @@ -169,22 +173,28 @@ all incoming requests in the global name space and sends back the results. +.. _socket-server: + Instantiate gateways through sockets ----------------------------------------------------- In cases where you do not have SSH-access to a machine you need a bootstrapping-point that listens on a socket. execnet ships one as the -``execnet-socketserver`` console command; run it on the target machine:: +``execnet server`` command; run it on the target machine:: - execnet-socketserver :8888 # bind to all IPs, port 8888 + execnet server :8888 # bind to all IPs, port 8888 If execnet is not installed there, uv_ can fetch and run it in one step without leaving anything behind:: - uvx --from execnet execnet-socketserver :8888 + uvx --from execnet execnet server :8888 .. _uv: https://docs.astral.sh/uv/ +.. versionchanged:: 2.2 + This used to be the ``execnet-socketserver`` console script, which still + works and forwards here with a ``DeprecationWarning``. + The server accepts connections in a loop and serves each one in its own worker subprocess; pass ``--once`` to serve a single connection and exit. Passing port ``0`` binds an ephemeral port -- the bound address is printed @@ -208,30 +218,30 @@ a popen- or SSH-based one. Keeping the socket server running +++++++++++++++++++++++++++++++++ -``execnet-socketserver`` is an ordinary long-running process, so restarts and +``execnet server`` is an ordinary long-running process, so restarts and boot-time startup are the job of your platform's service manager. On Linux, a systemd unit does it:: - # /etc/systemd/system/execnet-socketserver.service + # /etc/systemd/system/execnet-server.service [Unit] Description=execnet socket server [Service] - ExecStart=/usr/local/bin/execnet-socketserver :8888 + ExecStart=/usr/local/bin/execnet server :8888 Restart=always [Install] WantedBy=multi-user.target On Windows, wrap the console command with a service host such as NSSM_ or -WinSW_. Use ``where execnet-socketserver`` to find the installed -``execnet-socketserver.exe`` (it lives in the ``Scripts`` directory of the -environment it was installed into), then:: +WinSW_. Use ``where execnet`` to find the installed ``execnet.exe`` (it +lives in the ``Scripts`` directory of the environment it was installed +into), then:: - nssm install ExecNetSocketServer C:\path\to\Scripts\execnet-socketserver.exe :8888 - nssm set ExecNetSocketServer Start SERVICE_AUTO_START - net start ExecNetSocketServer + nssm install ExecNetServer C:\path\to\Scripts\execnet.exe server :8888 + nssm set ExecNetServer Start SERVICE_AUTO_START + net start ExecNetServer NSSM restarts the process if it exits. Note that ``sc.exe create`` on its own is not enough: it expects a binary that implements the Windows service control diff --git a/doc/example/test_multi.rst b/doc/example/test_multi.rst index 306b960d..1d2da285 100644 --- a/doc/example/test_multi.rst +++ b/doc/example/test_multi.rst @@ -49,8 +49,9 @@ data immediately and without blocking execution:: >>> ch.waitclose() >>> assert l == [42] -Note that the callback function will be executed in the -receiver thread and should not block or run for too long. +The callback runs on a pool thread, one item at a time per channel and in +order, so it may block without stalling the protocol loop -- but a channel +whose callback blocks receives nothing further until it returns. Robustly receive results and termination notification ----------------------------------------------------- diff --git a/doc/examples.rst b/doc/examples.rst index 3f09197a..ff6233bf 100644 --- a/doc/examples.rst +++ b/doc/examples.rst @@ -5,7 +5,9 @@ examples .. _`execnet-dev`: http://mail.python.org/mailman/listinfo/execnet-dev .. _`execnet-commit`: http://mail.python.org/mailman/listinfo/execnet-commit -Note: all examples with `>>>` prompts are automatically tested. +Note: the examples with ``>>>`` prompts are run as doctests by ``tox -e +docs``, except for the few marked ``# doctest: +SKIP``, which need a remote +account to talk to. .. toctree:: :maxdepth: 2 diff --git a/doc/implnotes.rst b/doc/implnotes.rst index eed92232..5db4d92c 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -1,74 +1,174 @@ -gateway_base.py +============================================================================== +Implementation notes +============================================================================== + +How a gateway is actually built. Everything here is internal: no name in +this document is part of the public API, and the on-wire protocol is +deliberately unversioned and unstandardised. + +The Message protocol +---------------------- + +Both sides speak a stream of Messages: a 9-byte header (type, channel id, +payload length) followed by the payload. ``execnet._message.FrameDecoder`` +turns arbitrary byte chunks into Messages and is sans-IO -- it never reads, +writes or awaits -- so a receiver only has to stream bytes into it. Payloads +carrying channel items are encoded by ``execnet._serialize``, which handles +builtin data plus channel references and nothing else (see +:ref:`serialization`). + +No source shipping ---------------------- -The code of this module is sent to the "other side" -as a means of bootstrapping a Gateway object -capable of receiving and executing code, -and routing data through channels. - -Gateways operate on InputOutput objects offering - a write and a read(n) method. - -Once bootstrapped a higher level protocol -based on Messages is used. Messages are serialized -to and from InputOutput objects. The details of this protocol -are locally defined in this module. There is no need -for standardizing or versioning the protocol. - -Trio host-thread IO -------------------- - -``popen`` and ``ssh`` gateways run their Message protocol IO inside a -dedicated OS thread hosting a Trio event loop -(``execnet._trio_host.TrioHost``). No source is sent over the wire; the -worker is launched as ``python -m execnet._trio_worker`` and imports the -installed ``execnet`` + ``trio`` (a rough major/minor version check guards -against an incompatible install). How the worker environment is obtained -depends on the target: - -* Same-interpreter ``popen`` -> ``sys.executable -m execnet._trio_worker``. -* A ``python=`` interpreter that already has execnet -> that interpreter - directly (so ``sys.executable`` is preserved). -* A bare ``python=`` interpreter or an ``ssh`` remote -> provisioned via - ``uv`` (``execnet._provision``): ``uv run --with `` where ```` - is ``execnet==`` for a released coordinator or a locally-built, - version-cached wheel for a dev coordinator. For an ``ssh`` remote on a - dev coordinator the wheel is not on the remote filesystem, so the remote - command is a POSIX-sh prelude that reads the wheel bytes from stdin - (``head -c N`` into a temp dir) and ``exec``s ``uv`` against it; the - coordinator streams those bytes before the Message protocol. - -The worker configuration (id, execmodel, coordinator version) is passed as a -single JSON CLI argument (``_provision.worker_cli_arg``), so every launcher -shares one contract instead of scattered positional args. - -Coordinator and worker roles: - -* Coordinator: ``trio.lowlevel.open_process`` (directly, or wrapped in - ``ssh``) plus async framed reader/writer tasks per gateway (one host - thread per ``Group``). It waits for the worker's ``b"1"`` handshake - before starting the Message protocol. -* Worker: ``serve_popen_trio`` adopts stdio pipe fds into Trio streams and - writes the handshake byte; ``remote_exec`` is scheduled from the Trio - nursery (``trio.to_thread`` for ``thread``, main-thread handoff for - ``main_thread_only``). - -Sync ``Channel`` / ``Gateway`` APIs are unchanged. Sends from non-host -threads wait until the frame is written (so abrupt ``os._exit`` cannot drop -queued data). Sends from the Trio host thread (receiver callbacks) only -enqueue, to avoid deadlocking the writer task. - -``socket`` and ``via`` gateways run on the Trio host too. ``socket`` -connects a Trio TCP stream to an ``execnet-socketserver`` (itself a Trio -listener spawning ``python -m execnet._trio_worker --socket-fd`` subprocess -workers). Infrastructure that used to be driven by ``remote_exec``-ing -source is now expressed as native protocol messages handled on the target's -Trio host: ``GATEWAY_START_SOCKET`` (installvia -> bind a one-shot listener, -reply with its address) and ``GATEWAY_START_SUB`` (``via`` -> spawn a -sub-worker — popen, foreign python, ssh, or vagrant — and relay its protocol -over the request channel). - -The Trio host is the only IO path; the legacy thread receiver and the -source-shipping bootstrap have been removed. The receiver task terminates -when the remote side sends a gateway termination message or the -IO-connection drops. +A worker is not sent its own source. It is launched as a command that runs +the ``execnet`` (and ``trio``) installed in its own environment, and refuses +a coordinator whose major/minor version differs from its own. This is what +makes provisioning a separate concern from connecting, and it is an +invariant: nothing may reintroduce shipping the core over the wire. + +The launch contract: ``execnet worker`` +---------------------------------------- + +Every launcher emits the same command line, so there is exactly one way a +worker starts:: + + execnet worker --protocol-stdio | --protocol-fd FD[,FD] + | --protocol-connect ADDR | --protocol-listen ADDR + | --protocol-share + --config JSON | --config-fd FD | --config-file PATH + --stdin/--stdout/--stderr DISPOSITION + +``ADDR`` is ``unix:/path`` or ``host:port``. Provisioning emits ``python -m +execnet worker ...`` for a direct interpreter launch (where the console +script's location is not knowable) and ``execnet worker ...`` under ``uv +run``; both are the same CLI. + +The **config** (gateway id, worker profile, coordinator version, ``env:`` +values, and for ``--protocol-share`` the socket blob) is JSON. It travels in +argv only where argv is private to the machine: a remote argv is readable +through ``ps`` by every user on that host, so ssh passes it on stdin with +``--config-fd 0``. Do not regress that. + +Naming the transport explicitly is what frees the worker's stdio. Once the +protocol has a stream of its own, fd 0/1/2 belong to the code the worker +runs, and ``--stdin/--stdout/--stderr`` say what to do with them (the +defaults come from the transport: leave them alone for a socket, close stdin +and fold stdout onto stderr for stdio). + +Two more subcommands round it out: ``execnet server [HOST:PORT] [--once]`` +accepts coordinator connections and hands each to a fresh worker (no code +runs in the server process itself), and ``execnet info`` prints version, +trio availability, executable, platform and supported transports as JSON -- +which is how a coordinator decides whether a ``python=`` interpreter can +host a worker directly, *before* connecting to it. + +Transports +---------------------- + +``transport=socket`` is the default for every worker execnet spawns; the +protocol only rides on stdin/stdout when asked to, or when nothing else can +work. + +How the worker gets its protocol stream, by gateway: + +``popen``, POSIX + an inherited socketpair (``pass_fds``, ``--protocol-fd``) + +``popen``, Windows + a socket duplicated into the child pid with ``socket.share()``, the blob + travelling in the config (``--protocol-share``) + +``socket=`` / ``installvia=`` + the server accepts the connection, then hands that socket to the worker + it spawns, by whichever of the two mechanisms the platform has + +``ssh=`` / ``vagrant_ssh=`` + an ``ssh -R`` forwarded unix socket the worker dials back on (POSIX + only) + +Windows has no ``pass_fds``, hence ``socket.share()`` (``WSADuplicateSocket``), +which duplicates into a *named pid* -- and the pid does not exist until the +child does, which is why the flag is in argv while the blob follows in the +config. The blob is bound to that one pid, so it is inert to anything else; +that beats handle inheritance, which would need ``close_fds=False`` and leak +every inheritable handle to the child and its grandchildren. + +Hand a socket over **as a socket, never as an fd**. Rebuilding one with +``socket.socket(fileno=fd)`` makes the constructor re-derive family, type and +proto by querying the handle, and PyPy on Windows cannot do that to a handle +produced by ``WSADuplicateSocket``. + +ssh on Windows stays on stdio and cannot do otherwise: CPython has never +exposed ``AF_UNIX`` there and Win32-OpenSSH does not implement +``StreamLocal`` forwarding. Asking for ``transport=socket`` anyway is an +error at ``makegateway`` time rather than a gateway that waits forever. + +Whether a socket can be handed over at all is settled by *doing* it once -- +sharing to our own pid and rebuilding the result -- not by looking for +``socket.share``: an implementation with the name but not a working call +would pass the check and fail later, at the point where the only thing left +to tell the coordinator is a closed socket. + +Provisioning the worker environment +------------------------------------ + +``execnet._provision`` decides what command to run, by target: + +* Same-interpreter ``popen`` -> ``sys.executable -m execnet worker``. +* A ``python=`` interpreter whose ``execnet info`` answers -> that + interpreter directly, so ``sys.executable`` is preserved. +* A bare ``python=`` interpreter or an ``ssh`` remote -> uv_: + ``uv run --with execnet worker``, where ```` is + ``execnet==`` for a released coordinator and a locally built, + version-cached wheel for a development one. A dev coordinator's wheel is + not on the remote filesystem, so it is copied over its own ssh connection + and cached there by name before the worker is launched -- the protocol + stream never carries a payload. +* ``EXECNET_PROVISION_WHEEL`` names a prebuilt wheel to use instead, which + is how a built artifact gets tested by the suite that built it. + +.. _uv: https://docs.astral.sh/uv/ + +The Trio host thread +---------------------- + +Protocol IO is a Trio program. :mod:`execnet.trio` runs it in the caller's +own nursery; every other surface has no loop to put it on, so it runs on a +``Host``: one OS thread running ``trio.run``, shared per process +(``execnet._host.Host`` -> ``execnet._trio_host.TrioHost``). ``execnet._host`` +deliberately does not ``import trio``, so ``import execnet`` does not load the +event loop machinery. + +Blocking calls cross into it through ``execnet._portal`` and park on a +wakener from ``execnet._boundary`` -- which is what makes +:mod:`execnet.gevent` possible: same host, same tasks, a different primitive +to park on. Calling a blocking API from inside a running asyncio or trio +loop would stall that loop, so it raises instead. + +Sends from a non-host thread wait until the frame is written, so an abrupt +``os._exit`` cannot drop queued data. Sends from the host thread itself +(inside a receiver callback) only enqueue, to avoid deadlocking the writer +task. ``setcallback`` runs its callback on a bounded thread pool rather than +on the loop. + +Inside the worker +---------------------- + +The worker adopts its transport, writes a single ``b"1"`` handshake byte, +and serves the Message protocol on its own Trio loop. Where exec'd code +runs is the ``profile=`` axis (:ref:`worker profiles `), +implemented as an exec strategy per profile in ``execnet._trio_worker``: +``HybridExec`` (main thread while free, pool threads for overflow), +``GreenletExec`` (gevent hub on the main thread) and ``TaskExec`` (tasks on +the worker's own loop, for ``profile=trio``, which is the only profile whose +sources must be async). + +Infrastructure that used to be expressed by ``remote_exec``-ing source is +now native protocol messages handled on the target's host: +``GATEWAY_START_SOCKET`` (``installvia=`` -- bind a one-shot listener and +reply with its address) and ``GATEWAY_START_SUB`` (``via=`` -- spawn a +sub-worker and relay its protocol over the request channel). A sub-gateway +that fails to start must not take its coordinator down with it, and a +failure that cannot be reported must still close the connection, so the +requesting side sees EOF instead of waiting for a handshake byte nobody will +send. diff --git a/doc/index.rst b/doc/index.rst index 47a918cc..ec901ca5 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -28,7 +28,9 @@ a minimal and fast API targeting the following uses: Features ------------------ -* Automatic bootstrapping: no manual remote installation. +* Automatic bootstrapping: a worker environment that lacks execnet is + provisioned with uv_, so there is no manual remote installation -- and no + source of our own is ever shipped over the wire. * Safe and simple serialization of Python builtin types for sending/receiving structured data messages; @@ -41,13 +43,17 @@ Features * Easy creation, handling and termination of multiple processes -* Well tested interactions between CPython 2.5-2.7, CPython-3.3, Jython 2.5.1 - and PyPy interpreters. +* One :doc:`namespace ` per concurrency library you drive it from: + threads, trio, asyncio or gevent. + +* Tested against CPython 3.10+ and PyPy 3. * Fully interoperable between Windows and Unix-ish systems. * Many tested :doc:`examples` +.. _uv: https://docs.astral.sh/uv/ + Known uses ------------------- diff --git a/doc/install.rst b/doc/install.rst index adbb9341..778149b2 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -28,6 +28,7 @@ Next checkout the basic api and examples: examples basics + api changelog .. _pip: http://pypi.python.org/pypi/pip diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index 5ea1d5cd..a744bb39 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -398,7 +398,7 @@ async def aclose(self) -> None: class AsyncChannel: - """Serialized object API over a :class:`RawChannel`. + """Serialized object API over a raw byte channel. Every payload is one dumps/loads-serialized item; close/EOF semantics and error propagation come from the raw layer. Channels are @@ -1359,7 +1359,7 @@ async def _terminate_one( @asynccontextmanager async def open_gateway(spec: str | Any = "popen") -> AsyncIterator[AsyncGateway]: - """Spawn one popen worker and serve an AsyncGateway over its stdio. + """Spawn one worker for ``spec`` and serve an AsyncGateway to it. Runs inside the caller's own trio run -- no host thread involved. Convenience for a single-gateway :class:`AsyncGroup`. diff --git a/tox.ini b/tox.ini index c938f81d..2cc15373 100644 --- a/tox.ini +++ b/tox.ini @@ -16,12 +16,15 @@ commands= [testenv:docs] skipsdist = True usedevelop = True -changedir = doc +# the doc examples are doctests and are run here, so this env needs what +# the suite needs as well as what sphinx needs +extras = testing deps = sphinx PyYAML commands = - sphinx-build -W -b html . _build + sphinx-build -W -b html doc doc/_build + pytest --doctest-glob=*.rst doc [testenv:linting] skip_install = True From 17ee4e41c729148f4b26a68fc64cfbe900a0b83b Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 11:11:33 +0200 Subject: [PATCH 84/91] docs: review the public surface for engine neutrality Takes inventory of what the four namespaces actually publish now that the docs freeze it, records the delta since 2.1, and asks what would be expensive to undo if the host thread ever ran something other than Trio. The one time-sensitive finding: `execnet info` answers `"trio": ` and `_provision.target_has_execnet` reads exactly that key to decide whether an interpreter can host a worker directly. Once 2.2 ships, every coordinator asks every future worker a question that names our engine. Co-Authored-By: Claude Opus 5 --- handoff-public-api-review.md | 199 +++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 handoff-public-api-review.md diff --git a/handoff-public-api-review.md b/handoff-public-api-review.md new file mode 100644 index 00000000..87e8261c --- /dev/null +++ b/handoff-public-api-review.md @@ -0,0 +1,199 @@ +# Public API review, and what pins us to Trio + +Branch `feat/trio-host-thread-io`, after `98eb08d` (the Phase D docs). +Companion to the "Surface review" section in +`handoff-phase-c-worker-axes.md`, which decided the *shape* of the +namespaces; this one takes inventory of what that shape actually publishes +now that the docs freeze it, and asks a second question: **if the host +thread ever ran plain asyncio instead of Trio, what in today's public +surface would we regret having published?** + +Nothing here has been applied. It is a list of decisions, with a +recommendation each. + +## 1. The surface as it stands + +`execnet` == `execnet.sync` plus `__version__` and `can_send` +(`test_namespaces.py` asserts exactly that): + +``` +Channel Gateway Group Host MultiChannel RSync XSpec +DataFormatError DumpError LoadError RemoteError TimeoutError HostNotFound +default_group makegateway set_profile set_execmodel +can_send __version__ +``` + +| namespace | exports | +|---|---| +| `execnet.sync` | the list above minus `can_send`/`__version__` | +| `execnet.trio` | `AsyncGroup`, `AsyncGateway`, `AsyncChannel`, `open_gateway`, `XSpec`, the six error types | +| `execnet.aio` | the same, **plus `Host` and `default_host`** | +| `execnet.gevent` | the sync surface, with its own `Group`/`default_group`/`makegateway` | + +Class surfaces (public attributes, inherited included): + +| class | members | +|---|---| +| `Gateway` | `id`, `remoteaddress`, `remote_exec`, `remote_status`, `newchannel`, `hasreceiver`, `exit`, `join`, `remote_init_threads`\* | +| `Channel` | `send`, `receive`, `setcallback`, `makefile`, `close`, `waitclose`, `isclosed`, `next`, `RemoteError`, `TimeoutError` | +| `Group` | `makegateway`, `remote_exec`, `terminate`, `allocate_id`, `defaultspec`, `host`, `profile`, `set_profile`, `execmodel`\*, `remote_execmodel`\*, `set_execmodel`\* | +| `Host` | `name`, `running`, `close` (context manager) | +| `MultiChannel` | `send_each`, `receive_each`, `make_receive_queue`, `waitclose` | +| `trio.AsyncGateway` | `remote_exec`, `terminate`, `remoteaddress`, `aclose`, `wait_closed`, `closed`, **`open_channel`, `open_raw_channel`, `enqueue_frame`** | +| `aio.AsyncGateway` | `id`, `remoteaddress`, `remote_exec`, `terminate` | +| `*.AsyncChannel` | `send`, `receive`, `send_eof`, `aclose`, `wait_closed`, `isclosed`, `id` | +| `aio.AsyncGroup` | `start`, `aclose`, `makegateway`, `host` | +| `trio.AsyncGroup` | `makegateway`, `terminate` | + +\* deprecated. + +Non-Python surface, equally public and harder to change once released: the +`execnet` CLI (`worker` / `server` / `info`), the spec keys, and the +`execnet info` JSON. + +## 2. What changed since 2.1 + +**Added** + +- Four namespaces (`sync` / `trio` / `aio` / `gevent`); the top level is + an alias surface over `sync`. +- `execnet.Host`, `Group(host=)`, `Group.host`, `AsyncGroup(host=)`. +- `execnet.can_send` — the supported replacement for probing with + `dumps`/`DumpError`. +- `set_profile`, `Group(profile=)`, `Group.profile`, `Group.set_profile`. +- Async gateways: `AsyncGroup` / `AsyncGateway` / `AsyncChannel` / + `open_gateway` on both async namespaces. +- Spec keys `profile=`, `transport=`, `stdin=`, `stdout=`, `stderr=`. +- The `execnet` console script and `python -m execnet`. + +**Removed** + +- `execnet.dump` / `load` / `loads`; `dumps` survives *only* as the + undocumented, non-warning xdist probe (`_XDIST_COMPAT`), scheduled for + deletion once xdist moves to `can_send`. +- `execnet.script.*` (`shell`, `quitserver`, `loop_socketserver`). +- The pre-Trio modules as real modules: `gateway`, `gateway_base`, + `multi`, `rsync`, `rsync_remote`, `xspec` now warn and forward, + removal in 3.0. `gateway_bootstrap`, `gateway_io`, `gateway_socket` + are simply gone. +- `execnet.portal`, and the `wait=` spec key — both existed only on this + branch. + +**Deprecated** + +- `set_execmodel`, `Group.execmodel`, `Group.remote_execmodel`, + `Group.set_execmodel`, the `execmodel=` spec key (permanent alias), + the `main_thread_only` profile, `Gateway.remote_init_threads`, + the `execnet-socketserver` script. + +**Behaviour changes that are API in practice**: worker stdio is no longer +swallowed; a blocking call inside a running event loop raises; a killed +worker is `EOFError` on every transport. + +## 3. What pins us to Trio + +Two different futures get conflated under "run on plain asyncio", and they +have different answers: + +- **(A) A non-Trio engine** — the host thread runs `asyncio.run`, or + execnet drops the hard `trio` dependency. This is an internals port + (`_trio_gateway` is the engine); the invariant that protects it is the + one already recorded: neutral `ByteStream`, sans-IO `FrameDecoder`. +- **(B) A native asyncio surface** — `execnet.aio` runs gateways on the + *caller's* loop with no host thread at all, symmetric with + `execnet.trio`. This one is visible in the public API today. + +Ranked by how much it would cost to undo after release: + +### 3.1 `execnet info` reports `"trio": ""` — HIGHEST + +`_provision.target_has_execnet()` decides whether a `python=` interpreter +can host a worker directly by asking whether `info["trio"]` is non-null. +That is a cross-version contract: a 2.2 coordinator will keep asking a 2.5 +worker that question forever, and the answer names our engine. + +**Recommend (before release)**: add a neutral key — `"worker": true` or +`"engines": ["trio"]` — have the coordinator prefer it and fall back to +`"trio"` only for a 2.2-vintage remote. Keep emitting `"trio"` +indefinitely; it costs one line and buys the freedom to answer honestly +from an engine that is not Trio. + +### 3.2 `execnet.trio` publishes the engine objects themselves — HIGH + +`execnet.trio.AsyncGateway` *is* `_trio_gateway.AsyncGateway`, so +`open_raw_channel`, `enqueue_frame` and `closed` are public API by +accident. They are the routing layer `_trio_host` and `_trio_worker` +drive; `test_trio_namespace_hides_raw_plumbing` already keeps `RawChannel` +and `ByteStream` out of `__all__`, but the methods that hand them out are +reachable on a documented class. + +That is also the shape that makes (A) expensive: an asyncio engine would +have to reproduce those exact methods to keep `execnet.trio` importable. + +**Recommend**: underscore-prefix `enqueue_frame` and `open_raw_channel` +(callers are `_trio_host`, `_trio_worker` and `testing/`, all ours), keep +`open_channel` public as the async `newchannel()`, and add a namespace +test that pins `trio.AsyncGateway`'s public method set. A facade like +`execnet.aio`'s would be cleaner still, but costs an allocation per +channel on the surface whose selling point is that it has none. + +### 3.3 `Host` and `default_host` on `execnet.aio` — MEDIUM + +Under (B) the asyncio surface has no host thread, so `aio.AsyncGroup(host=)` +and `aio.AsyncGroup.host` become meaningless — and `default_host` is +exported *only* from `aio`, which reads as an asyncio-specific concept when +it is the opposite. + +`host=` is honest today and can be deprecated later (a `None` default keeps +every caller working), so it stays. `default_host()` is different: nobody +needs it. Isolation is `Host()`, sharing is the default. + +**Recommend**: drop `default_host` from `execnet.aio.__all__` (keep the +name importable), and document `host=` as "which host thread serves this +group", i.e. as an implementation-shaped knob rather than part of the +asyncio model. + +### 3.4 `execnet.aio.trio` — LOW + +`aio.py` imports trio at module level for `trio.CancelScope`, so +`execnet.aio.trio` is the trio module. Harmless, but under (A) an +asyncio-only install must be able to `import execnet.aio`. + +**Recommend**: nothing now; note it as a tripwire for the engine port. + +### 3.5 `Host`'s own documentation — LOW + +The class is already engine-neutral in name and members (`name`, +`running`, `close`). Only its docstring says Trio. That is accurate +today and the right thing to write; it just means (A) is a docs change +here, not an API change. No action. + +## 4. Not about Trio, but found while looking + +1. **Deprecated names are in `__all__`.** `set_execmodel` is advertised + as supported API by `execnet.__all__` and `execnet.sync.__all__`. + Recommend removing from both (it stays importable and warning), and + updating `test_top_level_all_matches_sync_surface`, which compares the + two lists. +2. **`ExecModel` is still reachable** as `channel.gateway.execmodel`, and + hands out `RLock`/`Event`/`queue`/`subprocess`/`socket`. It exists + solely because xdist's remote builds its test queue on it. It is the + one place where a third party holds thread-shaped primitives of ours. + No action until xdist ports; delete with `_XDIST_COMPAT` in 3.0. +3. **`Gateway.remote_init_threads`** is a no-op that warns. Delete in + 3.0 with the shims. +4. **Async surfaces still lack** `remote_status()`, `MultiChannel`, group + iteration and `RSync` (carried over from the Phase C review — still a + deliberate gap, worth a line in the docs rather than silence). +5. **`Gateway.join`/`exit`** are public lifecycle methods that + `Group.terminate` supersedes. Harmless; leave. + +## 5. Suggested order + +1. `execnet info` neutral capability key — **before 2.2 ships**, because + it is the only item here that cannot be changed afterwards. +2. `set_execmodel` out of `__all__`; `default_host` out of `aio.__all__`. + One commit, two test updates. +3. Underscore the two engine methods on `AsyncGateway`, plus the pinning + test. +4. Everything else: 3.0, or never. From f63e2eca3ba4d80fb3d294e142df0af082269f06 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 11:13:07 +0200 Subject: [PATCH 85/91] docs: stop telling readers not to use execnet The front page opened with "Do not use in new projects" and a project status section promising no further improvements, in a release that adds four namespaces, socket transports, a CLI and a worker profile axis. Being pytest-xdist's backend is still the compatibility bar every change is held to -- that is worth saying, and is not the same as a warning. Co-Authored-By: Claude Opus 5 --- doc/index.rst | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/doc/index.rst b/doc/index.rst index ec901ca5..5bffa87c 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -2,11 +2,6 @@ :align: right -.. warning:: - - execnet currently is in maintenance-only mode, mostly because it is still the backend - of the pytest-xdist plugin. Do not use in new projects. - Python_ is a mature dynamic language whose interpreters can interact with all major computing platforms today. @@ -78,10 +73,10 @@ Known uses Project status -------------------------- -The project is currently in **maintenance-only mode**, with PRs fixing bugs being gracefully accepted. - -Currently there are no plans to improve the project further, being maintained mostly because it is -used as backend of the popular `pytest-xdist `__ plugin. +``execnet`` is the backend of the popular `pytest-xdist +`__ plugin, which is both what +keeps it maintained and the compatibility bar every change is held to. +Bug reports and PRs are welcome; see :doc:`support`. ``execnet`` was conceived originally by `Holger Krekel`_ and is licensed under the MIT license since version 1.2. From 30c77bd932dddefff337a3c524e5cf9a8dc1ed88 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 11:15:28 +0200 Subject: [PATCH 86/91] docs: mark the docs handoff item done and point at the surface review Co-Authored-By: Claude Opus 5 --- handoff-windows-and-transports.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/handoff-windows-and-transports.md b/handoff-windows-and-transports.md index 8462b785..751059cd 100644 --- a/handoff-windows-and-transports.md +++ b/handoff-windows-and-transports.md @@ -153,13 +153,16 @@ relaying one is named; the requesting side is "here"/"us". ## Open work -1. **`doc/basics.rst` still documents `set_execmodel` / - `main_thread_only`** as the API, with a whole threading-models - section. Stale since the `execmodel=` → `profile=` rename; needs a - rewrite, not a substitution. Two doctests in `doc/example/` expected - `thread model` in a gateway repr and were already broken — fixed, but - note **docs are not built in CI**, only `tox -e py` runs. Consider - adding `tox -e docs` to the workflow. +1. ~~**`doc/basics.rst` still documents `set_execmodel` / + `main_thread_only`**~~ — DONE (`98eb08d`). `basics.rst` and + `implnotes.rst` rewritten, new `doc/api.rst` namespace reference, + `tox -e docs` now builds with `-W` *and* runs the doc examples as + doctests, from a new CI job. Those examples had not been collectable + at all since pytest 7 (a `pytest_plugins` line in a non-top-level + conftest), which is why so much of them had rotted. The public-surface + review that came out of the same pass is + `handoff-public-api-review.md`; **its recommendations are not + applied** — the `execnet info` one wants deciding before 2.2 ships. 2. **`installvia` still needs a socket handoff at all.** It would not, if the server spawned the worker as the *listener* (`--protocol-listen 127.0.0.1:0`) and reported its address back: no From ec3d703fcfd239f0c35d89cd7be3f8fecf6a51a9 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 13:00:51 +0200 Subject: [PATCH 87/91] docs: reconcile the handoff docs, and aim the branch at 3.0 Five handoff documents had grown by accretion: each phase appended its own file, superseded parts of the previous one in place, and left the reader to work out which decisions still stood. Three now: HANDOFF.md is the state and the invariants, ROADMAP-3.0.md is what is left to do, and handoff-history.md keeps the landed record -- including a table of decisions that were made and then unmade, so they stay unmade. The release framing they were missing: this branch ships as 3.0, not 2.2. It changes the launch contract, the default transport and the ownership of a worker's stdio, and parts of the tree already said "pre-3.0" while the changelog said 2.2.0. Two goals are written down for the first time. Unmodified pytest-xdist must keep working on 3.0, which makes the deprecated shims part of the release rather than something to remove in it; they go later in 3.x, once the consumers that need them have released without them. And the next xdist should be able to stop hand-rolling deployment: uv-bootstrap a remote python with the project under test installed, rsync the tests, and get back a local-to-remote path mapping. Driving test runs across Kubernetes pods over the protocol is the same problem with a shorter-lived remote, so the transport and proxy options are recorded next to it. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 14 +- HANDOFF.md | 189 +++++++++++++ ROADMAP-3.0.md | 264 ++++++++++++++++++ doc/basics.rst | 10 +- handoff-boundary-protocol-rethink.md | 163 ----------- handoff-history.md | 178 ++++++++++++ handoff-phase-b-async-core.md | 182 ------------ handoff-phase-c-worker-axes.md | 398 --------------------------- handoff-public-api-review.md | 199 -------------- handoff-windows-and-transports.md | 190 ------------- src/execnet/_execmodel.py | 9 +- src/execnet/_shim.py | 6 +- 12 files changed, 658 insertions(+), 1144 deletions(-) create mode 100644 HANDOFF.md create mode 100644 ROADMAP-3.0.md delete mode 100644 handoff-boundary-protocol-rethink.md create mode 100644 handoff-history.md delete mode 100644 handoff-phase-b-async-core.md delete mode 100644 handoff-phase-c-worker-axes.md delete mode 100644 handoff-public-api-review.md delete mode 100644 handoff-windows-and-transports.md diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 53790bbe..ecd1beaa 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,18 @@ -2.2.0 (UNRELEASED) +3.0.0 (UNRELEASED) ------------------ +This release rebuilds execnet on an async-native Trio core. It is a major +release: a worker is now launched through the ``execnet`` command line +rather than by bootstrapping source over the wire, the protocol rides a +socket instead of the worker's stdin/stdout, and a worker's stdio belongs +to the code it runs. + +**pytest-xdist keeps working unmodified.** The deprecated names that +released xdist reaches for -- ``execnet.gateway_base.ExecModel``, +``execnet.dumps``, ``Group(execmodel=...)``, the ``execmodel=`` spec key -- +all still work here; they are scheduled for removal later in the 3.x +series, once the consumers that need them have released without them. + * New ``execnet`` command line, and it is now the launch contract between a coordinator and the worker process it starts:: diff --git a/HANDOFF.md b/HANDOFF.md new file mode 100644 index 00000000..796a499d --- /dev/null +++ b/HANDOFF.md @@ -0,0 +1,189 @@ +# Handoff: execnet on a Trio core + +Branch `feat/trio-host-thread-io`, draft PR **pytest-dev/execnet#422**. +This is the doc to read first. Two companions: + +- **`ROADMAP-3.0.md`** — what this branch ships as, and what is still open. + The work list lives there, not here. +- **`handoff-history.md`** — the compressed record of what landed, with + commit ranges and the lessons that cost a debugging round each. + +## How to work here + +``` +uv run pytest testing/ # 552 passed, 66 skipped +uv run pytest testing/ -n 12 # must stay green (~7s) +uv run pre-commit run -a # never grep-filter its output +uv run tox -e docs # builds with -W and doctests doc/example/ +``` + +ssh paths have a real local harness in `testing/test_ssh_local.py` (an +asyncssh server; needs a system ssh client). Hypothesis stress coverage +is `testing/test_channel_stress.py` behind `--stress=N`. + +Known flakes, all timing: + +- `test_socket_installvia` EOFs rarely under load. +- `test_gateway_status_busy` (numexecuting race) and + `test_popen_stderr_tracing` (capfd race) keep their `flakytest` marks + and XPASS when idle — see `handoff-history.md`. +- `test_info_reports_what_a_coordinator_needs` failed once under `-n 12` + (2026-07-31), green in isolation and on rerun; **not diagnosed**. + +CI runs pytest-xdist's own suite against this execnet — see +"The xdist contract" in `ROADMAP-3.0.md`. That job is the one that +catches what our suite structurally cannot. + +## Where the repo stands + +There is **one protocol engine**, the async-native `AsyncGateway` +(`_trio_gateway.py`), and everything else is a surface over it. The wire +protocol (`Message` framing) is unchanged from 2.1. + +**No source is shipped over the wire, ever.** Workers are launched as +`execnet worker `; foreign and remote interpreters are +uv-provisioned; a dev coordinator builds and ships a wheel. Version skew +gets a rough major/minor check (`_trio_worker._check_version`). + +### Four namespaces, one per concurrency library you drive execnet from + +| namespace | what it is | +|---|---| +| `execnet` / `execnet.sync` | the blocking API, a facade over the engine; top level aliases into `sync` | +| `execnet.trio` | trio-native `AsyncGroup`/`AsyncGateway`/`AsyncChannel`, awaited in your own `trio.run` | +| `execnet.aio` | the same surface for asyncio, bridged per call onto the host loop | +| `execnet.gevent` | the sync surface with gevent-parking waits | + +`trio`/`aio`/`gevent` load lazily via module `__getattr__`, so +`import execnet` does not import an event loop (pinned by +`testing/test_namespaces.py`). + +A `Host` is one thread running one Trio loop; there is **one shared host +per process**, `Group(host=...)` to override. Blocking calls made from +inside a running event loop raise and name `execnet.aio` / `execnet.trio` +— worker-side channels are exempt, since exec'd code may run its own loop. + +### The CLI is the launch contract + +``` +execnet worker --protocol-stdio | --protocol-fd FD[,FD] + | --protocol-connect ADDR | --protocol-listen ADDR + | --protocol-share + --config JSON | --config-fd FD | --config-file PATH + --stdin/--stdout/--stderr DISPOSITION +execnet server [HOST:PORT] [--once] +execnet info +``` + +`ADDR` is `unix:/path` or `host:port`. Everything that starts a worker +emits these tokens; there is no second launch path. `execnet info` +answers JSON (version, trio, executable, platform, protocols) so +provisioning learns a remote's version *before* connecting. + +`transport=socket|stdio` is a spec key; **`socket` is the default for +every worker execnet spawns**, which is why a worker's stdio is free for +the code it runs. + +| gateway | handoff | +|---|---| +| popen, POSIX | `pass_fds` + `--protocol-fd` (socketpair) | +| popen, Windows | `socket.share(pid)` + `--protocol-share`, blob in the config | +| `socket=` / `installvia=` | the same two, server-side | +| `ssh=` / `vagrant_ssh=` | `ssh -R` unix socket, worker dials back (`--protocol-connect`) | +| `via=` | the sub's stdio, relayed over the coordinator's protocol | + +`--protocol-listen` has no user today; it is what a trampoline or a +port-forwarded worker would use (see the Kubernetes section of the +roadmap). ssh on Windows stays on stdio and cannot do otherwise: CPython +has never exposed `AF_UNIX` there (cpython#77589) and Win32-OpenSSH has no +`StreamLocal` forwarding. `resolve_transport` raises for an impossible +request rather than letting a gateway hang. + +### Worker profiles (`profile=`, spelled `execmodel=` before 3.0) + +| profile | loop thread | exec'd code runs | channel | extra deps | +|---|---|---|---|---| +| `thread` (default) | side thread | hybrid: the first `remote_exec` claims the worker's main thread, further ones overflow to pool threads | sync | — | +| `trio` | **main thread** | async sources as tasks, one thread total; sync sources rejected | `AsyncChannel` | — | +| `gevent` | side thread | a greenlet per `remote_exec` on a main-thread hub | sync | `execnet[gevent]`, auto-added by uv provisioning | +| ~~`main_thread_only`~~ | deprecated alias for `thread` | | | | + +`TrioWorkerExec` is a pure FIFO admission pump delegating to strategy +objects (`WORKER_EXEC_STRATEGIES`); subinterpreters are a future strategy +slot, not built. `AsyncGroup.makegateway` defaults workers to `thread` — +the coordinator's shape does not dictate the worker's. + +### File map (src/execnet/) + +| file | role | +|---|---| +| `_message.py` / `_serialize.py` | wire protocol + sans-IO `FrameDecoder`; serializer (CHANNEL opcode incl. duck-typed `save_AsyncChannel`) | +| `_channel.py` / `_gateway_base.py` / `_errors.py` | sync `Channel`/`ChannelFactory`; `BaseGateway`/`WorkerGateway`; error types | +| `_trio_gateway.py` | **the engine**: `ByteStream` Protocol, `RawChannel`/`AsyncChannel`, `AsyncGateway` (outbound queue of `(frame, on_written)`, `_finalize` hook), `AsyncGroup` (all transports, reapers, bounded terminate), `ThreadedFdStream`, stream/argv helpers | +| `_trio_host.py` | `Host`'s loop thread, `SyncBridgeGateway`, `FacadeAsyncGroup`, `SyncIOHandle`, `RawTunnelStream`, the `GATEWAY_START_*` handlers | +| `_trio_worker.py` | worker entry, `TrioWorkerExec` + exec strategies, `_prepare_protocol_fds`, `_check_version` | +| `_boundary.py` / `_portal.py` | the (private) boundary kit: `Wakener`/`Mailbox`/`OneShot`/`Flag`, `LoopPortal` | +| `_host.py` / `_gateway.py` / `_multi.py` | shared `Host`; sync `Gateway`; sync `Group` + `MultiChannel` | +| `sync.py` / `trio.py` / `aio.py` / `gevent.py` | the four public namespaces | +| `_cli.py` / `_socketserver.py` / `_provision.py` | the CLI, `execnet server`, uv provisioning + argv builders | +| `_execmodel.py` | `WORKER_PROFILES`, `resolve_profile`, and the deprecated `ExecModel` xdist shim | +| `_rsync.py` / `_rsync_remote.py` / `_xspec.py` / `_exec_source.py` | rsync, spec parsing, remote_exec source normalization | +| `_shim.py` + `gateway*.py`, `multi.py`, `rsync*.py`, `xspec.py` | the deprecated pre-Trio module names, warning and forwarding | + +## Invariants — do not regress + +**Protocol and lifecycle** + +- Sends from non-loop threads block until the frame hit the OS write (120s + → `OSError`), so an abrupt `os._exit` cannot drop "sent" data; loop-thread + sends only enqueue. All sends go through one portal-posted FIFO. +- After close: `OSError("cannot send (already closed?)")`; + `trio.RunFinishedError` maps to the same. +- exec admission order == message arrival order (`TrioWorkerExec._pump`). + Trio shuffles its run batch, so never rely on task-spawn order. +- Channel callbacks run in a threadpool thread driven by a per-channel + consumer *task*: per-channel order is strict, a slow callback does not + block the reader, and `waitclose()` still returns only after every + callback including the endmarker has run. +- `Group.terminate(timeout)` never hangs (~2×timeout bound, issues + #43/#221). +- Sync blocking waits (send-ack, receive, waitclose, join) stay on + `threading.Event`/queue so KeyboardInterrupt can interrupt them; + `portal.run` (KI-deferred) is only for management ops. +- A killed worker is `EOFError` on every transport — a dead peer *resets* + a socket where a pipe reaches EOF, and the reader maps that. + +**Launch and provisioning** + +- No source shipping. Workers import an installed execnet + trio. +- The worker config never travels in a remote argv — it carries `env:` + values, and `ps` is world-readable. ssh uses `--config-fd 0`. +- **Hand a socket over as a socket, never as an fd.** Rebuilding one with + `socket.socket(fileno=fd)` re-derives family/type/proto by querying the + handle, which PyPy on Windows fails with `WinError 10014`. +- Filling in a *missing* spec value is idempotent and fine; rewriting one + the caller set is not. xdist reuses one spec object and re-reads it. +- Worker teardown ends in `os._exit(0)` because trio's `to_thread` cache + uses non-daemon threads. +- `import execnet` must not import the trio event loop. +- Keep engine idioms portable — neutral `ByteStream`, sans-IO + `FrameDecoder`. See "What pins us to Trio" in `ROADMAP-3.0.md`. + +**Failure modes that each cost a debugging round** + +- A socket worker that cannot be spawned must not hang the coordinator. + It is spawned by the *server*, so the exception dies there while the + coordinator waits for a handshake byte. A host that cannot hand a + socket over refuses *before replying with an address* — the last moment + a reason can reach the coordinator — and a spawn that fails anyway + closes the connection so the wait ends. +- A failed socket gateway must not kill the gateway it was requested + through. It runs as a task on that coordinator's host; letting it + propagate cost the coordinator too, which is how one unsupported + gateway became 51 errors. +- `_check_event_loop` runs *before* the channel-state check in + `send`/`receive`. Both are caller bugs, but which one you were told + about used to depend on whether the peer had closed yet. +- Anything that warns in a *worker* can livelock a pytest run: a warning + raised inside pytest's warning-recording hook records a warning. The + `execnet.dumps` shim warns once per process for exactly this reason. diff --git a/ROADMAP-3.0.md b/ROADMAP-3.0.md new file mode 100644 index 00000000..dcb5153a --- /dev/null +++ b/ROADMAP-3.0.md @@ -0,0 +1,264 @@ +# Road to execnet 3.0 + +State and invariants live in `HANDOFF.md`; the landed record lives in +`handoff-history.md`. This doc is what is *left*, and what the release is +for. + +## Why 3.0, not 2.2 + +The branch was drafted as 2.2 and the changelog still called it that. It +is a major release: + +- the launch contract changed — a worker is `execnet worker …`, and no + source is bootstrapped over the wire; +- the default transport changed — the protocol is a socket, not the + worker's stdin/stdout; +- a worker's stdio now belongs to the code it runs, so remote `print()` + reaches the coordinator instead of the null device; +- `execnet.script.*`, `dump`/`load`/`loads` and the pre-Trio module names + are gone or deprecated; a blocking call inside a running event loop now + raises; a killed worker is uniformly `EOFError`. + +Everything already in the tree that says "before execnet 3.0" (the +`execmodel=` → `profile=` rename, the worker config key) means this +release. + +### What 3.0 is *not* allowed to break + +**pytest-xdist keeps working, unmodified.** That is the headline +compatibility goal, and it is stronger than "the sync API still exists": +the currently released xdist must drive an execnet 3.0 coordinator with +no changes on its side. The deprecated shims therefore **ship in 3.0**. +They are removed later in the 3.x series, once the consumers that need +them have released a version that does not — not before. + +## The xdist contract + +Released pytest-xdist reaches into more of execnet than the documented +surface. Everything here is load-bearing until xdist stops using it: + +| what xdist does | where | +|---|---| +| `execnet.Group(execmodel="main_thread_only")` as a keyword | `workermanage.py` | +| prefixes specs with `execmodel=main_thread_only//`, **re-reading `spec.execmodel` to decide whether to prefix again** | `workermanage.py` | +| `execnet.gateway_base.ExecModel` → `RLock()`/`Event()` for the remote test queue | `remote.py` | +| `execnet.dumps` + `DumpError` to probe serializability, *inside pytest's warning-recording hook* | `remote.py` | +| subclasses `execnet.RSync`, touching `self._sourcedir` / `self._verbose` and overriding `filter` | `workermanage.py` | +| `gateway.spec.chdir`, and assigns `gateway.node` | `workermanage.py` | +| `execnet.makegateway("execmodel=main_thread_only//popen")` for looponfail | `looponfail.py` | + +Two rules fall out of that list and are recorded as invariants in +`HANDOFF.md`: normalization must not rewrite a caller's spec, and nothing +in a worker may warn unboundedly. + +**The tripwire is CI, not review.** `.github/workflows/test.yml` runs +xdist's own suite against the branch in two variants: a pinned `release` +target (xdist `v3.8.0` + `pytest<9`) that blocks, and a floating +`default-branch` target that may fail. Running it for the first time +found 16 real regressions our own suite could not see, because our suite +uses xdist as a *tool* and never exercises crash-replacement or report +serialization. Bump the pin and the pytest pin together, and only after +re-checking the new pair against *released* execnet. + +One known deselect (`.github/xdist-known-failures.txt`): +`test_remote_inner_argv` asserts `sys.argv == ["-c"]`, which the +no-source-shipping launch deliberately changed. It needs an xdist PR. + +## Before 3.0 ships + +Ordered by how expensive they are to undo afterwards. + +### 1. A neutral capability key in `execnet info` — the only irreversible one + +`_provision.target_has_execnet()` decides whether a `python=` interpreter +can host a worker directly by asking whether `info["trio"]` is non-null. +That is a *cross-version* contract: a 3.0 coordinator will keep asking a +3.5 worker that question forever, and the answer names our engine. + +Add `"worker": true` (or `"engines": ["trio"]`), have the coordinator +prefer it and fall back to `"trio"` only for a 2.x-vintage remote. Keep +emitting `"trio"` indefinitely. One line, and it buys the freedom to +answer honestly from an engine that is not Trio. + +### 2. Deprecated names out of `__all__` + +`set_execmodel` is advertised as supported API by `execnet.__all__` and +`execnet.sync.__all__`. Remove from both (it stays importable and +warning) and update `test_top_level_all_matches_sync_surface`. +`default_host` should leave `execnet.aio.__all__` too — it is exported +only from `aio`, which reads as an asyncio concept when it is the +opposite; isolation is `Host()`, sharing is the default. + +### 3. Underscore the engine methods on `trio.AsyncGateway` + +`execnet.trio.AsyncGateway` *is* `_trio_gateway.AsyncGateway`, so +`open_raw_channel` and `enqueue_frame` are public by accident — they are +the routing layer `_trio_host`/`_trio_worker` drive. Underscore both +(every caller is ours), keep `open_channel` as the async `newchannel()`, +and add a namespace test pinning the public method set. + +### 4. Stale wording + +`_execmodel.ExecModel`'s docstring still describes the `loop=`/`exec=`/ +`wait=` axes, which were dropped before they shipped. `_shim.REMOVED_IN` +says `execnet 3.0`, which is now wrong per the policy above. + +## What pins us to Trio + +Two futures get conflated and have different answers: + +- **A non-Trio engine** — the host thread runs `asyncio.run`, or execnet + drops the hard `trio` dependency. This is an internals port; the + invariant that protects it is already recorded (neutral `ByteStream`, + sans-IO `FrameDecoder`). Tripwire: `aio.py` imports trio at module + level for `trio.CancelScope`, so `execnet.aio.trio` is the trio module + and an asyncio-only install could not import `execnet.aio` today. +- **A native asyncio surface** — `execnet.aio` running gateways on the + *caller's* loop with no host thread, symmetric with `execnet.trio`. + This one is visible in the API: `aio.AsyncGroup(host=)` and + `aio.AsyncGroup.host` would become meaningless. `host=` is honest + today and can be deprecated later behind its `None` default; document + it as "which host thread serves this group", i.e. an + implementation-shaped knob rather than part of the asyncio model. + +`Host` itself is already engine-neutral in name and members; only its +docstring says Trio, which is accurate and would be a docs change. + +## Provisioning and workspaces: what the next xdist should stand on + +3.0's second goal is that the *next* pytest-xdist can stop hand-rolling +deployment. Today xdist does it itself, crudely: + +- one `execnet.Group`, specs prefixed by hand; +- `HostRSync` pushes each rsync root to `basename(root)` under the + gateway's chdir; +- `make_reltoroot()` rewrites command-line args to `root.name + "/" + rel` + and raises if an arg is not under a root; +- the remote interpreter is assumed to already have the project under test + installed — which is why remote xdist is mostly used against a shared + filesystem. + +What execnet should own instead, so xdist's version becomes a few calls: + +1. **Provision the environment, not just execnet.** `_provision.py` + already builds, caches and ships a wheel of a dev-version *execnet* + (`provisioning_wheel`, `wheel_delivery_command`, remote cache under + `"$HOME"/.cache/execnet/wheels`) and launches under `uv run --with`. + Generalize that to arbitrary requirements, including "a wheel of the + project I am running from" — that is the same code path with a + different source directory. Wanted: spec keys along the lines of + `with=` (repeatable) and `project=`. +2. **Deploy a workspace, and hand back the mapping.** One object that + rsyncs a set of roots into a remote workspace directory and returns a + local→remote path mapping, so the caller can translate paths in its own + config instead of reconstructing the layout by convention. The + translation is execnet's to give because the remote root is a + provisioning fact — the caller only knows local paths. +3. **rsync as a first-class operation.** `RSync` still works by + `remote_exec`-ing `_rsync_remote`'s source and tunnelling data as + serialized channel messages. Since execnet is now always *installed* + on the worker, this should follow the pattern the branch established + for `GATEWAY_START_SUB` and friends: a protocol-level request served by + the worker, with the data plane on a raw channel. That also gets the + async surfaces an `RSync`, which they lack. + +Open questions, none decided: + +- Does the workspace API live on `Group` (one deploy fans out to every + gateway, like `HostRSync.add_target_host` does per gateway) or is it a + standalone object gateways are added to, as `RSync` is today? +- Is the mapping a dumb prefix swap, or does it need to survive symlinks, + case-insensitive remotes, and roots that overlap? +- What is the remote workspace root — a per-gateway temp dir, a + content-addressed cache keyed like the wheel cache, or caller-supplied? + Reuse across gateways on the same host is the whole point on a cluster. +- Cleanup: who deletes a workspace, and what happens when the coordinator + dies without terminating. +- How much of this is execnet's job at all versus a thin xdist layer over + points 1 and 3. Answer this one first. + +Doing this well is also what makes the Kubernetes goal tractable — a pod +is just a remote with no shared filesystem and a short life. + +## Kubernetes: test runs in a cluster over the protocol + +The goal is to drive a test run across pods in a Kubernetes cluster using +execnet's protocol, rather than a bespoke agent. Nothing here is built; +the design space, and what the branch already provides: + +**Getting a stream to a pod.** Three shapes, roughly in order of cost: + +1. *A command transport.* Generalize the ssh launcher to any argv that + yields a process whose stdio is the worker's protocol — e.g. + `kubectl exec -i POD -- execnet worker --protocol-stdio`. Cheapest, + works with any cluster access that `kubectl` has, and needs no new + code in the worker. Costs the worker's stdio and offers no dial-back. +2. *Listen plus a proxy.* The worker runs + `execnet worker --protocol-listen 0.0.0.0:0` and reports its address; + the coordinator connects through a `kubectl port-forward` (or directly, + where pod IPs are routable). This is precisely what `--protocol-listen` + was added for and it keeps the worker's stdio. **This is where "an + integrated Kubernetes proxy setup" belongs**: managing the forward's + lifetime, learning the port, and tearing it down. +3. *API-native.* Speak `pods/exec` (SPDY/WebSocket) from the coordinator, + no `kubectl` binary. A real dependency and a streaming adapter that + has to satisfy the `ByteStream` protocol — which is the point of that + protocol being neutral. + +**Getting code into the pod** is the workspace story above, unchanged: an +image with `uv`, execnet provisioned exactly as it is over ssh, the +project under test installed from a shipped wheel, and the tests rsynced. +Once the gateway exists, rsync over the established protocol is strictly +better than a second connection — no extra credentials, no second +authorization path. + +**The decision to take first**: does the pod and proxy machinery live in +execnet, or out of tree? It needs a Kubernetes client and cluster +credentials, which core does not want; but `AsyncGroup._make_gateway`, +`_open_via_stream` and `_resolve_socket_address` are *already* overridable, +so an out-of-tree transport is nearly possible today. Making that +extension point deliberate and documented may be the better 3.x +deliverable, with `execnet-kubernetes` on top. + +Other open questions: pod lifecycle (does execnet create a Job or attach +to something that exists?); image and Python selection; how a `Group` of N +pods maps onto scheduling; and cleanup when the coordinator dies, since a +cluster needs an owner reference or TTL and cannot rely on `terminate` +arriving. + +## Deferred, and one rejection + +- **`execnet.anyio`** (the old Phase E) — a coordinator core on anyio with + an asyncio backend. Deferred, not cancelled; the portability invariants + keep the door cheap. +- **Async-surface gaps**: no `remote_status()`, no `MultiChannel`, no + group iteration, no `RSync` on `execnet.trio`/`execnet.aio`. Deliberate; + worth a documented line rather than silence. +- **`installvia` still needs a socket handoff at all.** It would not, if + the server spawned the worker as the *listener* + (`--protocol-listen 127.0.0.1:0`) and reported its address back: no + `pass_fds`, no `share()`, works on any interpreter, and the spawn happens + *before* the reply so failures are diagnosable by construction. The open + question is how the server learns the port. Standalone `execnet server` + still needs the handoff — it has already accepted the connection. +- **eventlet** stays dead. **Subinterpreters** are a strategy slot, not a + plan. +- **A trampoline process was considered and rejected**: the design already + frees the worker's stdio in-process, and a pump's own stdio side is still + a blocking pipe, so it relocates the thread rather than removing it — at + the cost of a process and two copies per message. Do not revisit without + a new reason. +- **Unverified**: whether the socket/`installvia` path works on the Windows + CI job at all. Assume any platform CI has not exercised is broken. + +## Suggested order + +1. `execnet info` capability key, `__all__` cleanups, underscore the engine + methods, fix the stale wording — small, and item 1 cannot be changed + after release. +2. Changelog and docs renumbered to 3.0; undraft PR #422. +3. Decide the execnet/xdist split for provisioning + workspaces, then build + points 1–3 of that section. This is what the next xdist waits on. +4. Kubernetes: decide in-tree versus extension point, then the proxy. +5. Remove the shims — later in 3.x, gated on xdist having released without + them, with the CI `release` target as the check. diff --git a/doc/basics.rst b/doc/basics.rst index 232804a2..1d1f94dd 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -258,7 +258,7 @@ yourself and specify a larger or not timeout. Worker profiles ==================================================================== -.. versionchanged:: 2.2 +.. versionchanged:: 3.0 The ``execmodel=`` key is now spelled ``profile=`` and only ever described the *worker*. The local execution model it was named after no longer exists: see `Namespaces`_ for the local choice. @@ -300,7 +300,7 @@ default group. Transports ==================================================================== -.. versionadded:: 2.2 +.. versionadded:: 3.0 The Message protocol does not have to be the worker's stdin/stdout. The ``transport=`` key selects: @@ -326,7 +326,7 @@ not implement ``StreamLocal`` forwarding. Worker output ==================================================================== -.. versionchanged:: 2.2 +.. versionchanged:: 3.0 A worker's stdio belongs to the code it runs. It used to be redirected to the null device, so a remote ``print()`` went nowhere at all. @@ -352,7 +352,7 @@ keeping its output visible. The host thread ==================================================================== -.. versionadded:: 2.2 +.. versionadded:: 3.0 The blocking, asyncio and gevent surfaces have no event loop of their own to put gateways on, so protocol IO runs on a :class:`Host`: one OS thread @@ -374,7 +374,7 @@ not terminate them for you. :mod:`execnet.trio` uses no host at all. The execnet command line ==================================================================== -.. versionadded:: 2.2 +.. versionadded:: 3.0 ``execnet server [HOST:PORT] [--once]`` Accept gateway connections on a socket and hand each to a fresh worker diff --git a/handoff-boundary-protocol-rethink.md b/handoff-boundary-protocol-rethink.md deleted file mode 100644 index 5f108df2..00000000 --- a/handoff-boundary-protocol-rethink.md +++ /dev/null @@ -1,163 +0,0 @@ -# Handoff: host boundary protocol rethink (P1–P5, precedes Phase C proper) - -Decided with Ronny 2026-07-26. This supersedes the *ordering* of -`handoff-phase-c-worker-axes.md` (still valid for C content): the boundary -rethink lands first, then C's `loop=`/`exec=` work continues on the -smaller core. The wire protocol (Message framing) is untouched — this is -about how things cross between the trio host loop and its consumers. - -## Rationale - -Consumer→loop is already universal: `LoopPortal.post` rides -`TrioToken.run_sync_soon`, thread-safe from any foreign context (plain -thread, gevent greenlet, asyncio callback, another trio loop). One -ingress, strict FIFO — unchanged. - -Loop→consumer is hardwired to threading primitives (`Channel._items` -execmodel queue, `threading.Event` for waitclose/join/write-acks, -`SyncReceiver`), which is the only reason `ExecModel` still exists. The -fix: the loop never blocks and never knows who listens — it fires a -thread-safe wakeup callable the consumer supplied. Every event loop has -exactly one such primitive: - -| consumer | wakeup primitive | -|---|---| -| plain thread | `threading.Event.set` | -| asyncio | `loop.call_soon_threadsafe` | -| gevent | `hub.loop.async_()` watcher — `send()` is gevent's one documented thread-safe op | -| foreign trio loop | `TrioToken.run_sync_soon` | - -## The boundary kit (grows in `execnet.portal`) - -1. **`Wakener`** — protocol, one thread-safe method `notify()`. The - entire integration surface for a new event loop (eventlet, Qt, …) - is implementing this; no more ExecModel. -2. **`Mailbox[T]`** — unbounded deque + Wakener. `put()` from the loop - (append + notify, never blocks the loop); `get(timeout)` blocking for - sync backends — generalizes the KI-safe `SyncReceiver` drain pattern - (Event.wait + drain, re-check after clear) — and `await`-able for - asyncio/trio wakeners. Replaces `Channel._items`, `SyncReceiver`, - `MultiChannel`'s queue. -3. **`OneShot[T]`** — single-result future on the same wakener. - Replaces the per-send write-ack `threading.Event`; adds - `portal.start(async_fn) -> OneShot` so asyncio callers await - management ops instead of blocking a thread in `portal.run`. - -## Channel unification (decided: full rebase) - -`AsyncGateway._dispatch` becomes the ONLY message router. `RawChannel` -grows a consumer hook: a bound facade diverts payloads into a `Mailbox` -instead of the internal memory channel (callbacks: invoked on the loop -thread, preserving current semantics; moving them later is just posting -via the wakener). - -Sync `Channel` becomes a genuine facade over (raw id, mailbox, portal): - -- `receive()` = `mailbox.get(timeout)` + `loads_internal` at the call - site — deserialization moves off the loop thread; `DataFormatError` - now surfaces at `receive()`. -- `send()` = portal-posted frame + `OneShot` write-ack (non-loop threads - keep block-until-written, 120s → OSError, KI-safe). -- `waitclose()`/`join()` = closed-events on the wakener. -- `__del__` best-effort close keeps `portal.post` (post_message path). -- Unknown inbound ids still auto-create (`_channel_for`). - -Deleted outright: `_receivelock`, `ChannelFactory` dispatch paths -(`_local_receive`/`_local_close`), `SyncBridgeGateway._dispatch`, the -`Message.received` handler-table use, the duplicate close state machine. -`CHANNEL_EXEC`/`GATEWAY_START_*` become per-gateway hooks on the async -core. `TrioWorkerExec` FIFO pump survives unchanged; exec'd code gets -facade channels. - -## Execmodels: retired as machinery, preserved as presets - -execmodel names become presets over three orthogonal axes (spec + worker -CLI config; decided: **both** coordinator and worker side): - -| axis | values | meaning | -|---|---|---| -| `loop` | `main` \| `thread` | where the trio host runs (Phase C) | -| `exec` | `thread` \| `main` \| `task` | where remote_exec code runs (Phase C) | -| `wait` | `thread` \| `gevent` \| `asyncio` | which Wakener blocking facade waits park on (new) | - -Presets: `execmodel=thread` → `loop=main, exec=thread, wait=thread`; -`main_thread_only` → `loop=thread, exec=main, wait=thread`; `gevent` -returns as `loop=thread, exec=thread, wait=gevent` (greenlet channel -waits park the greenlet, not the hub; sends already greenlet-safe via -the portal). eventlet stays dead (third parties can implement Wakener). - -Dies: `ExecModel` ABC, `WorkerPool`, `Reply` (`multi.safe_terminate` -moves to plain threads or delegates to `FacadeAsyncGroup.terminate`). -Stays as deprecated preset-mappers: `get_execmodel`, `set_execmodel`, -`group.execmodel`, `spec.execmodel` (xdist compat). - -## asyncio (decided: land now) - -`execnet.aio` namespace mirroring `execnet.trio`'s surface; every op is -portal ingress + `OneShot`/`Mailbox` on an `AsyncioWakener`. Full -asyncio-native API over the trio host loop, all transports, no anyio -port. Phase E (anyio-native core) becomes optional purity/perf work. - -## Staging — ALL LANDED 2026-07-26 (commits 2c78416..b01d009) - -1. **P1 — boundary kit** (`2c78416`): Wakener/Mailbox/OneShot + - ThreadWakener; SyncReceiver, write-acks, `_done_sync` ported. -2. **P2 — channel unification** (`2a40a55`): sync Channel onto - RawChannel(+set_consumer)+Mailbox, classic dispatch deleted. The kit - lives in trio-free `execnet._boundary` (portal re-exports) so - `import execnet` stays trio-free. Fix worth knowing: an unconsumed - remotely-closed RawChannel stays registered until a consumer claims - it — call-site deserialization means a passed channel can bind after - its data AND close arrived (was a payload-loss race, latent in the - async core's `open_channel(id)` too). -3. **P3 — retirement** (`811fd99`): ExecModel ABC/WorkerPool/Reply gone; - one deprecated ExecModel preset KEEPS the stdlib-delegate members - because pytest-xdist's remote worker calls - `channel.gateway.execmodel.RLock()/Event()`; `wait=` axis end to end - (spec validation, worker CLI config, `BaseGateway._new_wakener`, - `_boundary` registry + Flag); safe_terminate on daemon threads; - test_threadpool.py deleted. -4. **P4 — `execnet.aio`** (`845510e`): implemented as a per-call - _HostBridge (host task + `loop.call_soon_threadsafe` future), NOT the - originally sketched AsyncioWakener/Mailbox — real awaitables over the - trio-native AsyncGroup/AsyncChannel, simpler and semantically exact. - The "asyncio" wait= registry entry is therefore unused/unregistered. - v1 limitation: cancelling a bridged await abandons it aio-side only. -5. **P5 — gevent wakener** (`b01d009`): `execnet._gevent_support`, - lazily imported by `make_wakener("gevent")`; hub-bound pieces created - in the first waiter's hub (carriers may be constructed on the host - loop); tests behind the `gevent` dependency group. - -## SUPERSEDED 2026-07-29 by the surface review - -Phase C landed, and reviewing the whole public surface at once (before -Phase D docs froze it) retired several of the decisions above. See -"Surface review" in `handoff-phase-c-worker-axes.md` for what stands. In -short: - -- **The Wakener extension point is gone.** There was never a plan to let - third parties add event loops, and the asyncio wakener sketched here was - deliberately never built (P4 chose per-call bridging). What is left is - two wait backends, threads and gevent, behind a two-branch - `make_wakener`. `execnet.portal` is now private `execnet._portal`. -- **`wait=` is gone as a spec key.** It described the *caller's* - concurrency library, which is what picking a namespace already says. - gevent got the facade it was missing: `execnet.gevent`. -- **The `loop=` / `exec=` axes stayed dropped**; `execmodel=` became - `profile=` (see below). - -## Invariants (unchanged, re-mapped) - -- Non-loop-thread sends block until the frame hit the OS write (OneShot - write-ack; 120s → OSError); loop-thread sends only enqueue; all sends - one portal FIFO. -- After close: `OSError("cannot send (already closed?)")`. -- exec admission order = message arrival order (TrioWorkerExec._pump). -- Channel callbacks run off the loop thread: a per-channel consumer task - drains into a threadpool call (moved 2026-07-26; the kit made it cheap, - as predicted). See `handoff-phase-c-worker-axes.md` "Callback consumer - tasks". -- `Group.terminate(timeout)` never hangs (~2×timeout). -- Sync blocking waits stay KeyboardInterrupt-interruptible on the main - thread (thread Wakener keeps the Event.wait + drain pattern). -- No source shipping; `import execnet` must not import trio. diff --git a/handoff-history.md b/handoff-history.md new file mode 100644 index 00000000..f0f4c27c --- /dev/null +++ b/handoff-history.md @@ -0,0 +1,178 @@ +# What landed on `feat/trio-host-thread-io`, and what it taught us + +The record, compressed. Current state is `HANDOFF.md`; open work is +`ROADMAP-3.0.md`. This file exists for two reasons: commit messages do +not carry the *why*, and several decisions were made, unmade and remade — +the last section says which ones are dead so nobody resurrects them. + +## The phases + +**A — transports and demolition** (`92969c5`, `48d328c`, `3d1d31e`, +2026-07-24). `via=` generalized to a `GATEWAY_START_SUB` protocol message +covering popen/python/ssh/vagrant sub-specs; `vagrant_ssh` ported; +`gateway_io`, `gateway_bootstrap`, `Popen2IO`, the thread receiver and +`WorkerGateway.serve` deleted. `HostNotFound` became a `ConnectionError` +subclass. This is where the pattern was set: **infra operations are +protocol messages, not `remote_exec`'d source** — possible only because +execnet is now always installed on the worker. + +**B — the inversion** (`0eb6cc5`..`4f84335`). One protocol engine +(`AsyncGateway`), with the sync API as a facade over it; `ProtocolSession` +deleted. Along the way: transports unified on a neutral stream protocol; +the **sans-IO `FrameDecoder`** (receivers pump bytes, a `feed()` state +machine yields messages) which is what made the frame-native `via` relay +and IO-free tests possible; the two-level `RawChannel` / `AsyncChannel` +model that killed the double-framed `ChannelByteIO` tunnel; and the +namespace split. + +Two fixes from B worth keeping in mind: + +- The bridge must attach itself to the sync gateway in `__init__`, *before* + serving starts (`51b9053`). Otherwise a first message replies through + the IO stub and kills the fresh gateway — invisible until `pytest -n 12`. +- A stream-closing `aclose` on the via tunnel must feed EOF to its *own* + reader, or the serve task never finishes and `terminate` hangs. + +**Boundary kit — P1..P5** (`2c78416`..`b01d009`, 2026-07-26). Consumer→loop +was already universal (`LoopPortal.post` on `TrioToken.run_sync_soon`); +loop→consumer was hardwired to threading primitives, which was the only +reason `ExecModel` still existed. The fix: the loop never blocks and never +knows who listens — it fires a thread-safe wakeup the consumer supplied. +`Wakener` / `Mailbox` / `OneShot` / `Flag` live in the trio-free +`execnet._boundary`. The sync `Channel` became a genuine facade over +(raw id, mailbox, portal): deserialization moved to the `receive()` call +site, `AsyncGateway._dispatch` became the only router, and `_receivelock`, +the classic `ChannelFactory` dispatch, `WorkerPool` and `Reply` all went. + +One fix there was subtle: an unconsumed, remotely-closed `RawChannel` must +stay registered until a consumer claims it. Call-site deserialization +means a *passed* channel can bind after both its data and its close have +arrived — a payload-loss race that was latent in the async core too. + +`ExecModel` survived as a deprecated preset **because pytest-xdist's remote +worker calls `channel.gateway.execmodel.RLock()`**. + +**C — worker profiles** (`b2f43c3`..`cbce183`). The `loop=` × `exec=` axes +framing was dropped in favour of named use-case profiles on one key. +`TrioWorkerExec` became a pure FIFO admission pump over strategy objects. +The pytest-relevant fix landed here too: `Message.GATEWAY_INFO` answers +`rinfo()` from the dispatch loop and chdir/nice/env ship in the worker +config, so coordinator bookkeeping can no longer *claim an exec slot* — +previously an info call could occupy the main thread, and pytest ended up +on a worker thread. + +**Callbacks off the loop thread** (2026-07-26). `setcallback` starts a +per-channel consumer *task* that drains an inbox and runs each callback via +`trio.to_thread` under a bounded limiter, holding a strong reference to the +channel (which replaced the old `_callback_channels` registry — lifecycle +is now the task's). `waitclose()` waits on `_consumer_done`, so it still +returns only after every callback including the endmarker. + +**Surface review** (`7aa17fe`..`e75cd0a`, 2026-07-29). The public surface +had settled commit by commit and was never looked at whole; doing that +before the docs froze it retired several earlier decisions (see below). +Result: one namespace per concurrency library you drive execnet from, one +shared `Host` per process, `profile=` as the spec key, and blocking calls +inside a running loop raising instead of hanging. + +**CLI and transports** (`5710eed`, `a84380f`, `2fc4013`, 2026-07-30). The +protocol stopped being the worker's stdin/stdout. `execnet worker` names +the transport and is the launch contract; `execnet info` replaced the +`import execnet, trio` probe. Three things this fixed, each verified while +doing it: remote `print()` used to go to `/dev/null`; the ssh worker config +carried `env:` values in the remote argv, readable by every user on that +host via `ps`; and the dev-coordinator wheel used to be framed in-band on +the protocol pipe with `head -c N`, where it now travels on its own ssh +connection into a remote cache. + +**Windows** (`9413a2e`..`f484d14`). It had never been tested and every +worker died at startup on `trio.lowlevel.FdStream`, which is POSIX-only; +`ThreadedFdStream` does those reads and writes in the thread pool. +`pass_fds` does not exist there, so popen hands the socket over with +`socket.share(pid)` — the flag in argv, the blob in the config on stdin, +because the pid does not exist until the child is spawned. The blob is +bound to that one pid and inert to anything else, which beats handle +inheritance (that would need `close_fds=False` and leak every inheritable +handle to the child *and its grandchildren*). + +**xdist in CI** (`320c89e`, `17e7c7a`, 2026-07-30). See "The xdist +contract" in `ROADMAP-3.0.md`. It found 16 regressions on the first run. + +## Lessons that cost a debugging round + +- **CI was lying.** Until `1839800`, CI had executed *zero tests* since + 2026-07-26: `testing/test_ssh_local.py` imported `asyncssh` at module + level while `tox.ini` carried its own hand-written `deps` list that had + drifted from the `testing` extra, so collection errored and every job + reported `2 skipped, 1 error` while looking like an ordinary failure. + There must never be a second list of test requirements — `tox.ini` uses + `extras = testing`. Relatedly, `[tool.uv] default-groups = ["testing"]` + had been *replacing* uv's `dev` default, so `uv sync` silently + uninstalled pytest-xdist. +- **Hand a socket over as a socket, not as an fd.** `socket.socket(fileno=fd)` + re-derives family/type/proto by querying the handle; PyPy on Windows + raises `WinError 10014` doing that to a `WSADuplicateSocket` handle. No + in-process probe catches it — `share()` and `fromshare()` both work + in-process. What cracked it was noticing which test *passed*: + `popen//transport=socket` was green while the server path failed, which + isolated the difference to one line. +- **`execnet server :0` reported a port nothing listened on.** A wildcard + bind with an ephemeral port gives *each* address family its own random + port and only the first was reported — and which family comes first is + platform-dependent (IPv4 on Linux, IPv6 on Windows). +- **A worker warning can livelock a pytest run.** `execnet.dumps` warned on + every access and xdist calls it from `serialize_warning_message`, i.e. + from inside pytest's warning-recording hook: one `DeprecationWarning` in + a worker recorded a warning that recorded a warning, unbounded. +- **Do not rewrite a caller's spec.** `makegateway` wrote the normalized + profile back onto the caller's `XSpec`; xdist reuses one spec object and + re-reads it to decide whether to prefix again, so it prefixed twice and + built `execmodel=…//execmodel=…//popen`. Every crashed-worker-replacement + test failed. Filling in a *missing* value is idempotent and fine. +- **A killed worker resets a socket where a pipe reaches EOF** — the reader + has to map `BrokenResourceError` to `EOFError` (`f484d14`). Applies to + `socket=` on POSIX too; only nobody had looked. +- `shlex.quote("~/…")` creates a directory literally named `~`; use + `"$HOME"`. And a cached-wheel skip branch must still drain stdin, or the + coordinator gets EPIPE. +- **Generated source is read as UTF-8** (PEP 3120) regardless of locale; + `test_basics` wrote it in the locale encoding and one em-dash in + `_message` broke it off UTF-8 locales. +- Hypothesis found a real serializer bug while stress-testing channels: + `_save_integral` only bounds-checked the *upper* int4 limit, so an int + below `-2**31` overflowed `struct.pack('!i', …)` instead of taking the + long path. +- The doc examples had not been collectable since pytest 7 (a + `pytest_plugins` line in a non-top-level conftest), which is why so much + of them had rotted. `tox -e docs` now runs them as doctests with `-W`. +- The 11 consistent XPASSes were investigated and the `flakytest` marks + kept deliberately: trio's single-loop dispatch plus FIFO admission makes + them pass when idle, but `test_gateway_status_busy` (a `_track_start` + scheduling race) and `test_popen_stderr_tracing` (capfd) still fail under + sustained load. To retire the status marks for real, retry-poll for + `numexecuting == 2` the way those tests already poll for `== 0`. +- The hybrid main-thread claim is **best-effort**, and that surfaced as an + `-n 12` flake rather than by reasoning: `main_thread_only` serialized, so + every sequential `remote_exec` got the main thread, whereas `thread` + releases its claim just after the channel close that lets the coordinator + send the next request — so an immediate re-exec can rarely land on a pool + thread. The *first* request is still deterministic. + +## Decisions that were made and then unmade + +Do not resurrect these; each was tried on paper or in code and dropped. + +| dropped | replaced by | why | +|---|---|---| +| `loop=` / `exec=` spec axes | named worker profiles on one key | use-cases, not axes; the combinations were not all meaningful | +| `wait=` spec key | the namespace you import | it described the *caller's* concurrency library, which the namespace already says | +| a `Wakener` registry (`register_wakener`, lazy backend modules) | a two-branch `make_wakener("thread"\|"gevent")` | exactly two backends exist and there was never a plan to let third parties add event loops | +| `execnet.portal` as public API | private `execnet._portal` + `execnet._boundary` | it published the kit but not the registration hook, so the advertised extension point was unreachable | +| an `AsyncioWakener` + `Mailbox` for `execnet.aio` | a per-call `_HostBridge` (host task + `call_soon_threadsafe` future) | real awaitables over the trio-native objects; simpler and semantically exact. Cancellation was made real in the surface review | +| `main_thread_only`'s concurrent-exec deadlock guard | nothing | it was a 1s-timeout guard whose window false-fired under CPU contention; the restored hybrid `thread` profile already gives the first exec the main thread | +| `aio.Group` / `Gateway` / `Channel` | `aio.AsyncGroup` / … | matches `execnet.trio`, so swapping the import ports the code | +| `open_popen_gateway` | `open_gateway` | it always accepted any spec | +| one `TrioHost` per `Group` | one shared `Host` per process, `Group(host=)` | a host is a thread and a loop, not something groups need isolated | +| a trampoline process for Windows stdio | `ThreadedFdStream` | see the rejection note in `ROADMAP-3.0.md` | +| an anyio/asyncio *core* port | `execnet.aio` over the trio host | rejected for now; the portability invariants keep the door open | +| eventlet | — | dead, deliberately | diff --git a/handoff-phase-b-async-core.md b/handoff-phase-b-async-core.md deleted file mode 100644 index d3a69ff6..00000000 --- a/handoff-phase-b-async-core.md +++ /dev/null @@ -1,182 +0,0 @@ -# Handoff: Phase B — invert execnet onto an async-native Trio core - -**SUPERSEDED by `handoff-phase-c-worker-axes.md`** — kept as the Phase B -record. **Phase B is complete (B.1–B.6)** on branch -`feat/trio-host-thread-io`; work continues at **Phase C**. - -Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` -(never grep-filter pre-commit output). ssh paths have a real local harness -in `testing/test_ssh_local.py` (asyncssh server; system ssh client needed). -Known flake: `test_socket_installvia` EOFs rarely under load. - -## Where the repo stands (2026-07-24, after B.5) - -Trio is the only IO path; no source is shipped over the wire (workers run -`python -m execnet._trio_worker `, foreign/remote interpreters -are uv-provisioned via `_provision.py`, dev coordinators ship a wheel). All -transports work: popen, `python=`, ssh, vagrant_ssh, socket, and `via` -sub-gateways (`GATEWAY_START_SUB` spawn-request + byte relay on the master). - -**The inversion landed (B.5)**: there is one protocol engine — -`AsyncGateway` — and the sync API is a facade over it. `ProtocolSession` -is gone. Coordinator and worker sessions are `SyncBridgeGateway` -(an `AsyncGateway` subclass) whose `_dispatch` runs the classic sync -`Message.received` handlers under `_receivelock`; the sync -`Channel`/`ChannelFactory` state machine in `gateway_base` is unchanged -and remains the semantic layer for blocking users (and the worker's -exec'd code). `multi.Group` owns a `FacadeAsyncGroup` task on its -`TrioHost`; `makegateway` and the bounded process shutdown delegate to -`AsyncGroup`, while `terminate` still calls each member's -`exit()`/`join()` (pinned by `test_basic_group`). - -File map (src/execnet/): - -| file | role | -|---|---| -| `gateway_base.py` | `Message` wire protocol + **`FrameDecoder`** (sans-IO), serializer (incl. duck-typed `save_AsyncChannel` → CHANNEL opcode), sync `Channel`/`ChannelFactory` (raw-receiver registry: `register_raw_receiver`/`allocate_id`, verbatim CHANNEL_DATA routing), `BaseGateway`/`WorkerGateway` (`_send` via bridge session, `_send_nonblocking` for GC), `ExecModel`, `WorkerPool`, `HostNotFound` | -| `_trio_gateway.py` | **the async-native core**: `ByteStream` Protocol, `RawChannel`, `AsyncChannel`, `AsyncGateway` (single serve task; outbound queue items are `(frame, on_written)` so sync senders can wait for the OS write; writer fails pending frames on shutdown; `_finalize` hook for subclass shutdown), `AsyncGroup` (async CM owning the nursery; **all transports**: popen/`python=`/ssh/vagrant_ssh/socket(+installvia)/`via=`; overridable `_make_gateway`/`_open_via_stream`/`_resolve_socket_address`; per-process reaper tasks; terminate = tunneled first, GATEWAY_TERMINATE + timeout + kill, ~2×timeout bound), `RawChannelStream`, `open_popen_gateway`, transport helpers (`connect_command_worker` incl. ssh-255→HostNotFound, `connect_socket_worker`, `start_socketserver_via` (async), `ssh_transport_args`, `popen_worker_argv`) | -| `portal.py` | **`LoopPortal`** (trio-token holder) and **`SyncReceiver`** (loop→plain-thread queue, KI-interruptible `get()`) | -| `_trio_host.py` | sync-facade host: `TrioHost` (loop thread; `start_session` returns a bridge), **`SyncBridgeGateway`** (sync dispatch + portal-FIFO `enqueue_message`/`post_message`/`request_close_write`, threading-`Event` `wait_done`), **`FacadeAsyncGroup`** (bridge-building AsyncGroup; via/installvia through the sync master), `makegateway_trio`, `SyncIOHandle` (remoteaddress/wait/kill/close_write), `RawTunnelStream` (via tunnel; `aclose` feeds own reader EOF), `GATEWAY_START_*` handlers (`_start_sub_and_relay` frame-native) | -| `_trio_worker.py` | worker entry, `TrioWorkerExec` (FIFO `_pump` admission), `_prepare_protocol_fds`; serves on a `SyncBridgeGateway` | -| `gateway.py` | sync coordinator `Gateway` (remote_exec, exit, rinfo) | -| `_exec_source.py` | remote_exec source normalization shared by sync + async coordinators | -| `multi.py` | sync `Group` facade (`_ensure_async_group`, terminate via `AsyncGroup`), `MultiChannel`, `safe_terminate` (WorkerPool-based, kept for tests) | -| `_provision.py` | uv provisioning, wheel build/ship/materialize, argv builders | - -Async-core tests live in `testing/test_trio_gateway.py` (memory_stream_pair -protocol tests + popen/via integration inside `trio.run`). - -Semantics that MUST survive (xdist depends on them): - -- Sends from non-loop threads block until the frame hit the OS write - (120s → OSError) so abrupt `os._exit` cannot drop "sent" data; sends from - the loop thread only enqueue. After close: `OSError("cannot send (already - closed?)")`; `trio.RunFinishedError` maps to the same. -- exec admission order = message arrival order (see `TrioWorkerExec._pump`; - `test_main_thread_only_concurrent_remote_exec_deadlock` guards this — - trio shuffles its run batch, so never rely on task-spawn order). -- Channel callbacks run on the receiver (loop) thread — keep for now. - -## Phase B goal - -Three public namespaces, all pure Trio (**no anyio/asyncio** — an -`execnet.anyio` backend is deferred Phase E; keep idioms portable: neutral -stream protocol, sans-IO protocol logic, no gratuitous trio-only constructs): - -``` -execnet.sync – today's blocking API, rebuilt as a facade (top-level - execnet.Group etc. stay as compatibility aliases) -execnet.trio – trio-native AsyncGroup/AsyncGateway/AsyncChannel, awaited - directly inside the user's own trio.run -execnet.portal – the communicating API (exists as portal.py; public - exposure happens in B.6) -``` - -## Remaining work - -### B.4 Async-native core — DONE (commits `0eb6cc5`..`8286b77`) - -Everything lives in `_trio_gateway.py` (see file map). Landed: -RawChannel + AsyncChannel two-level model, AsyncGateway single-task -dispatch (no `_receivelock` on the async path), AsyncGroup with bounded -nursery-scoped termination, async popen/`python=`/`via=` gateways with -`remote_exec` inside the user's own `trio.run`, and the frame-native via -tunnel (raw-receiver registry in the sync `ChannelFactory`; -`ChannelByteStream` is gone). - -Leftovers deliberately deferred: - -- **ChannelFile / makefile over RawChannel** — do together with the B.5 - facade (the current `ChannelFileRead` is str-based; decide there whether - a text wrapper stays for backward compat). -- rsync data plane / wheel shipping over raw channels — later. -- async ssh/socket gateways — the async `makegateway` only does - popen-style and `via=`; other transports stay on the sync host path - until B.5/B.6 need them. -- worker-side CHANNEL_EXEC on an AsyncGateway is rejected with a - RemoteError ("unsupported message") — async exec is Phase C - (`exec=task`). - -### B.5 Rebuild the sync API as a facade — DONE (commit `f932084`) - -Public surface stayed byte-for-byte; the whole sync suite passes -unchanged (508 passed). What landed, and the decisions taken: - -- One engine: `SyncBridgeGateway(AsyncGateway)` serves both coordinator - and worker; `ProtocolSession` deleted. Sync dispatch still runs the - `gateway_base` Message handlers, so the sync `Channel`/`ChannelFactory` - semantics (queues, callbacks, ENDMARKER, weakref drop) are untouched — - and remain what the worker's exec'd code sees. -- Send invariant kept: `enqueue_message` posts every frame through the - portal (one global FIFO); non-loop threads wait on the frame's - `on_written` ack (120s → OSError), the loop thread only enqueues. -- `Channel.__del__` now goes through `_send_nonblocking` → - `SyncBridgeGateway.post_message` (portal post, never waits; falls back - to `_send` for duck-typed test gateways). -- KI decision: blocking data paths (send-ack, receive, waitclose, join) - wait on `threading.Event`/`queue.Queue` — KI-interruptible as before. - Management ops (makegateway, terminate) run inside `portal.run` - (`trio.from_thread.run`) where KI is deferred; accepted for now. -- `Group.terminate` still calls member `exit()`/`join()` (contract pinned - by `test_basic_group`), with process wait/kill delegated to - `AsyncGroup.terminate` (tunneled-first, ~2×timeout bound). -- Gotcha fixed on the way: a stream-closing `aclose` on the via tunnel - (`RawTunnelStream`) must feed EOF to its own reader, else the bridge's - serve task never finishes and terminate hangs. -- Post-B fix (`51b9053`): the session-attach startup race — the bridge - must attach itself to the sync gateway in `__init__`, before serving - starts, or a first-message reply goes through the IO stub and kills - the fresh gateway (predated B.5; surfaced by `pytest -n 12`). The - suite now runs xdist-clean. -- `ChannelFile`/`makefile`: kept the existing str-based `gateway_base` - classes as-is for backward compat (they only use channel send/receive). -- NOT yet retired: `ExecModel` internals and `WorkerPool` (still the - worker STATUS duck-type shape and used by `multi.safe_terminate`, - `testing/test_threadpool.py`, conftest's `pool` fixture). Do this at - the end of the phase (B.6 or later) if the tests pinning them move. - -### B.6 Namespace split — DONE (commit `4f84335`) - -`execnet.sync` (blocking facade re-exports; top-level `execnet.*` aliases -into it), `execnet.trio` (AsyncGroup/AsyncGateway/AsyncChannel, raw -channels, stream helpers, shared serialization + errors), and -`execnet.portal` (LoopPortal/SyncReceiver). `trio`/`portal` load lazily -via module `__getattr__`, so `import execnet` still does not import the -trio event loop (pinned by `testing/test_namespaces.py`). Trio-native -protocol/integration tests already live in `testing/test_trio_gateway.py` -(memory_stream_pair + real popen gateways inside `trio.run`). - -### End-of-phase leftovers (deferred into C/D) - -- Retire `ExecModel` internals and `WorkerPool`: still pinned by - `multi.safe_terminate`, `testing/test_threadpool.py`, and conftest's - `pool` fixture, and `get_execmodel`/backend names feed Phase C's - `execmodel=` compat mapping — retire once C lands the new worker - config axes. -- ChannelFile / makefile over RawChannel (async makefile) — untouched; - the sync str-based wrappers stay for backward compat. -- rsync data plane / wheel shipping over raw channels — later. -- Async remote_exec on the worker (`exec=task`) — Phase C. - -## Invariants (do not regress) - -- No source shipping, ever. Workers import installed execnet+trio; rough - major/minor version check (`_trio_worker._check_version`) warns. -- Wheel-on-demand for `GATEWAY_START_SUB` is a recorded TODO in - `_provision.spawn_request` (currently ships eagerly whenever the sub-spec - has `python=`/`ssh=`). -- Worker teardown: `_trio_worker._run_worker` ends with `os._exit(0)` - because trio's `to_thread` cache uses non-daemon threads. -- `Group.terminate(timeout)` must never hang (bounded even when kill is - stuck). - -## After Phase B (context, don't build now) - -- **C**: worker config axes `loop=thread|main` × `exec=thread|main|task` - (compat: `execmodel=thread` → loop=main+exec=thread; `main_thread_only` → - loop=thread+exec=main); `exec=task` later enables async remote_exec. -- **D**: docs + trio-surface tests, pytest-xdist verification. -- **E** (deferred): `execnet.anyio` — coordinator core on anyio, - `backend="asyncio"` knob for the facade. `FrameDecoder` and the B.4 - channel layers are IO-free by construction, so they port as-is; anyio's - `StapledByteStream` satisfies the `ByteStream` protocol structurally. diff --git a/handoff-phase-c-worker-axes.md b/handoff-phase-c-worker-axes.md deleted file mode 100644 index 4e91d32e..00000000 --- a/handoff-phase-c-worker-axes.md +++ /dev/null @@ -1,398 +0,0 @@ -# Handoff: Phase C — worker config axes (`loop=` × `exec=`) + Phase D - -For a fresh session on branch `feat/trio-host-thread-io`. Phase B (the -inversion onto the async-native Trio core) is **complete**; this doc -supersedes `handoff-phase-b-async-core.md` (kept for the B record). -Plan context lives in session memory (`trio-port-plan`), but everything -needed is restated here. - -The rework is up as **draft PR pytest-dev/execnet#422** (branch pushed -to the `origin` fork). Keep it updated as C/D land; undraft once the -docs overhaul (D.1) at least covers migration notes. - -Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` -(never grep-filter pre-commit output). The suite is xdist-clean: `uv run -pytest testing/ -n 12` passes (~7s) since the session-attach race fix -(`51b9053`) — keep it that way. ssh paths have a real local harness in -`testing/test_ssh_local.py` (asyncssh server; system ssh client needed). -Known flake: `test_socket_installvia` EOFs rarely under load. - -## Surface review — LANDED 2026-07-29 (`7aa17fe..e75cd0a`) - -The public surface had settled commit by commit and was never reviewed as -a whole. Doing that before the Phase D docs froze it produced the -following, which **wins over anything below or in -`handoff-boundary-protocol-rethink.md` that contradicts it**. - -**Namespaces are now one per concurrency library you drive execnet from**: -`execnet.sync` (threads; the top-level aliases), `execnet.trio`, -`execnet.aio`, `execnet.gevent`. `execnet.portal` is gone -- -`execnet._portal` plus the trio-free `execnet._boundary`. - -| was | is | why | -|---|---|---| -| `execnet.portal` public | `execnet._portal` private | it exported `Wakener`/`Mailbox`/`OneShot`/`LoopPortal` but *not* `register_wakener`, so the advertised extension point was unreachable -- and there is no plan to let third parties add event loops at all | -| Wakener registry (`register_wakener`, lazy module table) | two-branch `make_wakener(Literal["thread","gevent"])` | exactly two backends exist; every other library gets a facade | -| `wait=` spec key | gone; `Group._wait_backend`, set by the facade | it described the *caller*, which the namespace already says. Worker-side wait was always derived from the profile | -| gevent via `wait=gevent` | `execnet.gevent.Group` | symmetric with the other surfaces | -| `execmodel=` spec key | `profile=` (`execmodel=` a permanent alias) | the key selects the *worker profile*; the local execution model it was named after no longer exists | -| `Group(execmodel=)`, `set_execmodel`, `group.execmodel` | deprecated; only the remote default survives | they had no behavioural effect. **xdist passes `Group(execmodel=...)` as a keyword** -- that must keep working | -| `main_thread_only` profile | deprecated alias for `thread` | `HybridExec` already gives the first remote_exec the real main thread. Its extra behaviour (refusing a second concurrent remote_exec) was a 1s-timeout deadlock guard, now deleted | -| one `TrioHost` per `Group` | one shared `execnet.Host` per process, `Group(host=...)` to override | a host is a thread and a loop, not something groups need isolated | -| blocking inside a running loop hangs | raises, naming `execnet.aio` / `execnet.trio` | worker-side channels stay exempt: exec'd code may run its own loop | -| `aio.Group`/`Gateway`/`Channel` | `aio.AsyncGroup`/`AsyncGateway`/`AsyncChannel` | matches `execnet.trio`; swapping the import ports the code | -| `open_popen_gateway` | `open_gateway` (both async surfaces) | it always accepted any spec | - -Behaviour changes worth a changelog line: - -- a second concurrent `remote_exec` under `main_thread_only` used to close - the channel with `MAIN_THREAD_ONLY_DEADLOCK_TEXT`; it now runs on a pool - thread. `_executetask_complete`, `MAIN_THREAD_ONLY_ADMIT_TIMEOUT` and - the error text are deleted; `MainExec` became `PrimaryThreadPump`. -- `execnet.aio` cancellation is now real: a cancelled `receive` cancels - the host-side operation instead of consuming and discarding an item. - `send`/`send_eof`/`aclose`/`terminate` are shielded instead. -- the boundary carriers raise `execnet.TimeoutError`, not the builtin; - `OneShot` double-resolve is a `RuntimeError`, not an `assert`. -- `STATUS` answers both `profile` and (legacy) `execmodel`. -- **Latent livelock fixed**: `execnet.dumps` warned on *every* access, and - xdist calls it from `serialize_warning_message` -- i.e. from inside - pytest's warning-recording hook. One DeprecationWarning in a worker - therefore recorded a warning that recorded a warning, unbounded, and - wedged the run. The shim warns once per process - (`execnet._xdist_compat_warned`). Anything that warns in a worker can - hit this class of bug; keep it in mind. - -New tests: `testing/test_host.py` (sharing, explicit `Host`, fork, the -loop guards), `testing/test_boundary.py` (renamed from `test_portal.py`), -aio cancellation contracts in `testing/test_aio.py`. The `execmodel` -fixture parametrization collapsed to a single `profile` fixture, so the -suite is ~540 items rather than ~765. - -### D.3 done: the xdist suite runs in CI (2026-07-30) - -The `xdist` job in `.github/workflows/test.yml` runs pytest-xdist's own -test suite against the execnet built from the branch, in two variants: - -- **`release`** — pinned `v3.8.0` + `pytest<9`, a combination verified - green against *released* execnet 2.1.2 (196 passed / 0 failed). This - one blocks: a failure means we broke something. Bump the ref and the - pytest pin together, and only after re-checking the new pair against - released execnet — xdist 3.8.0 with pytest 9 fails two of its own tests - regardless of execnet, which is why the pin exists. -- **`default-branch`** — floating, `continue-on-error`. Early warning for - drift on either side, without letting an xdist-side breakage block - execnet PRs. The `ref` is left empty so it follows xdist's default - branch, which is `master`, not `main` — worth knowing before hardcoding - a name. - -**Doing this for the first time found 16 real regressions** that our suite -could not see (it runs xdist as a *tool*, which exercises none of the -crash-replacement or report-serialization paths). Baseline for xdist's -default branch: released execnet 2.1.2 gives 220 passed / 0 failed. Two -root causes, both fixed: - -1. `makegateway` wrote the *normalized* profile back onto the caller's - XSpec. xdist reuses one spec object and re-reads `spec.execmodel` to - decide whether it still needs the `execmodel=main_thread_only//` - prefix, so after normalization it prefixed a second time and built - `execmodel=...//execmodel=...//popen` -> `ValueError: duplicate key`. - Every crashed-worker-replacement test failed. Fix: `resolve_profile` - validates and warns but callers no longer assign its result to a - caller-owned spec; `effective_profile` (no warning) maps at the - consumption points. **Rule: filling in a missing spec value is - idempotent and fine; rewriting one the caller set is not.** -2. the `execnet.dumps` deprecation warning (see above). - -Remaining known failure, deselected via `.github/xdist-known-failures.txt` -(both jobs are green otherwise): `test_remote_inner_argv` asserts -`sys.argv == ["-c"]` and documents "the behavior due to execnet using -`python -c`" -- which the no-source-shipping worker launch deliberately -changed. That one needs an xdist PR. The deselect list is shared by both -targets; pytest ignores a `--deselect` that matches nothing, so an entry -may name a test only one xdist version has. - -Still open from the review, deliberately not done: the async surfaces have -no `remote_status()`, no `MultiChannel`, no group iteration, and no -`RSync`. `AsyncGroup.makegateway` defaults workers to the `thread` -profile (the coordinator's shape does not dictate the worker's). - -## CLI + transports — LANDED 2026-07-30 - -The protocol used to *be* the worker's stdin/stdout. `execnet worker` now -names the transport, and the CLI is the launch contract (every launcher -emits `python -m execnet worker ...`, or `execnet worker ...` under -`uv run`). `_trio_worker` keeps the serving/profile layer; argument -parsing lives in `_cli`. - -| transport | who uses it | how | -|---|---|---| -| `--protocol-fd N` / `R,W` | popen (`transport=socket`), socketserver, `installvia` | inherited socketpair via `pass_fds`; a single fd must be a socket (checked with `S_ISSOCK`), a pipe needs the pair form | -| `--protocol-connect ADDR` | ssh / vagrant_ssh (`transport=socket`) | coordinator binds a private unix socket, `ssh -R` forwards it, the worker dials back | -| `--protocol-listen ADDR` | nothing yet — for trampolines | binds, prints `{"listening": ...}`, serves the first connection | -| `--protocol-stdio` | Windows, `transport=stdio`, and the `via` tunnel (which relays over the sub's stdio by construction) | the classic pipe pair | - -`transport=socket|stdio` is a spec key defaulting per platform -(`_provision.resolve_transport`); Windows stays on stdio because -`subprocess` rejects `pass_fds` there and `ssh -R` cannot forward a unix -socket. **Still unverified: whether the pre-existing socket/`installvia` -path (which already used `pass_fds`) works on the Windows CI job at all.** - -Things this fixed, each verified while doing it: - -- remote `print()` used to go to `/dev/null`; a worker's stdio is now the - user's, with `stdin=`/`stdout=`/`stderr=` spec keys to override. On the - stdio transport the fallback is close stdin + stdout→stderr, so output - is visible without a second stream. -- the ssh worker config carried `env:` values **in the remote argv**, - readable by every user on that host via `ps`. The socket transport - frees ssh's stdin, so it goes there (`--config-fd 0`). -- the dev-coordinator wheel was framed in-band on the protocol pipe with - `head -c `; it now travels on its own ssh connection before the - launch, cached remotely under `"$HOME"/.cache/execnet/wheels`. Two - traps found writing that: `shlex.quote("~/...")` creates a directory - literally named `~` (use `"$HOME"`), and the cached-skip branch must - still drain stdin or the coordinator gets EPIPE. - -`HostNotFound` needed rework for the dial-back: with no shared pipe there -is no EOF to observe, so `_accept_or_diagnose` races the accept against -`process.wait()` and still maps ssh's exit 255. - -The asyncssh harness in `testing/test_ssh_local.py` needed a -`server_factory` whose `unix_server_requested` returns True — asyncssh -refuses `-R` unix forwards otherwise, which is what a dial-back needs. - -Known semantic loss, documented on `HybridExec` and in the changelog: -`main_thread_only` serialized, so *every* sequential `remote_exec` got the -main thread. The `thread` profile releases its claim just after the -channel close that lets the coordinator send the next request, so an -immediate re-exec can rarely land on a pool thread. The first request is -still deterministic (FIFO admission). This surfaced as a `-n 12` flake, -not by reasoning — worth remembering that the hybrid claim is best-effort. - -## Where the repo stands (2026-07-25, after `a69b844`) - -One protocol engine: `AsyncGateway` (`_trio_gateway.py`). The sync API -is a facade over it — coordinator and worker sessions are -`SyncBridgeGateway` (an `AsyncGateway` subclass in `_trio_host.py`) whose -`_dispatch` runs the classic sync `Message.received` handlers under -`_receivelock`; `SyncBridgeGateway.__init__` attaches itself to the sync -gateway *before* serving starts (the fix for the startup race where a -STATUS/CHANNEL_EXEC arriving first replied through the IO stub and killed -the session). `multi.Group` owns a `FacadeAsyncGroup` task on its -`TrioHost`; makegateway and bounded process shutdown delegate to -`AsyncGroup`, while `terminate` keeps the member `exit()`/`join()` -contract (pinned by `test_basic_group`). `AsyncGroup` supports every -transport (popen, `python=`, ssh, vagrant_ssh, socket with installvia, -`via=`). Public namespaces: `execnet.sync` (top-level `execnet.*` -aliases into it), `execnet.trio`, `execnet.portal` (trio/portal lazy via -module `__getattr__`; `import execnet` must not import trio — pinned by -`testing/test_namespaces.py`). - -No source is shipped over the wire: workers run -`python -m execnet._trio_worker `; foreign/remote -interpreters are uv-provisioned via `_provision.py`; dev coordinators -ship a wheel. - -File map (src/execnet/): - -| file | role | -|---|---| -| `_message.py` / `_serialize.py` / `_channel.py` / `_gateway_base.py` / `_errors.py` / `_execmodel.py` | split by concern since this table was written: wire protocol + sans-IO `FrameDecoder`; serializer (CHANNEL opcode incl. duck-typed `save_AsyncChannel`); sync `Channel`/`ChannelFactory`; `BaseGateway`/`WorkerGateway`; error types; `WORKER_PROFILES` + `resolve_profile` + the deprecated `ExecModel` xdist shim. `gateway_base.py` is now only a warning shim | -| `_trio_gateway.py` | async core: `ByteStream` Protocol, `RawChannel`/`AsyncChannel`, `AsyncGateway` (outbound queue of `(frame, on_written)`; `_finalize` hook), `AsyncGroup` (all transports, overridable `_make_gateway`/`_open_via_stream`/`_resolve_socket_address`, reapers, bounded terminate), stream/argv helpers | -| `_trio_host.py` | `TrioHost` (loop thread), `SyncBridgeGateway`, `FacadeAsyncGroup`, `makegateway_trio`, `SyncIOHandle`, `RawTunnelStream` (via tunnel; `aclose` feeds own reader EOF), `GATEWAY_START_*` handlers | -| `_trio_worker.py` | worker entry (`_main`/`serve_popen_trio`/`serve_socket_trio`), `TrioWorkerExec` (FIFO `_pump` admission; `integrate_as_primary_thread`), `_prepare_protocol_fds`, `_check_version` | -| `gateway.py` / `multi.py` | sync `Gateway` / sync `Group` facade, `MultiChannel`, `safe_terminate` (WorkerPool-based, kept for tests) | -| `sync.py` / `trio.py` / `aio.py` / `gevent.py` | the four public namespaces; `_host.py` holds the shared `Host`, `_portal.py`/`_boundary.py` the (private) boundary kit | -| `_exec_source.py`, `_provision.py`, `xspec.py`, `rsync.py` | source normalization, uv provisioning + argv builders, spec parsing, rsync | - -## Semantics that MUST survive (xdist depends on them) - -- Sends from non-loop threads block until the frame hit the OS write - (120s → OSError) so abrupt `os._exit` cannot drop "sent" data; loop - thread sends only enqueue. All sends go through one portal-posted FIFO. -- After close: `OSError("cannot send (already closed?)")`; - `trio.RunFinishedError` maps to the same. -- exec admission order = message arrival order (`TrioWorkerExec._pump`; - `test_main_thread_only_concurrent_remote_exec_deadlock` guards this — - trio shuffles its run batch, so never rely on task-spawn order). -- Channel callbacks now run in a threadpool thread driven by a per-channel - consumer *task* on the loop (RESOLVED 2026-07-26, was the "revisit in D" - open decision — moved off the loop). A slow callback no longer blocks the - reader; per-channel order is still strict, and `waitclose()` still returns - only after every callback (incl. the endmarker) has run. See "Callback - consumer tasks" below. -- `Group.terminate(timeout)` never hangs (~2×timeout bound, issues - #43/#221). -- Sync blocking waits (send-ack, receive, waitclose, join) stay on - `threading.Event`/`queue.Queue` so KeyboardInterrupt can interrupt - them; `portal.run` (KI-deferred) is only for management ops. - -## Phase C — LANDED: use-case worker profiles on `execmodel=` (2026-07-26) - -Where this conflicts with anything below or elsewhere, this section -wins. The axes framing (`loop=`/`exec=` as spec keys, a reserved -`backend=` axis) was **dropped** in the final rethink with Ronny: -different use-cases get named profiles, `execmodel=` is the public mode -key (xdist already passes it), and `wait=` is the only other public -knob. Implemented in commits `b2f43c3..cbce183`: - -| `profile=` (was `execmodel=`) | loop thread | exec'd code runs | channel | worker wait | extra deps | -|---|---|---|---|---|---| -| `thread` (default) | side thread | **classic hybrid restored**: primary on the main thread, overflow on pool threads (claim decided during FIFO admission) | sync | thread | — | -| ~~`main_thread_only`~~ | *deprecated 2026-07-29, aliases to `thread`* | | | | | -| `trio` (new) | **main thread** | async sources as tasks — one single thread total; top-level await or async def; sync sources rejected; termination cancels tasks | AsyncChannel | (loop) | — | -| `gevent` (revived) | side thread | greenlets on a main-thread hub, one per remote_exec | sync | gevent (derived) | `execnet[gevent]`, auto-added by uv provisioning | - -(The coordinator-side counterpart of the last row is now `execnet.gevent`, -not `wait=gevent`.) - -Architecture: `TrioWorkerExec` is a pure FIFO admission pump delegating -to strategy objects (`WORKER_EXEC_STRATEGIES` in `_trio_worker.py`: -PoolExec building block, MainExec, HybridExec, GreenletExec; TaskExec -serves a plain AsyncGateway via its pluggable `_exec_handler` — no sync -bridge at all in the trio profile). Subinterpreters: future strategy -slot, not built. `WORKER_PROFILES` (`_execmodel.py`) validates -coordinator-side in makegateway, via `resolve_profile`. - -**Native info/setup** (the pytest fix): `Message.GATEWAY_INFO` (code 10) -answers `_rinfo()` from the dispatch loop; chdir/nice/env ship in the -worker config JSON and apply at startup. Coordinator bookkeeping can no -longer claim an exec slot (previously an info call could occupy the main -thread so pytest landed on a worker thread) — `rinfo_source` and the -post-start remote_exec setup block are gone. - -Coordinator-gevent integration: `TrioHost.call_pending` (OneShot from a -host task) backs makegateway / `SyncIOHandle.wait/kill` / -`Group.terminate` whenever the group's wait backend is not `thread`, so a -gevent app's management ops park only the calling greenlet. The default -`thread` backend keeps the KI-deferred `portal.run` path. - -Still decided/standing: core stays trio-only (anyio/asyncio-core port -rejected; asyncio apps use `execnet.aio`); eventlet stays dead; exec'd -code may start its own loop in every profile except `trio`. - -## Phase C — original gap analysis (pre-decision record, superseded) - -## Phase C — original gap analysis (pre-decision record) - -Target axes: `loop=thread|main` × `exec=thread|main|task`. -Compat mapping: `execmodel=thread` → `loop=main` + `exec=thread`; -`execmodel=main_thread_only` → `loop=thread` + `exec=main`. - -1. **Spec surface**: `xspec.py` only knows `execmodel=`. Add `loop=` / - `exec=` keys, combination validation, and the compat mapping. Same - for the worker CLI config JSON (`_provision.worker_cli_arg` still - ships `{"id", "execmodel", "coordinator_version"}`). -2. **`loop=main` does not exist**: the worker always starts `TrioHost` - on a dedicated thread (`serve_popen_trio`), main thread parked in - `gateway.join()` or integrated as exec primary. The compat mapping - means plain `execmodel=thread` workers should end up with `trio.run` - on the main thread — a structural change to `_run_worker`, not a - flag. -3. **`exec=main`** is today's `main_thread_only` machinery - (`integrate_as_primary_thread` + `_executetask_complete` deadlock - guard) — keep behavior, re-home under the new naming. -4. **`exec=task` (async remote_exec)**: `CHANNEL_EXEC` on a pure - `AsyncGateway` is still rejected ("unsupported message" in - `_trio_gateway._dispatch`). Needs a task-based exec scheduler - handing exec'd code an `AsyncChannel`, preserving FIFO admission, - plus explicit cancellation semantics. -5. **STATUS / `remote_status()`** reports `execmodel`; must speak the - new axes compatibly. -6. **Retire `ExecModel`/`WorkerPool`** (deferred from B): replace - internals with plain `threading`/`queue`; deprecated shims for - `group.execmodel`/`set_execmodel`/`spec.execmodel`. Pinned today by - `multi.safe_terminate`, `testing/test_threadpool.py`, conftest's - `pool` fixture, `Channel._items` (`execmodel.queue`), and the worker - STATUS duck-type — gated on the compat mapping landing first. -7. **Wheel-on-demand for `GATEWAY_START_SUB`**: recorded TODO in - `_provision.spawn_request` (currently ships eagerly whenever the - sub-spec has `python=`/`ssh=`). - -## Phase D — what is missing - -1. **Docs are entirely stale**: `doc/` still describes source - bootstrapping and execmodels; nothing on uv provisioning, the - no-source-shipping/version-compat policy, the three namespaces, or - `execnet.trio`/`execnet.portal` APIs. `doc/implnotes.rst` references - pre-inversion internals. Changelog entries for the whole branch. -2. **Trio-surface test gaps**: async ssh/socket/vagrant transport tests - (the asyncssh harness only exercises the sync facade; both share - `connect_command_worker`, so a direct `AsyncGroup` ssh test is - cheap), cancellation mid-`remote_exec`/`receive`, `send_eof` and - reconfigure against real workers, B.5 engine paths (write-ack - failure, `enqueue_message` after close). -3. **pytest-xdist verification**: run pytest-xdist's own suite against - this branch plus real `-n` smoke runs (crash/endmarker tests, - `main_thread_only` GUI case). Our own suite running `-n 12` green is - necessary but not sufficient. -4. **Open decision CLOSED (2026-07-26)**: channel callbacks moved off the - loop thread — see "Callback consumer tasks" below. -5. **xfail markers audit (done 2026-07-26, keep as-is)**: the 11 - consistent XPASSes were investigated — trio's single-loop dispatch + - FIFO admission makes them pass reliably when idle, but under - sustained load `test_gateway_status_busy` (numexecuting race: - `_track_start` runs in a separately scheduled task) and - `test_popen_stderr_tracing` (capfd race) still fail, so the - `flakytest` marks stay. `test_safe_terminate2`'s xpass is CPython - dummy-thread accounting, unrelated to trio. To retire the status - marks for real: retry-poll for `numexecuting == 2` like the tests - already do for `== 0`. - -Recommended order: C.1+C.2 first (spec axes + loop placement) since the -compat mapping unblocks the ExecModel retirement, then `exec=task`; run -the xdist verification (D.3) mid-C as an early canary rather than only -at the end. - -## Callback consumer tasks (`setcallback`, LANDED 2026-07-26) - -`setcallback` no longer delivers inline on the loop thread and no longer -pins the channel in a `ChannelFactory._callback_channels` registry. Instead -`Channel.setcallback` → `BaseGateway._start_channel_consumer` → -`SyncBridgeGateway.attach_consumer`, which on the loop: - -- moves any already-buffered mailbox items into a fresh per-channel inbox - (a `trio` memory channel), diverts future raw payloads there - (`raw.set_consumer(inbox.send_nowait, on_close)`), and nulls `_mailbox`; -- starts a consumer *task* on the host root nursery (`_run_consumer`) that is - handed a **strong** reference to the sync `Channel` — so the channel's - lifecycle is now bound to the task (and to GC once the stream closes), - replacing the strong-ref registry. - -`_run_consumer` does `async for data in inbox:` and runs each callback via -`trio.to_thread.run_sync(..., limiter=host.callback_limiter)` — off the loop, -one thread of a bounded pool (`DEFAULT_CALLBACK_THREADS=40`), sequential per -channel (order preserved), concurrent across channels. A raising callback -(or a `loads_internal` failure) → `CHANNEL_CLOSE_ERROR` + local close -(`_consumer_failed`). On EOF/close the endmarker fires (shielded, bounded by -`CONSUMER_ENDMARKER_GRACE`) and the task sets `channel._consumer_done`. - -Key invariant preserved: `waitclose()` waits on `_consumer_done` (not -`_receiveclosed`) for callback channels, so it still returns only after every -callback — including the endmarker — has run (`test_waiting_for_callbacks`, -`test_channel_endmarker_callback`). Deleted along the way: `_callback`, -`_endmarker`, `_fire_endmarker`, `_callback_channels`, -`_register_callback_channel`, and the callback branch of `_deliver_payload` -(now uniformly `mailbox.put`). `__del__`'s `CHANNEL_LAST_MESSAGE` case folds -away (a callback channel is never GC'd while open). - -Stress coverage: `testing/test_channel_stress.py` (Hypothesis) with a -`--stress=N` pytest option (profiles registered in -`testing/conftest.py::pytest_configure`; default quick profile). Hypothesis -also surfaced and we fixed a latent serializer bug: `_save_integral` only -bounds-checked the upper int4 limit, so a negative int below `-2**31` -overflowed `struct.pack('!i', …)` instead of taking the long path -(`FOUR_BYTE_INT_MIN` added; regression `test_serializer.test_int_boundaries`). - -`hypothesis` was added to the `testing` extra in `pyproject.toml`. - -## Invariants (do not regress) - -- No source shipping, ever. Workers import installed execnet+trio; - rough major/minor version check (`_trio_worker._check_version`) warns. -- Worker teardown: `_trio_worker._run_worker` ends with `os._exit(0)` - because trio's `to_thread` cache uses non-daemon threads. -- `import execnet` must not import the trio event loop. -- Keep async-core idioms anyio-portable (neutral `ByteStream`, sans-IO - `FrameDecoder`) — `execnet.anyio` is deferred Phase E. diff --git a/handoff-public-api-review.md b/handoff-public-api-review.md deleted file mode 100644 index 87e8261c..00000000 --- a/handoff-public-api-review.md +++ /dev/null @@ -1,199 +0,0 @@ -# Public API review, and what pins us to Trio - -Branch `feat/trio-host-thread-io`, after `98eb08d` (the Phase D docs). -Companion to the "Surface review" section in -`handoff-phase-c-worker-axes.md`, which decided the *shape* of the -namespaces; this one takes inventory of what that shape actually publishes -now that the docs freeze it, and asks a second question: **if the host -thread ever ran plain asyncio instead of Trio, what in today's public -surface would we regret having published?** - -Nothing here has been applied. It is a list of decisions, with a -recommendation each. - -## 1. The surface as it stands - -`execnet` == `execnet.sync` plus `__version__` and `can_send` -(`test_namespaces.py` asserts exactly that): - -``` -Channel Gateway Group Host MultiChannel RSync XSpec -DataFormatError DumpError LoadError RemoteError TimeoutError HostNotFound -default_group makegateway set_profile set_execmodel -can_send __version__ -``` - -| namespace | exports | -|---|---| -| `execnet.sync` | the list above minus `can_send`/`__version__` | -| `execnet.trio` | `AsyncGroup`, `AsyncGateway`, `AsyncChannel`, `open_gateway`, `XSpec`, the six error types | -| `execnet.aio` | the same, **plus `Host` and `default_host`** | -| `execnet.gevent` | the sync surface, with its own `Group`/`default_group`/`makegateway` | - -Class surfaces (public attributes, inherited included): - -| class | members | -|---|---| -| `Gateway` | `id`, `remoteaddress`, `remote_exec`, `remote_status`, `newchannel`, `hasreceiver`, `exit`, `join`, `remote_init_threads`\* | -| `Channel` | `send`, `receive`, `setcallback`, `makefile`, `close`, `waitclose`, `isclosed`, `next`, `RemoteError`, `TimeoutError` | -| `Group` | `makegateway`, `remote_exec`, `terminate`, `allocate_id`, `defaultspec`, `host`, `profile`, `set_profile`, `execmodel`\*, `remote_execmodel`\*, `set_execmodel`\* | -| `Host` | `name`, `running`, `close` (context manager) | -| `MultiChannel` | `send_each`, `receive_each`, `make_receive_queue`, `waitclose` | -| `trio.AsyncGateway` | `remote_exec`, `terminate`, `remoteaddress`, `aclose`, `wait_closed`, `closed`, **`open_channel`, `open_raw_channel`, `enqueue_frame`** | -| `aio.AsyncGateway` | `id`, `remoteaddress`, `remote_exec`, `terminate` | -| `*.AsyncChannel` | `send`, `receive`, `send_eof`, `aclose`, `wait_closed`, `isclosed`, `id` | -| `aio.AsyncGroup` | `start`, `aclose`, `makegateway`, `host` | -| `trio.AsyncGroup` | `makegateway`, `terminate` | - -\* deprecated. - -Non-Python surface, equally public and harder to change once released: the -`execnet` CLI (`worker` / `server` / `info`), the spec keys, and the -`execnet info` JSON. - -## 2. What changed since 2.1 - -**Added** - -- Four namespaces (`sync` / `trio` / `aio` / `gevent`); the top level is - an alias surface over `sync`. -- `execnet.Host`, `Group(host=)`, `Group.host`, `AsyncGroup(host=)`. -- `execnet.can_send` — the supported replacement for probing with - `dumps`/`DumpError`. -- `set_profile`, `Group(profile=)`, `Group.profile`, `Group.set_profile`. -- Async gateways: `AsyncGroup` / `AsyncGateway` / `AsyncChannel` / - `open_gateway` on both async namespaces. -- Spec keys `profile=`, `transport=`, `stdin=`, `stdout=`, `stderr=`. -- The `execnet` console script and `python -m execnet`. - -**Removed** - -- `execnet.dump` / `load` / `loads`; `dumps` survives *only* as the - undocumented, non-warning xdist probe (`_XDIST_COMPAT`), scheduled for - deletion once xdist moves to `can_send`. -- `execnet.script.*` (`shell`, `quitserver`, `loop_socketserver`). -- The pre-Trio modules as real modules: `gateway`, `gateway_base`, - `multi`, `rsync`, `rsync_remote`, `xspec` now warn and forward, - removal in 3.0. `gateway_bootstrap`, `gateway_io`, `gateway_socket` - are simply gone. -- `execnet.portal`, and the `wait=` spec key — both existed only on this - branch. - -**Deprecated** - -- `set_execmodel`, `Group.execmodel`, `Group.remote_execmodel`, - `Group.set_execmodel`, the `execmodel=` spec key (permanent alias), - the `main_thread_only` profile, `Gateway.remote_init_threads`, - the `execnet-socketserver` script. - -**Behaviour changes that are API in practice**: worker stdio is no longer -swallowed; a blocking call inside a running event loop raises; a killed -worker is `EOFError` on every transport. - -## 3. What pins us to Trio - -Two different futures get conflated under "run on plain asyncio", and they -have different answers: - -- **(A) A non-Trio engine** — the host thread runs `asyncio.run`, or - execnet drops the hard `trio` dependency. This is an internals port - (`_trio_gateway` is the engine); the invariant that protects it is the - one already recorded: neutral `ByteStream`, sans-IO `FrameDecoder`. -- **(B) A native asyncio surface** — `execnet.aio` runs gateways on the - *caller's* loop with no host thread at all, symmetric with - `execnet.trio`. This one is visible in the public API today. - -Ranked by how much it would cost to undo after release: - -### 3.1 `execnet info` reports `"trio": ""` — HIGHEST - -`_provision.target_has_execnet()` decides whether a `python=` interpreter -can host a worker directly by asking whether `info["trio"]` is non-null. -That is a cross-version contract: a 2.2 coordinator will keep asking a 2.5 -worker that question forever, and the answer names our engine. - -**Recommend (before release)**: add a neutral key — `"worker": true` or -`"engines": ["trio"]` — have the coordinator prefer it and fall back to -`"trio"` only for a 2.2-vintage remote. Keep emitting `"trio"` -indefinitely; it costs one line and buys the freedom to answer honestly -from an engine that is not Trio. - -### 3.2 `execnet.trio` publishes the engine objects themselves — HIGH - -`execnet.trio.AsyncGateway` *is* `_trio_gateway.AsyncGateway`, so -`open_raw_channel`, `enqueue_frame` and `closed` are public API by -accident. They are the routing layer `_trio_host` and `_trio_worker` -drive; `test_trio_namespace_hides_raw_plumbing` already keeps `RawChannel` -and `ByteStream` out of `__all__`, but the methods that hand them out are -reachable on a documented class. - -That is also the shape that makes (A) expensive: an asyncio engine would -have to reproduce those exact methods to keep `execnet.trio` importable. - -**Recommend**: underscore-prefix `enqueue_frame` and `open_raw_channel` -(callers are `_trio_host`, `_trio_worker` and `testing/`, all ours), keep -`open_channel` public as the async `newchannel()`, and add a namespace -test that pins `trio.AsyncGateway`'s public method set. A facade like -`execnet.aio`'s would be cleaner still, but costs an allocation per -channel on the surface whose selling point is that it has none. - -### 3.3 `Host` and `default_host` on `execnet.aio` — MEDIUM - -Under (B) the asyncio surface has no host thread, so `aio.AsyncGroup(host=)` -and `aio.AsyncGroup.host` become meaningless — and `default_host` is -exported *only* from `aio`, which reads as an asyncio-specific concept when -it is the opposite. - -`host=` is honest today and can be deprecated later (a `None` default keeps -every caller working), so it stays. `default_host()` is different: nobody -needs it. Isolation is `Host()`, sharing is the default. - -**Recommend**: drop `default_host` from `execnet.aio.__all__` (keep the -name importable), and document `host=` as "which host thread serves this -group", i.e. as an implementation-shaped knob rather than part of the -asyncio model. - -### 3.4 `execnet.aio.trio` — LOW - -`aio.py` imports trio at module level for `trio.CancelScope`, so -`execnet.aio.trio` is the trio module. Harmless, but under (A) an -asyncio-only install must be able to `import execnet.aio`. - -**Recommend**: nothing now; note it as a tripwire for the engine port. - -### 3.5 `Host`'s own documentation — LOW - -The class is already engine-neutral in name and members (`name`, -`running`, `close`). Only its docstring says Trio. That is accurate -today and the right thing to write; it just means (A) is a docs change -here, not an API change. No action. - -## 4. Not about Trio, but found while looking - -1. **Deprecated names are in `__all__`.** `set_execmodel` is advertised - as supported API by `execnet.__all__` and `execnet.sync.__all__`. - Recommend removing from both (it stays importable and warning), and - updating `test_top_level_all_matches_sync_surface`, which compares the - two lists. -2. **`ExecModel` is still reachable** as `channel.gateway.execmodel`, and - hands out `RLock`/`Event`/`queue`/`subprocess`/`socket`. It exists - solely because xdist's remote builds its test queue on it. It is the - one place where a third party holds thread-shaped primitives of ours. - No action until xdist ports; delete with `_XDIST_COMPAT` in 3.0. -3. **`Gateway.remote_init_threads`** is a no-op that warns. Delete in - 3.0 with the shims. -4. **Async surfaces still lack** `remote_status()`, `MultiChannel`, group - iteration and `RSync` (carried over from the Phase C review — still a - deliberate gap, worth a line in the docs rather than silence). -5. **`Gateway.join`/`exit`** are public lifecycle methods that - `Group.terminate` supersedes. Harmless; leave. - -## 5. Suggested order - -1. `execnet info` neutral capability key — **before 2.2 ships**, because - it is the only item here that cannot be changed afterwards. -2. `set_execmodel` out of `__all__`; `default_host` out of `aio.__all__`. - One commit, two test updates. -3. Underscore the two engine methods on `AsyncGateway`, plus the pinning - test. -4. Everything else: 3.0, or never. diff --git a/handoff-windows-and-transports.md b/handoff-windows-and-transports.md deleted file mode 100644 index 751059cd..00000000 --- a/handoff-windows-and-transports.md +++ /dev/null @@ -1,190 +0,0 @@ -# Handoff: transports off stdio, the CLI, and Windows - -For a fresh session on branch `feat/trio-host-thread-io` (draft PR -pytest-dev/execnet#422). Continues `handoff-phase-c-worker-axes.md`, -which covers the async core and the worker profile axes and is still -accurate for those; nothing here contradicts it. - -Run checks with `uv run pytest testing/` and `uv run pre-commit run -a` -(never grep-filter pre-commit output). Local state at handoff: **552 -passed, 66 skipped**, pre-commit clean. - -## Read this first: CI was lying - -Until `1839800`, **CI had been executing zero tests since 2026-07-26**. -`testing/test_ssh_local.py` imported `asyncssh` at module level while -`tox.ini` carried its own hand-written `deps` list that had drifted from -the `testing` extra — so collection errored and every job reported -`2 skipped, 1 error` while looking like an ordinary failure. - -Two lessons worth keeping: - -- `tox.ini` now uses `extras = testing`; there must never be a second - list of test requirements. -- `[tool.uv] default-groups = ["testing"]` had been *replacing* uv's - `dev` default, so `uv sync` silently uninstalled pytest-xdist. The - `dev` group now absorbs `execnet[testing]` plus the tooling. - -Turning the suite back on is what surfaced almost everything below. -Assume any platform CI has not actually exercised is broken. - -## What landed this session - -| commit | what | -|---|---| -| `320c89e`, `17e7c7a` | run pytest-xdist's own suite against this execnet: a pinned `release` target that blocks, a floating `default-branch` target that may fail | -| `5710eed` | the `execnet` CLI — it is now the launch contract | -| `a84380f` | ssh dial-back over `ssh -R`, popen over a socketpair | -| `e43db63` | one typed endmarker sentinel | -| `1839800` | the CI fix above | -| `cd42ce9` | `EXECNET_PROVISION_WHEEL` | -| `9413a2e` … `f484d14` | Windows: eight commits, see below | - -## The CLI is the launch contract - -``` -execnet worker --protocol-stdio | --protocol-fd FD[,FD] - | --protocol-connect ADDR | --protocol-listen ADDR - | --protocol-share - --config JSON | --config-fd FD | --config-file PATH - --stdin/--stdout/--stderr DISPOSITION -execnet server [HOST:PORT] [--once] -execnet info -``` - -Everything that starts a worker emits these tokens; there is no second -launch path. `execnet info` (JSON: version, trio, executable, platform, -protocols) replaced the `import execnet, trio` probe, so provisioning -learns the remote version *before* connecting. - -**Config off argv**: `--config-fd 0` exists because the config carries -`env:` values and a remote argv is readable via `ps` by every user on -that host. ssh uses it. Do not regress this to `--config`. - -## Transports - -`transport=socket|stdio`. **`socket` is now the default for every worker -execnet spawns**, on both platforms (`2fc4013`). The worker's stdio is -then untouched, so remote `print()` reaches the coordinator. - -| gateway | handoff | notes | -|---|---|---| -| popen, POSIX | `pass_fds` + `--protocol-fd` | socketpair | -| popen, Windows | `socket.share()` + `--protocol-share` | see below | -| `socket=`/`installvia=` | same two, server-side | server accepts, then hands over | -| `ssh=`/`vagrant_ssh=` | `ssh -R` unix socket, worker dials back | POSIX only | - -ssh on Windows stays on stdio and **cannot** do otherwise: CPython has -never exposed `AF_UNIX` there (cpython#77589) and Win32-OpenSSH does not -implement `StreamLocal` forwarding. `resolve_transport` raises for an -impossible request rather than letting the gateway hang. - -### The share transport (Windows) - -`subprocess` refuses `pass_fds` on Windows. `socket.share(pid)` -(`WSADuplicateSocket`) duplicates the socket into a named pid instead — -but that needs the pid, which does not exist until the child is spawned. -Hence: **the flag goes in argv, the blob follows in the config on -stdin**. The blob is bound to that one pid, so it is inert to anything -else; this beats handle inheritance, which would need `close_fds=False` -and leak every inheritable handle to the child *and its grandchildren*. - -**The rule that took four CI rounds to learn: hand over a socket as a -socket, never as an fd.** Rebuilding one with `socket.socket(fileno=fd)` -makes the constructor re-derive family/type/proto by querying the handle, -and PyPy on Windows raises `WinError 10014` doing that to a handle from -`WSADuplicateSocket`. Both sites were wrong; both are fixed -(`5365105`, `f05f66d`). `adopt_socket` now takes either. - -No in-process probe can catch this: `share()` and `fromshare()` both work -in-process on PyPy. What cracked it was noticing which test *passed* — -`popen//transport=socket` was green while the server path failed, which -isolated the difference to one line. - -## Windows, which had never been tested - -Every Windows worker died at startup on `trio.lowlevel.FdStream`, which -is POSIX-only. Trio has Windows pipe streams but they need OVERLAPPED -handles registered with an IOCP, and inherited stdio is an ordinary -synchronous pipe — so `ThreadedFdStream` (`_trio_gateway.py`) does those -reads and writes in the thread pool. It is now only reachable via -explicit `transport=stdio`. - -Latent bugs that Linux was hiding, all found once Windows ran: - -- **`execnet server :0` reported a port nothing listened on.** A - wildcard bind with an ephemeral port gives *each* address family its - own random port (trio documents this) and only the first was reported. - Which family comes first is platform-dependent — IPv4 on Linux, IPv6 - on Windows. `_socketserver._one_port` re-binds them to one port. -- **`test_basics` wrote generated source in the locale encoding.** - Python reads source as UTF-8 (PEP 3120); one em-dash in `_message` was - enough to break it off UTF-8 locales. -- **A killed worker reported `BrokenResourceError`, not `EOFError`**, on - a socket: a dead peer *resets* a socket where a pipe reaches EOF. The - reader maps it (`f484d14`). Applies to `socket=` on POSIX too. -- **`test__rinfo` raced**: `receive()` returns when the *send* arrives, - and the `os.chdir('..')` after it had not run yet. - -## Failure modes to preserve - -These were all real, and each cost a debugging round: - -- **A socket worker that cannot be spawned must not hang the - coordinator.** It is spawned by the *server*, so the exception dies - there while the coordinator waits for a handshake byte. A host that - cannot hand a socket over refuses *before replying with an address* — - the last moment a reason can reach the coordinator — and a spawn that - fails anyway closes the connection so the wait ends. -- **A failed socket gateway must not kill the gateway it was requested - through.** It runs as a task on that coordinator's host; letting it - propagate cost the coordinator too, which is how one unsupported - gateway became 51 errors. -- **`_check_event_loop` runs before the channel-state check** in - `send`/`receive`. Both are caller bugs, but which one you were told - about depended on whether the peer had closed yet. - -## Naming - -`master` → `coordinator` throughout for the `via=`/`installvia=` gateway -(`3a8f182`): it spawns and relays for the sub-worker, which is what a -coordinator does. Where one sentence covers both parties, only the -relaying one is named; the requesting side is "here"/"us". - -## Open work - -1. ~~**`doc/basics.rst` still documents `set_execmodel` / - `main_thread_only`**~~ — DONE (`98eb08d`). `basics.rst` and - `implnotes.rst` rewritten, new `doc/api.rst` namespace reference, - `tox -e docs` now builds with `-W` *and* runs the doc examples as - doctests, from a new CI job. Those examples had not been collectable - at all since pytest 7 (a `pytest_plugins` line in a non-top-level - conftest), which is why so much of them had rotted. The public-surface - review that came out of the same pass is - `handoff-public-api-review.md`; **its recommendations are not - applied** — the `execnet info` one wants deciding before 2.2 ships. -2. **`installvia` still needs a socket handoff at all.** It would not, - if the server spawned the worker as the *listener* - (`--protocol-listen 127.0.0.1:0`) and reported its address back: no - `pass_fds`, no `share()`, works on any interpreter, and the spawn - happens *before* the reply so failures are diagnosable by - construction. The open question is how the server learns the port - (worker prints it, or writes it to a path given in the config). - Standalone `execnet server` still needs the handoff — it has already - accepted the connection. -3. **A trampoline process was considered and rejected** — see the - analysis: the current design already frees the worker's stdio - in-process (`_dup_protocol_fds` + `apply_stdio`), and a pump's own - stdio side is still a blocking pipe, so it relocates the thread - rather than removing it, at the cost of a process and two copies per - message. Do not revisit without a new reason. -4. `execnet.anyio` remains deferred (Phase E). - -## Invariants (do not regress) - -- No source shipping, ever. Workers import installed execnet+trio. -- The worker config never travels in a remote argv. -- `import execnet` must not import the trio event loop. -- Hand sockets over as sockets, not fds. -- Keep async-core idioms anyio-portable (neutral `ByteStream`, sans-IO - `FrameDecoder`). diff --git a/src/execnet/_execmodel.py b/src/execnet/_execmodel.py index 7c25448c..c4af3af0 100644 --- a/src/execnet/_execmodel.py +++ b/src/execnet/_execmodel.py @@ -21,10 +21,11 @@ class ExecModel: The machinery behind execution models was retired: protocol IO always runs on the Trio host and blocking waits go through the boundary kit's - wakeners (``execnet._boundary``); the name maps onto the worker config - axes (``loop=`` / ``exec=`` / ``wait=``). The stdlib-delegating - members stay for API compatibility (pytest-xdist builds its test queue - on ``execmodel.RLock``/``Event``) -- every preset is thread-shaped. + wakeners (``execnet._boundary``). What the name selected survives as + the ``profile=`` spec key (:data:`WORKER_PROFILES`). The + stdlib-delegating members stay for API compatibility (pytest-xdist + builds its test queue on ``execmodel.RLock``/``Event``) -- every preset + is thread-shaped. """ def __init__(self, backend: str) -> None: diff --git a/src/execnet/_shim.py b/src/execnet/_shim.py index 0839c2e3..93ebe0b1 100644 --- a/src/execnet/_shim.py +++ b/src/execnet/_shim.py @@ -17,8 +17,10 @@ import warnings from typing import Any -#: shims are scheduled for removal in this release -REMOVED_IN = "execnet 3.0" +#: shims are scheduled for removal in this release -- later in the 3.x +#: series, once the consumers that still import these names (pytest-xdist +#: above all) have released a version that does not. +REMOVED_IN = "a later execnet 3.x release" def forwarder(shim: str, moved: dict[str, str]) -> Any: From 7a7936a741ffff612508fc058ed5bd47e72a6d78 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 14:44:29 +0200 Subject: [PATCH 88/91] fix: closing a host breaks what it served, instead of half-working A host is the loop a gateway's protocol IO runs on, so closing one ends every connection it carried. Two paths did not say so. `Host.close()` left the object reusable, so the next `makegateway()` on a group whose host went away started a *second* loop thread -- which none of that group's gateways are attached to -- and then failed on the stale FacadeAsyncGroup with " is not entered", leaking a thread per attempt. Closing is final now, and `_ensure_started` says which host died and why the group cannot be reused. `setcallback()` after the loop stopped went through `run_on_loop`'s inline fallback, which is not valid for the consumer switch: it drained the mailbox and set `_has_consumer` before reaching `start_soon`'s guard, so the buffered items were lost, `receive()` answered "channel has receiver callback", and `waitclose()` waited out its timeout for a consumer task that could never exist. It now refuses before touching the channel. Co-Authored-By: Claude Opus 5 (1M context) --- doc/basics.rst | 7 ++- src/execnet/_host.py | 20 ++++++- src/execnet/_trio_host.py | 10 ++++ testing/test_host.py | 106 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 140 insertions(+), 3 deletions(-) diff --git a/doc/basics.rst b/doc/basics.rst index 1d1f94dd..38364473 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -368,7 +368,12 @@ you pass when you want an isolated loop with deterministic teardown:: # the thread is joined here, rather than at interpreter exit Gateways served by a host must be terminated before it closes; closing does -not terminate them for you. :mod:`execnet.trio` uses no host at all. +not terminate them for you -- it *breaks* them. Their protocol IO no longer +has a loop to run on, so their channels reach EOF, sending raises, and the +groups they belong to refuse to make new gateways. Closing is final: a host +cannot be reopened, and a group whose host went away needs a new host and a +new group rather than quietly getting a second loop thread that none of its +gateways are attached to. :mod:`execnet.trio` uses no host at all. The execnet command line diff --git a/src/execnet/_host.py b/src/execnet/_host.py index e57511d8..da4a1be2 100644 --- a/src/execnet/_host.py +++ b/src/execnet/_host.py @@ -113,9 +113,13 @@ def __init__( self.callback_threads = callback_threads self._lock = threading.Lock() self._trio_host: Any = None + self._closed = False def __repr__(self) -> str: - state = "running" if self._trio_host is not None else "idle" + if self._trio_host is not None: + state = "running" + else: + state = "closed" if self._closed else "idle" return f"" @property @@ -125,6 +129,13 @@ def running(self) -> bool: def _ensure_started(self) -> Any: """The started :class:`~execnet._trio_host.TrioHost` (internal).""" with self._lock: + if self._closed: + raise RuntimeError( + f"{self!r} was closed: the loop thread is gone, and with it" + " every gateway and channel it served. Closing is final --" + " build a new Host (and a new Group on it) instead of" + " reusing this one." + ) if self._trio_host is None: from . import _trio_host @@ -139,10 +150,15 @@ def close(self, timeout: float | None = 5.0) -> None: """Stop the loop and join the thread (no-op when not running). Gateways served by this host must already be terminated; closing - does not terminate them for you. + does not terminate them for you -- it *breaks* them, along with + their groups and channels: their protocol IO no longer has a loop + to run on. Closing is final, so a group whose host went away + fails loudly instead of quietly resurrecting a second loop thread + that none of its gateways are attached to. """ with self._lock: trio_host, self._trio_host = self._trio_host, None + self._closed = True if trio_host is not None: trio_host.stop(timeout=timeout) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index c18637ee..84882c13 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -345,6 +345,16 @@ def attach_consumer( """ def switch() -> None: + if not self.host.is_host_thread(): + # run_on_loop fell back to running us inline: the loop is + # gone, so no consumer task can ever drain this channel. + # Fail before touching the channel -- a half-switched channel + # loses its buffered items, refuses receive(), and leaves + # waitclose() waiting for a consumer that will never run. + raise OSError( + f"cannot set callback on {channel!r}: the host loop has" + " stopped, so nothing can deliver to it" + ) mailbox = channel._mailbox if mailbox is None: raise OSError(f"{channel!r} has callback already registered") diff --git a/testing/test_host.py b/testing/test_host.py index f5678a31..e5ebab58 100644 --- a/testing/test_host.py +++ b/testing/test_host.py @@ -105,6 +105,112 @@ def test_forked_child_gets_a_fresh_default_host(self) -> None: assert result == b"\x00" +class TestHostDestruction: + """Closing a host breaks what it served -- loudly, and without hanging. + + A gateway's protocol IO lives on the host loop, so stopping that loop + is not a resource being freed underneath a working object: it ends the + connection. Every operation that needs the loop must say so at the + call site rather than hang, deliver nothing silently, or quietly start + a second loop thread that none of the existing gateways are on. + """ + + def test_close_breaks_the_channels_it_served(self) -> None: + host = Host(name="execnet-host-broken-channel") + group = execnet.Group(host=host) + gateway = group.makegateway("popen") + channel = gateway.remote_exec("while 1: channel.send(channel.receive() + 1)") + channel.send(1) + assert channel.receive(TESTTIMEOUT) == 2 + + host.close() + + assert not gateway.hasreceiver() + with pytest.raises(EOFError): + channel.receive(TESTTIMEOUT) + with pytest.raises(OSError): + channel.send(3) + # closed for receiving, so this returns instead of timing out + channel.waitclose(TESTTIMEOUT) + group.terminate(timeout=5.0) + + def test_close_breaks_the_gateways_it_served(self) -> None: + host = Host(name="execnet-host-broken-gateway") + group = execnet.Group(host=host) + gateway = group.makegateway("popen") + assert gateway.remote_exec("channel.send(1)").receive(TESTTIMEOUT) == 1 + + host.close() + + with pytest.raises(OSError): + gateway.newchannel() + with pytest.raises(OSError): + gateway.remote_exec("channel.send(1)") + # the receiver is finished, so this must not block + gateway.join(TESTTIMEOUT) + group.terminate(timeout=5.0) + + def test_close_breaks_the_group_and_starts_no_second_loop(self) -> None: + host = Host(name="execnet-host-broken-group") + group = execnet.Group(host=host) + group.makegateway("popen") + + host.close() + + assert not host.running + with pytest.raises(RuntimeError, match="was closed"): + group.makegateway("popen") + # the failed attempt must not have resurrected a loop thread: the + # group's existing gateways could never be attached to it + assert not host.running + assert "execnet-host-broken-group" not in host_thread_names() + # cleaning up a broken group still returns + group.terminate(timeout=5.0) + + def test_closing_is_final_even_for_an_unused_host(self) -> None: + host = Host(name="execnet-host-unused") + host.close() + with pytest.raises(RuntimeError, match="was closed"): + execnet.Group(host=host).makegateway("popen") + + def test_aio_group_on_a_closed_host_raises(self) -> None: + host = Host(name="execnet-host-closed-aio") + host.close() + + async def main() -> None: + with pytest.raises(RuntimeError, match="was closed"): + await execnet.aio.AsyncGroup(host=host).start() + + asyncio.run(main()) + + def test_setcallback_after_close_fails_without_wedging_the_channel(self) -> None: + # the consumer task runs on the host loop, so with the loop gone + # there is nothing to attach to -- but the failure must land on the + # caller, not on the channel: a half-switched channel drops what it + # had buffered, refuses receive(), and makes waitclose() wait for a + # consumer that will never run + host = Host(name="execnet-host-late-callback") + group = execnet.Group(host=host) + gateway = group.makegateway("popen") + channel = gateway.remote_exec("channel.send(1); channel.send(2)") + assert channel.receive(TESTTIMEOUT) == 1 + # everything the worker sent has arrived and is buffered by now + channel.waitclose(TESTTIMEOUT) + + host.close() + + received: list[object] = [] + with pytest.raises(OSError, match="host loop"): + channel.setcallback(received.append, endmarker="END") + assert received == [] + # untouched: the buffered item is still there, then EOF + assert channel.receive(TESTTIMEOUT) == 2 + with pytest.raises(EOFError): + channel.receive(TESTTIMEOUT) + channel.waitclose(TESTTIMEOUT) + group.terminate(timeout=5.0) + + class TestEventLoopGuard: """Blocking on the host from inside a running loop must not hang.""" From 9c48594fe1e786681d5dc4e781d5e1f5240106ff Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 18:56:54 +0200 Subject: [PATCH 89/91] fix: the host boundary fails loudly, instead of hanging or crashing the loop Three ways a facade could reach a host that cannot serve it, none of which ended in an error the caller could act on. **A posted callback that raises took the whole loop with it.** Trio turns an exception from an entry-queue callback into a TrioInternalError and tears the run down, so `TrioHost.start_soon`'s guard clause -- reached from inside a `portal.post` by `call_pending` and by the aio bridge whenever a call lost its race with shutdown -- killed every gateway in the process and told the user to file a trio bug. Both spawns now report through the OneShot/future they already resolve every other failure through, and "nothing posted through the portal may raise" is written down as an invariant. **Nothing survives os.fork(), but everything waited as if it might.** The host's loop thread is not duplicated into the child and the worker connections belong to the parent -- yet the parent loop's trio token still *accepts* work in the child, so `run_sync_soon` queued callbacks nobody would ever run and `from_thread.run` waited for a reply nobody would send. A child using an inherited group, gateway or channel (the module-level `execnet.makegateway` among them) blocked forever. LoopPortal and BaseGateway now compare pids and raise ForkedResourceError, an OSError subclass so `__del__` and `except OSError` cleanup keep working, distinct so `_send` can let the real reason through instead of rewriting it as "cannot send (already closed?)". Recovery is the child's to make explicitly: ask for the default host and you get a fresh one to build new groups on. A child no longer runs the parent's atexit cleanup either -- those gateways are not its to terminate. **Group.terminate() and Gateway.join() joined the event-loop guard.** Both block on the host with no bound worth waiting out, which is exactly the stall the guard turns into an error naming `execnet.aio`. An empty group has nothing to block on, so cleaning one up from inside a loop stays fine. Also drops what the IO object no longer does: `wait`/`kill` had no caller left in the tree or in xdist (the async group owns the process handle), and with them goes `SyncBridgeGateway.host_call`. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 19 +++ HANDOFF.md | 17 ++- doc/basics.rst | 7 + src/execnet/_channel.py | 13 +- src/execnet/_errors.py | 30 +++++ src/execnet/_gateway_base.py | 22 +++- src/execnet/_host.py | 31 ++++- src/execnet/_message.py | 12 +- src/execnet/_multi.py | 11 ++ src/execnet/_portal.py | 25 +++- src/execnet/_trio_host.py | 75 +++-------- src/execnet/_trio_worker.py | 6 - src/execnet/aio.py | 12 +- testing/test_basics.py | 2 +- testing/test_host.py | 241 +++++++++++++++++++++++++++++++---- 15 files changed, 412 insertions(+), 111 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ecd1beaa..8e0608f7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -154,6 +154,25 @@ series, once the consumers that need them have released without them. Pass ``execnet.Host()`` as ``Group(host=...)`` (or ``AsyncGroup(host=...)``) for an isolated loop with deterministic teardown -- ``Host`` is a context manager and joins its thread on exit, where the shared one stops at interpreter exit. + + Closing a host is final, and it *breaks* the groups, gateways and channels it + served rather than freeing a resource underneath them: their protocol IO has no + loop to run on any more, so channels reach EOF, sends raise, and the group refuses + to build new gateways instead of quietly starting a second loop thread that none of + its existing gateways are attached to. +* execnet objects do not survive ``os.fork()``, and now say so instead of blocking. + The host's loop thread is not duplicated into the child and the worker connections + belong to the parent, but the parent loop's trio token still *accepts* work in the + child -- so a forked child using an inherited group, gateway or channel (including + the module-level ``execnet.makegateway``) used to wait forever for a reply nobody + would send. Every route to the host now checks which process it is in and raises, + naming the fork. Recovery is explicit and belongs to the child: build a new + ``Host`` and a new ``Group`` on it. A child that asks for the default host gets a + fresh one, and it no longer inherits the parent's atexit cleanup. +* ``Group.terminate()`` and ``Gateway.join()`` join the calls that refuse to run + inside a running asyncio or trio loop. Both block on the host with no useful + bound -- ``join()`` until the worker dies -- which is the stall the guard exists to + turn into an error. Terminating a group with nothing in it stays allowed. * A spec's ``profile=``/``execmodel=`` value is no longer rewritten in place when it names a deprecated profile. pytest-xdist reuses one ``XSpec`` across gateways and re-reads ``spec.execmodel`` to decide whether it still needs prefixing, so normalizing diff --git a/HANDOFF.md b/HANDOFF.md index 796a499d..624de186 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -152,6 +152,17 @@ the coordinator's shape does not dictate the worker's. `portal.run` (KI-deferred) is only for management ops. - A killed worker is `EOFError` on every transport — a dead peer *resets* a socket where a pipe reaches EOF, and the reader maps that. +- **Nothing posted through the portal may raise.** Trio turns an exception + from an entry-queue callback into `TrioInternalError` and tears the whole + run down, so one call losing a race with shutdown takes every gateway in + the process with it — and tells the user to file a trio bug. A posted + callback reports through its `OneShot`/future instead. +- A host that goes away breaks what it served, loudly. `Host.close()` is + final (no second loop thread the existing gateways are not on), and + nothing survives `os.fork()`: the parent loop's token still *accepts* + work in a child, so `LoopPortal` and `BaseGateway._check_usable` compare + pids and raise `ForkedResourceError` rather than let the child wait for a + reply nobody will send. Recovery after a fork is the child's, explicitly. **Launch and provisioning** @@ -181,9 +192,9 @@ the coordinator's shape does not dictate the worker's. through. It runs as a task on that coordinator's host; letting it propagate cost the coordinator too, which is how one unsupported gateway became 51 errors. -- `_check_event_loop` runs *before* the channel-state check in - `send`/`receive`. Both are caller bugs, but which one you were told - about used to depend on whether the peer had closed yet. +- `_check_usable` (fork, then event loop) runs *before* the channel-state + check in `send`/`receive`. All of them are caller bugs, but which one you + were told about used to depend on whether the peer had closed yet. - Anything that warns in a *worker* can livelock a pytest run: a warning raised inside pytest's warning-recording hook records a warning. The `execnet.dumps` shim warns once per process for exactly this reason. diff --git a/doc/basics.rst b/doc/basics.rst index 38364473..0552bfeb 100644 --- a/doc/basics.rst +++ b/doc/basics.rst @@ -375,6 +375,13 @@ cannot be reopened, and a group whose host went away needs a new host and a new group rather than quietly getting a second loop thread that none of its gateways are attached to. :mod:`execnet.trio` uses no host at all. +``os.fork()`` is the same situation arriving by surprise: the loop thread is +not duplicated into the child and the worker connections belong to the +parent, so every group, gateway and channel the child inherits is dead there +and raises rather than waiting on a loop that will never run again. A child +that wants gateways of its own builds a new group -- and gets a fresh host +with it. + The execnet command line ==================================================================== diff --git a/src/execnet/_channel.py b/src/execnet/_channel.py index 0745831b..4ac30c3c 100644 --- a/src/execnet/_channel.py +++ b/src/execnet/_channel.py @@ -306,7 +306,7 @@ def waitclose(self, timeout: float | None = None) -> None: # For a callback channel wait on the consumer task finishing (so every # callback, including the endmarker, has run); otherwise wait for the # non-"opened" state directly. - self.gateway._check_event_loop("channel.waitclose()") + self.gateway._check_usable("channel.waitclose()") signal = ( self._consumer_done if self._consumer_done is not None @@ -328,10 +328,11 @@ def send(self, item: object) -> None: OSError is raised if the write pipe was prematurely closed. """ - # before the state check: calling a blocking API from inside an event - # loop is a bug in the caller either way, and whether the channel has - # closed yet is a race -- the diagnostic should not depend on it - self.gateway._check_event_loop("channel.send()") + # before the state check: an unusable gateway (inherited by a fork, + # or driven from inside an event loop) is a caller bug either way, + # and whether the channel has closed yet is a race -- the diagnostic + # should not depend on it + self.gateway._check_usable("channel.send()") if self.isclosed(): raise OSError(f"cannot send to {self!r}") self.gateway._send(Message.CHANNEL_DATA, self.id, dumps_internal(item)) @@ -347,7 +348,7 @@ def receive(self, timeout: float | None = None) -> Any: reraised as channel.RemoteError exceptions containing a textual representation of the remote traceback. """ - self.gateway._check_event_loop("channel.receive()") + self.gateway._check_usable("channel.receive()") mailbox = self._mailbox if mailbox is None: raise OSError("cannot receive(), channel has receiver callback") diff --git a/src/execnet/_errors.py b/src/execnet/_errors.py index 276fbb5e..b8763f4d 100644 --- a/src/execnet/_errors.py +++ b/src/execnet/_errors.py @@ -27,6 +27,36 @@ class HostNotFound(ConnectionError): """The remote side of a gateway could not be reached.""" +class ForkedResourceError(OSError): + """An execnet object was inherited by ``os.fork()`` and is dead here. + + Nothing execnet builds survives a fork: the host's loop thread is not + duplicated into the child, and the worker connections belong to the + parent that opened them. Rather than let the child block forever on a + loop that will never run again, every route to the host checks which + process it is in and raises this. + + An ``OSError`` because that is what execnet already means by "this + connection is gone" -- ``__del__`` paths and ``except OSError`` cleanup + keep working -- but a distinct type, so the paths that would otherwise + rewrite it as "cannot send (already closed?)" can let the real reason + through. + + Recovery is explicit and belongs to the child: build a new + :class:`~execnet.Host` and a new ``Group`` on it. + """ + + +def forked_error(what: str, origin_pid: int) -> ForkedResourceError: + """The :class:`ForkedResourceError` for using ``what`` after a fork.""" + return ForkedResourceError( + f"{what} belongs to pid {origin_pid} and this is pid {os.getpid()}:" + " execnet objects do not survive os.fork() -- the host's loop thread" + " is not duplicated into the child, and the worker connections stay" + " with the parent. Build a new Host and a new Group in the child." + ) + + def geterrortext( exc: BaseException, format_exception=traceback.format_exception, diff --git a/src/execnet/_gateway_base.py b/src/execnet/_gateway_base.py index 773ba314..16c4d5fa 100644 --- a/src/execnet/_gateway_base.py +++ b/src/execnet/_gateway_base.py @@ -29,6 +29,7 @@ from ._channel import ChannelFactory from ._channel import Endmarker from ._errors import INTERRUPT_TEXT +from ._errors import ForkedResourceError from ._errors import geterrortext from ._errors import sysex from ._message import IO @@ -51,7 +52,19 @@ class BaseGateway: #: run its own loop and talk to its channel from inside it. _guard_event_loop = False - def _check_event_loop(self, what: str) -> None: + def _check_usable(self, what: str) -> None: + """Refuse ``what`` when this gateway cannot possibly serve it. + + Two caller bugs, checked before anything else (in particular before + the channel-state check, so which one you are told about does not + depend on whether the peer has closed yet): using a gateway that a + fork left behind in another process, and blocking a running event + loop's own thread. + """ + if self._pid != os.getpid(): + from ._errors import forked_error + + raise forked_error(what, self._pid) if self._guard_event_loop: from ._host import check_not_in_event_loop @@ -61,6 +74,8 @@ def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel self._io = io self.id = id + #: pid this gateway's connection (and its host loop) belongs to + self._pid = os.getpid() self._channelfactory = ChannelFactory(self, _startcount) # globals may be NONE at process-termination self.__trace = trace @@ -132,6 +147,10 @@ def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: try: session.enqueue_message(message) self._trace("sent", message) + except ForkedResourceError: + # "already closed?" would be a guess, and a wrong one + self._trace("failed to send", message, "(inherited by a fork)") + raise except (OSError, ValueError) as e: self._trace("failed to send", message, e) raise OSError("cannot send (already closed?)") from e @@ -171,6 +190,7 @@ def newchannel(self) -> Channel: def join(self, timeout: float | None = None) -> None: """Wait for the receiver (Trio session) to terminate.""" + self._check_usable("gateway.join()") self._trace("waiting for receiver to finish") session = self._trio_session if session is not None: diff --git a/src/execnet/_host.py b/src/execnet/_host.py index da4a1be2..d7d356e7 100644 --- a/src/execnet/_host.py +++ b/src/execnet/_host.py @@ -30,6 +30,8 @@ from typing import TYPE_CHECKING from typing import Any +from ._errors import forked_error + if TYPE_CHECKING: from typing_extensions import Self @@ -114,17 +116,21 @@ def __init__( self._lock = threading.Lock() self._trio_host: Any = None self._closed = False + #: pid the loop thread was started in; a fork does not copy it + self._pid: int | None = None def __repr__(self) -> str: - if self._trio_host is not None: - state = "running" - else: - state = "closed" if self._closed else "idle" - return f"" + return f"" + + def _state(self) -> str: + if self._trio_host is None: + return "closed" if self._closed else "idle" + return "running" if self._pid == os.getpid() else "inherited" @property def running(self) -> bool: - return self._trio_host is not None + """Whether this host has a loop thread *in this process*.""" + return self._state() == "running" def _ensure_started(self) -> Any: """The started :class:`~execnet._trio_host.TrioHost` (internal).""" @@ -136,6 +142,11 @@ def _ensure_started(self) -> Any: " build a new Host (and a new Group on it) instead of" " reusing this one." ) + if self._trio_host is not None and self._pid != os.getpid(): + # Recovery after a fork is the child's to make explicitly: + # silently starting a second loop here would hand back a Host + # that none of the inherited gateways are attached to. + raise forked_error(f"{self!r}", self._pid) # type: ignore[arg-type] if self._trio_host is None: from . import _trio_host @@ -143,6 +154,7 @@ def _ensure_started(self) -> Any: name=self.name, callback_threads=self.callback_threads ) trio_host.start() + self._pid = os.getpid() self._trio_host = trio_host return self._trio_host @@ -184,7 +196,12 @@ def default_host() -> Host: """The process-wide host, started lazily and stopped at interpreter exit. After ``os.fork()`` the child inherits a Host whose thread does not - exist there, so the first use in a child builds a fresh one. + exist there, so a child asking for the default host gets a fresh one + and can build new groups on it. What it does *not* get is the + inherited one working again: everything already attached to that host + -- the pre-fork groups, gateways and channels, including the + module-level ``execnet.makegateway`` group -- stays dead in the child + and says so (:class:`~execnet._errors.ForkedResourceError`). """ global _default, _default_pid with _default_lock: diff --git a/src/execnet/_message.py b/src/execnet/_message.py index 4d31339f..fb213304 100644 --- a/src/execnet/_message.py +++ b/src/execnet/_message.py @@ -28,6 +28,14 @@ def read(self, numbytes: int, /) -> bytes: ... class IO(Protocol): + """What a gateway still needs from the object it was built around. + + Reading and writing moved to the Trio session long ago; what is left is + the write-side close behind ``Gateway.exit``. Waiting for and killing a + worker process belongs to whoever holds the process handle -- the async + group -- not here. + """ + execmodel: ExecModel def read(self, numbytes: int, /) -> bytes: ... @@ -38,10 +46,6 @@ def close_read(self) -> None: ... def close_write(self) -> None: ... - def wait(self) -> int | None: ... - - def kill(self) -> None: ... - class Message: """Encapsulates Messages and their wire protocol. diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py index ac0b86d1..5fa90518 100644 --- a/src/execnet/_multi.py +++ b/src/execnet/_multi.py @@ -7,6 +7,7 @@ from __future__ import annotations import atexit +import os import queue import threading import time @@ -83,6 +84,8 @@ def __init__( self._autoidcounter = 0 self._autoidlock = Lock() self._gateways_to_join: list[Gateway] = [] + #: pid this group belongs to; a fork does not carry its gateways over + self._pid = os.getpid() self._host = default_host() if host is None else host self._async_group: Any = None self.set_profile("thread" if profile is None else profile) @@ -268,6 +271,10 @@ def _unregister(self, gateway: Gateway) -> None: def _cleanup_atexit(self) -> None: # The host is shared and stops itself at exit; a group only owns # its gateways and the async group task running on that host. + if self._pid != os.getpid(): + # a forked child inherited this registration along with a group + # whose gateways are the parent's to terminate, not ours + return trace(f"=== atexit cleanup {self!r} ===") self.terminate(timeout=1.0) if self._async_group is not None: @@ -285,6 +292,10 @@ def terminate(self, timeout: float | None = None) -> None: Timeout defaults to None meaning open-ended waiting and no kill attempts. """ + if self or self._gateways_to_join: + # blocks on the host (termination grace, then joins), so it has + # the same event-loop problem as makegateway() and receive() + check_not_in_event_loop("Group.terminate()") while self or self._gateways_to_join: vias: set[str] = set() for gw in self: diff --git a/src/execnet/_portal.py b/src/execnet/_portal.py index 6c1253c4..3db8413d 100644 --- a/src/execnet/_portal.py +++ b/src/execnet/_portal.py @@ -20,6 +20,7 @@ from __future__ import annotations +import os from collections.abc import Awaitable from collections.abc import Callable from typing import Any @@ -31,6 +32,7 @@ from ._boundary import OneShot from ._boundary import ThreadWakener from ._boundary import Wakener +from ._errors import forked_error __all__ = ["LoopPortal", "Mailbox", "OneShot", "ThreadWakener", "Wakener"] @@ -46,6 +48,7 @@ class LoopPortal: def __init__(self) -> None: self._token = trio.lowlevel.current_trio_token() + self._pid = os.getpid() def is_loop_thread(self) -> bool: """Whether the calling thread is running this portal's loop.""" @@ -54,12 +57,26 @@ def is_loop_thread(self) -> bool: except RuntimeError: return False + def _check_process(self) -> None: + """Refuse a loop that lives in another process (see fork, below). + + The token of a forked parent's loop still *works* in the child -- + ``run_sync_soon`` happily queues a callback that nothing will ever + run, and ``from_thread.run`` waits for a reply forever. This is the + one choke point every route to the loop goes through, so the check + sits here rather than on each of them. + """ + if self._pid != os.getpid(): + raise forked_error("the execnet host loop", self._pid) + def run(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: """Run ``await async_fn(*args)`` on the loop, blocking this thread.""" + self._check_process() return trio.from_thread.run(async_fn, *args, trio_token=self._token) def run_sync(self, sync_fn: Callable[..., T], *args: Any) -> T: """Run ``sync_fn(*args)`` on the loop, blocking this thread.""" + self._check_process() return trio.from_thread.run_sync(sync_fn, *args, trio_token=self._token) def post(self, sync_fn: Callable[..., object], *args: Any) -> None: @@ -67,6 +84,12 @@ def post(self, sync_fn: Callable[..., object], *args: Any) -> None: Thread-safe and callable from the loop thread itself; all posts run in strict FIFO order (``TrioToken.run_sync_soon``). Raises - ``trio.RunFinishedError`` once the loop has shut down. + ``trio.RunFinishedError`` once the loop has shut down, and + :class:`~execnet._errors.ForkedResourceError` in a forked child. + + ``sync_fn`` must not raise: trio turns an exception from an + entry-queue callback into ``TrioInternalError`` and tears the whole + loop down, taking every gateway in the process with it. """ + self._check_process() self._token.run_sync_soon(sync_fn, *args) diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 84882c13..81069ec8 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -95,7 +95,13 @@ async def adopt_socket(sock: int | Any) -> trio.SocketStream: class SyncIOHandle: - """Sync IO facade for Gateway.exit close_write and terminate wait/kill.""" + """What is left of the sync IO object a ``Gateway`` is built around. + + The Message IO itself belongs to the session, so the gateway's ``_io`` + is down to one live duty: ``Gateway.exit`` closing the write side. + Waiting for and killing the worker process is the async group's + (``AsyncGroup._terminate_one``), which is where the process handle is. + """ remoteaddress: str @@ -104,12 +110,10 @@ def __init__( execmodel: ExecModel, session: SyncBridgeGateway, *, - process: trio.Process | None = None, remoteaddress: str | None = None, ) -> None: self.execmodel = execmodel self._session = session - self._process = process if remoteaddress is not None: self.remoteaddress = remoteaddress @@ -125,36 +129,6 @@ def close_read(self) -> None: def close_write(self) -> None: self._session.request_close_write() - def wait(self) -> int | None: - process = self._process - if process is None: - return None - - async def _wait() -> int | None: - # Always await wait() so the child is reaped (no zombies). - code: int | None = await process.wait() - return code - - try: - return self._session.host_call(_wait) - except Exception: - return process.returncode - - def kill(self) -> None: - process = self._process - if process is None: - return - - async def _kill() -> None: - with trio.move_on_after(5): - process.kill() - await process.wait() - - try: - self._session.host_call(_kill) - except Exception as exc: - trace("ERROR killing trio process:", exc) - class SyncBridgeGateway(AsyncGateway): """Async engine serving a sync ``BaseGateway``. @@ -463,21 +437,6 @@ def _consumer_failed(self, channel: Any, exc: BaseException) -> None: ) channel._close_from_remote(RemoteError(errortext), sendonly=False) - def host_call(self, async_fn: Callable[..., Awaitable[T]], *args: Any) -> T: - """Blocking host call that parks correctly for the wait backend. - - ``wait=thread`` keeps the KI-deferred ``portal.run`` path; other - backends (gevent) wait on a OneShot with the gateway's wakener so - only the calling greenlet parks, not the whole hub. - """ - gateway = self.sync_gateway - if gateway._wait_backend == "thread": - return self.host.call(async_fn, *args) - pending = self.host.call_pending( - async_fn, *args, wakener=gateway._new_wakener() - ) - return pending.wait() - def run_on_loop(self, sync_fn: Callable[[], T]) -> T: """Run ``sync_fn`` on the host loop, excluding dispatch interleaving. @@ -655,7 +614,18 @@ async def runner() -> None: result.set(value) def spawn() -> None: - self.start_soon(runner) + # Posted callbacks must not raise: trio turns an exception from + # an entry-queue callback into TrioInternalError and tears the + # whole loop down, taking every gateway in the process with it. + # A host that shut down between the post and here is exactly the + # failure this call already reports as a value. + try: + self.start_soon(runner) + except BaseException as exc: + error = RuntimeError("trio host was shut down") + error.__cause__ = exc + if not result.is_set(): + result.set_error(error) self.portal.post(spawn) return result @@ -718,12 +688,6 @@ def close_read(self) -> None: def close_write(self) -> None: return - def wait(self) -> int | None: - return None - - def kill(self) -> None: - return - class FacadeAsyncGroup(AsyncGroup): """AsyncGroup owning the async side of a sync ``Group``. @@ -805,7 +769,6 @@ def makegateway_trio(group: Group, spec: Any) -> Gateway: gw._io = SyncIOHandle( get_execmodel(spec.profile), bridge, - process=async_group._processes.get(bridge), remoteaddress=bridge.remoteaddress, ) return gw diff --git a/src/execnet/_trio_worker.py b/src/execnet/_trio_worker.py index 47d7a008..14ff0879 100644 --- a/src/execnet/_trio_worker.py +++ b/src/execnet/_trio_worker.py @@ -475,12 +475,6 @@ def close_read(self) -> None: def close_write(self) -> None: return - def wait(self) -> int | None: - return None - - def kill(self) -> None: - return - def _build_worker_gateway( host: _trio_host.TrioHost, diff --git a/src/execnet/aio.py b/src/execnet/aio.py index fb4f6c2a..40d4fc04 100644 --- a/src/execnet/aio.py +++ b/src/execnet/aio.py @@ -140,7 +140,17 @@ async def runner() -> None: post_result(result, None) def spawn() -> None: - self._host.start_soon(runner) + # Posted callbacks must not raise: trio turns an exception from + # an entry-queue callback into TrioInternalError and tears the + # whole host loop down, taking every other group with it. A + # host that shut down between the post and here resolves the + # future instead, like every other host-side failure. + try: + self._host.start_soon(runner) + except BaseException as exc: + error = RuntimeError("execnet aio host was shut down") + error.__cause__ = exc + post_result(None, error) try: self._host.portal.post(spawn) diff --git a/testing/test_basics.py b/testing/test_basics.py index c9fcae7c..7ddab050 100644 --- a/testing/test_basics.py +++ b/testing/test_basics.py @@ -371,7 +371,7 @@ class FakeGateway: _trio_session = None _new_wakener = staticmethod(_boundary.ThreadWakener) - def _check_event_loop(self, what: str) -> None: + def _check_usable(self, what: str) -> None: pass def _trace(self, *args) -> None: diff --git a/testing/test_host.py b/testing/test_host.py index e5ebab58..67f60d3c 100644 --- a/testing/test_host.py +++ b/testing/test_host.py @@ -10,13 +10,17 @@ import asyncio import os +import select +import signal import sys import threading +from collections.abc import Callable import pytest import trio import execnet +from execnet._errors import ForkedResourceError from execnet._host import Host from execnet._host import default_host @@ -76,33 +80,142 @@ def test_starting_is_lazy(self) -> None: assert not host.running assert "execnet-host-lazy" not in host_thread_names() - @pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork") - def test_forked_child_gets_a_fresh_default_host(self) -> None: - # the child inherits a Host object whose thread does not exist - # there, so the first use must build a new one - default_host()._ensure_started() - read_fd, write_fd = os.pipe() - pid = os.fork() - if pid == 0: # pragma: no cover - runs in the child - code = 1 - try: - os.close(read_fd) - child_host = default_host() - group = execnet.Group() - gateway = group.makegateway("popen") - got = gateway.remote_exec("channel.send(3)").receive(TESTTIMEOUT) - group.terminate(timeout=5.0) - code = 0 if (got == 3 and child_host.running) else 1 - finally: - os.write(write_fd, bytes([code])) - os._exit(0) - os.close(write_fd) + +def run_in_fork(child: Callable[[], list[str]], timeout: float = 20.0) -> list[str]: + """Run ``child`` in a forked process and return the problems it reports. + + The child reports rather than asserts, because an assertion there dies + with the child. A child that blocks fails the test instead of hanging + the suite -- most of what can go wrong after a fork is a wait for a loop + thread that does not exist in this process. + """ + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: # pragma: no cover - runs in the child + problems = ["the child died before reporting"] try: - result = os.read(read_fd, 1) - finally: os.close(read_fd) - os.waitpid(pid, 0) - assert result == b"\x00" + problems = child() + except BaseException as exc: + problems = [f"the child raised {type(exc).__name__}: {exc}"] + finally: + with os.fdopen(write_fd, "wb") as report: + report.write("\n".join(problems).encode()) + # not sys.exit: the parent's atexit handlers are not ours to run + os._exit(0) + os.close(write_fd) + chunks: list[bytes] = [] + try: + if not select.select([read_fd], [], [], timeout)[0]: + os.kill(pid, signal.SIGKILL) + pytest.fail(f"the forked child was still blocked after {timeout}s") + while chunk := os.read(read_fd, 4096): + chunks.append(chunk) + finally: + os.close(read_fd) + os.waitpid(pid, 0) + return [line for line in b"".join(chunks).decode().splitlines() if line] + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork") +class TestFork: + """Nothing execnet builds survives a fork, and it says so. + + The host's loop thread is not duplicated into the child and the worker + connections belong to the parent, so every inherited object is dead + there. Dead has to mean "raises and names the fork": the token of the + parent's loop still *accepts* work in the child, so without a check the + child waits forever for a reply nobody will send. Recovery is the + child's to make explicitly, by building a new host and group. + """ + + def test_a_new_group_in_the_child_works(self) -> None: + # the recovery path: default_host() hands a child its own Host + default_host()._ensure_started() + + def child() -> list[str]: + problems = [] + if default_host().running: + problems.append("the inherited default host claims to run here") + group = execnet.Group() + gateway = group.makegateway("popen") + got = gateway.remote_exec("channel.send(3)").receive(TESTTIMEOUT) + if got != 3: + problems.append(f"a fresh group returned {got!r}") + if not group.host.running: + problems.append("the child's own host is not running") + group.terminate(timeout=5.0) + return problems + + assert run_in_fork(child) == [] + + def test_inherited_channels_and_gateways_are_dead(self) -> None: + group = execnet.Group() + gateway = group.makegateway("popen") + channel = gateway.remote_exec("while 1: channel.send(channel.receive())") + channel.send(1) + assert channel.receive(TESTTIMEOUT) == 1 + + def child() -> list[str]: + problems: list[str] = [] + + def expect_forked(what: str, call: Callable[[], object]) -> None: + try: + call() + except ForkedResourceError as exc: + if "fork" not in str(exc): + problems.append(f"{what}: does not mention the fork: {exc}") + except BaseException as exc: + problems.append(f"{what}: {type(exc).__name__}: {exc}") + else: + problems.append(f"{what}: did not raise") + + expect_forked("channel.send()", lambda: channel.send(2)) + expect_forked("channel.receive()", lambda: channel.receive(TESTTIMEOUT)) + expect_forked("channel.waitclose()", lambda: channel.waitclose(5.0)) + expect_forked("gateway.remote_exec()", lambda: gateway.remote_exec("pass")) + expect_forked("gateway.join()", lambda: gateway.join(5.0)) + expect_forked("group.terminate()", lambda: group.terminate(timeout=5.0)) + return problems + + assert run_in_fork(child) == [] + # ... and the parent's own gateway is untouched by all of that + channel.send(2) + assert channel.receive(TESTTIMEOUT) == 2 + group.terminate(timeout=5.0) + + def test_the_inherited_default_group_is_dead(self) -> None: + # the module-level convenience group is built at import time, so it + # is always one of the objects a fork leaves behind + execnet.makegateway("popen") + + def child() -> list[str]: + try: + execnet.makegateway("popen") + except ForkedResourceError as exc: + return [] if "fork" in str(exc) else [f"unclear message: {exc}"] + except BaseException as exc: + return [f"raised {type(exc).__name__}: {exc}"] + return ["execnet.makegateway() did not raise"] + + assert run_in_fork(child) == [] + execnet.default_group.terminate(timeout=5.0) + + def test_the_child_does_not_run_the_parents_cleanup(self) -> None: + group = execnet.Group() + group.makegateway("popen") + + def child() -> list[str]: + # what atexit would call in the child: the parent's gateways are + # not ours to terminate, and trying would raise from an exit hook + group._cleanup_atexit() + if not len(group): + return ["the child unregistered the parent's gateways"] + return [] + + assert run_in_fork(child) == [] + assert group[0].remote_exec("channel.send(4)").receive(TESTTIMEOUT) == 4 + group.terminate(timeout=5.0) class TestHostDestruction: @@ -211,6 +324,64 @@ def test_setcallback_after_close_fails_without_wedging_the_channel(self) -> None group.terminate(timeout=5.0) +class TestPostedCallbacks: + """Work posted to the loop must never raise *on* the loop. + + Trio turns an exception from an entry-queue callback into a + TrioInternalError and tears the whole run down -- so one call losing a + race with shutdown would take every gateway in the process with it, and + tell the user to file a trio bug. A host that is already going away is + an ordinary failure of that one call. + """ + + def test_a_call_racing_shutdown_reports_instead_of_killing_the_loop( + self, + ) -> None: + host = Host(name="execnet-host-late-call") + group = execnet.Group(host=host) + gateway = group.makegateway("popen") + trio_host = host._ensure_started() + + async def never() -> None: # pragma: no cover - never spawned + raise AssertionError("should not run") + + nursery, trio_host._nursery = trio_host._nursery, None + try: + # the window between the root nursery closing and the run ending + pending = trio_host.call_pending(never) + with pytest.raises(RuntimeError, match="shut down"): + pending.wait(TESTTIMEOUT) + finally: + trio_host._nursery = nursery + + assert trio_host._thread is not None and trio_host._thread.is_alive() + assert gateway.remote_exec("channel.send(7)").receive(TESTTIMEOUT) == 7 + group.terminate(timeout=5.0) + host.close() + + def test_an_aio_call_racing_shutdown_reports_instead_of_killing_the_loop( + self, + ) -> None: + host = Host(name="execnet-host-late-aio-call") + + async def main() -> None: + async with execnet.aio.AsyncGroup(host=host) as group: + gateway = await group.makegateway("popen") + trio_host = host._ensure_started() + nursery, trio_host._nursery = trio_host._nursery, None + try: + with pytest.raises(RuntimeError, match="shut down"): + await gateway.remote_exec("channel.send(1)") + finally: + trio_host._nursery = nursery + assert trio_host._thread is not None and trio_host._thread.is_alive() + channel = await gateway.remote_exec("channel.send(7)") + assert await channel.receive() == 7 + + asyncio.run(main()) + host.close() + + class TestEventLoopGuard: """Blocking on the host from inside a running loop must not hang.""" @@ -248,6 +419,26 @@ async def main() -> None: finally: group.terminate(timeout=5.0) + def test_terminate_and_join_inside_asyncio_raise(self) -> None: + # both block on the host with no bound worth waiting out: join() + # until the worker dies, terminate() for the whole grace + group = execnet.Group() + try: + gateway = group.makegateway("popen") + + async def main() -> None: + with pytest.raises(RuntimeError, match=r"execnet\.aio"): + gateway.join(TESTTIMEOUT) + with pytest.raises(RuntimeError, match=r"execnet\.aio"): + group.terminate(timeout=5.0) + # an empty group has nothing to block on, so cleaning one up + # from inside a loop stays allowed + execnet.Group().terminate(timeout=5.0) + + asyncio.run(main()) + finally: + group.terminate(timeout=5.0) + def test_worker_channels_are_not_guarded(self) -> None: # exec'd code may run its own event loop and talk to its channel # from inside it -- that is the caller's own loop to block From d18f1469ecc0b2324114d7549f5f6b2a7967f3b5 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 19:45:30 +0200 Subject: [PATCH 90/91] fix: do not leave a worker running when makegateway fails late There is a seam between the connect helpers and the group: each helper kills its process if the handshake goes wrong, but once it returns, the worker is running with nobody owning it until `_processes[gateway]` is set a few lines later. A failure in between -- a cancellation, there is not much else -- left a worker no terminate() would ever reach. Forcing one (a `_make_gateway` that raises) leaves `running with PID ...` behind on the old code; now the stream is closed and the process killed and reaped, the same shielded, bounded cleanup the helpers already do. Also, two smaller things found while reading the same paths: - The sync `Group.terminate` holds coordinators back from the first exit pass, which reads like duplicated ordering (`AsyncGroup.terminate` already runs tunneled gateways first) but is not: `exit()` ends with close_write, and a tunneled gateway's termination frames still have to travel through that stream. Comment says so now, code unchanged. - The sync bridge's shutdown no longer hops to a thread to call a coordinator's `_terminate_execution`, which is a no-op. Only a worker has an exec pool to shut down. The receiver-callback pool is documented where it can bite: it is shared and bounded, so callbacks that wait on each other can fill it and stall every channel in the process. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 5 +++++ doc/implnotes.rst | 15 ++++++++++++++- src/execnet/_channel.py | 7 +++++++ src/execnet/_multi.py | 7 +++++++ src/execnet/_trio_gateway.py | 25 ++++++++++++++++++++++--- src/execnet/_trio_host.py | 4 ++++ testing/test_trio_gateway.py | 31 +++++++++++++++++++++++++++++++ 7 files changed, 90 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8e0608f7..0c5c9f94 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -169,6 +169,11 @@ series, once the consumers that need them have released without them. naming the fork. Recovery is explicit and belongs to the child: build a new ``Host`` and a new ``Group`` on it. A child that asks for the default host gets a fresh one, and it no longer inherits the parent's atexit cleanup. +* A ``makegateway`` that fails after the worker answered its handshake no longer + leaves that worker running. There is a seam between the connect helpers, which + each clean up after themselves, and the group taking ownership of the process; + a failure in it (a cancellation, realistically) used to leave a worker nothing + would ever terminate. * ``Group.terminate()`` and ``Gateway.join()`` join the calls that refuse to run inside a running asyncio or trio loop. Both block on the host with no useful bound -- ``join()`` until the worker dies -- which is the stall the guard exists to diff --git a/doc/implnotes.rst b/doc/implnotes.rst index 5db4d92c..e9d48dcc 100644 --- a/doc/implnotes.rst +++ b/doc/implnotes.rst @@ -149,7 +149,20 @@ Sends from a non-host thread wait until the frame is written, so an abrupt ``os._exit`` cannot drop queued data. Sends from the host thread itself (inside a receiver callback) only enqueue, to avoid deadlocking the writer task. ``setcallback`` runs its callback on a bounded thread pool rather than -on the loop. +on the loop: a consumer task per channel keeps that channel's order strict +while a slow callback blocks nothing but its own thread. The pool is shared +by every channel in the process and bounded (``Host(callback_threads=...)``, +40 by default), so callbacks that wait on *each other* can fill it and stall +the rest; work that waits belongs on a thread of its own. + +The host itself is not a resource that can be taken away quietly. Closing +it is final, and a fork leaves every inherited object dead in the child -- +both raise, because the alternative is a wait on a loop that will never run +again (``execnet._errors.ForkedResourceError``). For the same reason +nothing scheduled with ``portal.post`` may raise: trio turns an exception in +an entry-queue callback into a ``TrioInternalError`` that ends the whole +run, so a call that loses a race with shutdown reports through its own +result object instead. Inside the worker ---------------------- diff --git a/src/execnet/_channel.py b/src/execnet/_channel.py index 4ac30c3c..3557be05 100644 --- a/src/execnet/_channel.py +++ b/src/execnet/_channel.py @@ -117,6 +117,13 @@ def setcallback( specified the callback is eventually called with it when the channel closes, and ``waitclose()`` does not return until every callback (including the endmarker) has run. + + The pool the callbacks run on is shared and bounded (40 threads by + default, ``Host(callback_threads=...)``). A callback may block -- + that is the point of running it off the loop -- but callbacks that + block on *each other*, directly or through a queue only another + callback drains, can occupy the whole pool and stall every channel in + the process. Hand work that waits to a thread of your own. """ self.gateway._start_channel_consumer(self, callback, endmarker) diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py index 5fa90518..a4f45a3a 100644 --- a/src/execnet/_multi.py +++ b/src/execnet/_multi.py @@ -297,6 +297,13 @@ def terminate(self, timeout: float | None = None) -> None: # the same event-loop problem as makegateway() and receive() check_not_in_event_loop("Group.terminate()") while self or self._gateways_to_join: + # A coordinator is held back from this pass: a tunneled gateway + # rides *its* stream, and exit() ends with close_write, so + # exiting it first would shut the outbound side the sub's own + # termination frames still have to travel through. The held-back + # coordinators come round on the next pass, by which point the + # async group has terminated them and their exit() is a no-op + # that only unregisters them for the join below. vias: set[str] = set() for gw in self: if gw.spec.via: diff --git a/src/execnet/_trio_gateway.py b/src/execnet/_trio_gateway.py index a744bb39..1adbdcc7 100644 --- a/src/execnet/_trio_gateway.py +++ b/src/execnet/_trio_gateway.py @@ -1267,15 +1267,34 @@ async def makegateway(self, spec: str | Any = "popen") -> AsyncGateway: stream, process = await connect_popen_worker(spec) else: raise ValueError(f"unsupported spec for AsyncGroup: {spec!r}") - gateway = self._make_gateway(stream, spec) - gateway.remoteaddress = remoteaddress - await self._nursery.start(gateway._serve) + # From here to the registration below, the worker is running but + # nothing owns it yet: a failure (a cancel, most likely -- there is + # not much else) would leave a process the group never terminates. + # The connect helpers each clean up after themselves the same way; + # this is the seam between them and the group. + try: + gateway = self._make_gateway(stream, spec) + gateway.remoteaddress = remoteaddress + await self._nursery.start(gateway._serve) + except BaseException: + with trio.CancelScope(shield=True): + await self._abandon(stream, process) + raise self._gateways.append(gateway) if process is not None: self._processes[gateway] = process self._nursery.start_soon(self._reap_process, process) return gateway + async def _abandon(self, stream: ByteStream, process: trio.Process | None) -> None: + """Drop a worker nobody took ownership of (best effort, bounded).""" + with suppress(Exception): + await stream.aclose() + if process is not None: + with trio.move_on_after(5), suppress(Exception): + process.kill() + await process.wait() + def _make_gateway(self, stream: ByteStream, spec: Any) -> AsyncGateway: """Construct the gateway object for a freshly connected stream. diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 81069ec8..8cba3d7c 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -229,6 +229,10 @@ async def _finalize(self) -> None: # so the primary thread is not waiting on _done while terminate waits # on the primary thread draining work. self._done_sync.set(None) + if getattr(gateway, "_execpool", None) is None: + # a coordinator has no execution to shut down (its + # _terminate_execution is a no-op) and the thread hop is not free + return gateway._trace("[trio-bridge] terminating execution") # May sleep/SIGINT; keep it off the Trio scheduling thread. await trio.to_thread.run_sync( diff --git a/testing/test_trio_gateway.py b/testing/test_trio_gateway.py index 19c8a9f9..df3d317e 100644 --- a/testing/test_trio_gateway.py +++ b/testing/test_trio_gateway.py @@ -17,6 +17,7 @@ import trio.testing from execnet import _errors +from execnet import _trio_gateway from execnet._errors import RemoteError from execnet._message import Message from execnet._serialize import dumps_internal @@ -398,6 +399,36 @@ async def main() -> None: trio.run(main) + def test_a_makegateway_that_fails_late_leaves_no_worker( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # between the handshake and the group taking ownership, the worker is + # running and nothing would ever terminate it -- the connect helpers + # clean up after themselves, but only until they return + spawned: list[trio.Process] = [] + connect = _trio_gateway.connect_popen_worker + + async def spy(spec: object) -> tuple[object, trio.Process]: + stream, process = await connect(spec) + spawned.append(process) + return stream, process + + def boom(self: AsyncGroup, stream: object, spec: object) -> AsyncGateway: + raise RuntimeError("boom") + + monkeypatch.setattr(_trio_gateway, "connect_popen_worker", spy) + monkeypatch.setattr(AsyncGroup, "_make_gateway", boom) + + async def main() -> None: + async with AsyncGroup() as group: + with pytest.raises(RuntimeError, match="boom"): + await group.makegateway() + + trio.run(main) + assert len(spawned) == 1 + # killed and reaped, not left behind for the OS to inherit + assert spawned[0].returncode is not None + def test_unsupported_spec_is_rejected(self) -> None: async def main() -> None: async with AsyncGroup() as group: From 9b804abf2f6e77e95617d76a3cd95149700a6b91 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Fri, 31 Jul 2026 20:47:39 +0200 Subject: [PATCH 91/91] fix: a host loop that cannot start, and gevent's real constraint Cross-checking the host boundary against the things our own suite cannot see -- xdist's suite, and gevent as gevent is actually deployed. **A loop that dies at startup left the caller waiting 30s for a message that named nothing.** The loop comes up on a thread nobody is watching, so `trio.run` raising went to the thread's excepthook while `start()` sat on `_ready` until the timeout and then said "TrioHost failed to start". The exception is captured and re-raised at the call site now. A late failure still goes the loud way -- there, the thread traceback is the only report anyone gets. **execnet.gevent does not work in a monkey-patched process, and said the opposite.** The host loop is a trio program in a side thread; trio wants `select.epoll`, real sockets, a real thread and a real `SimpleQueue`, all of which `gevent.monkey` replaces process-wide. Verified in every variant: `patch_all()` loses epoll, `select=False` gets EBADF on trio's wakeup socketpair, `thread=False, socket=False, select=False` still hits gevent's SimpleQueue and LoopExit. Patching was never what made the namespace work -- its waits park the calling greenlet because they wait on a gevent primitive -- but the docs said "do that yourself, as early as usual", which invites exactly the broken setup. The docs now state the constraint, the startup error names gevent when it is the cause, and the roadmap carries the decision: honest limitation, fight the global patch with `monkey.get_original`, or a transport that needs no in-process loop. **The gevent facade took the blocking portal call after all.** Every management op on it is careful to park the greenlet rather than the hub -- except starting the group's async side, which went through `TrioHost.call` and blocked the whole hub. Short enough that the timing test stayed green, which is how it survived; the new test forbids the method outright. One `Group.host_call` now decides how this facade waits on the host, for makegateway, terminate and the async-group start alike. Also: `RemoteError.warn` can no longer raise. It is a best-effort diagnostic for a channel nobody kept, it can run on the loop and as late as interpreter shutdown with stderr already closed -- and an exception there ends the run for every gateway in the process. The xdist contract is green throughout (195 passed, `test_remote_inner_argv` deselected as documented). Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.rst | 15 ++++++++++ ROADMAP-3.0.md | 30 ++++++++++++++++++- doc/api.rst | 11 +++++-- src/execnet/_errors.py | 11 +++++-- src/execnet/_multi.py | 37 +++++++++++++----------- src/execnet/_trio_host.py | 61 +++++++++++++++++++++++++++++++-------- src/execnet/gevent.py | 13 +++++++-- testing/test_gevent.py | 29 +++++++++++++++++-- testing/test_host.py | 16 ++++++++++ 9 files changed, 184 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0c5c9f94..53d88b96 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -169,6 +169,21 @@ series, once the consumers that need them have released without them. naming the fork. Recovery is explicit and belongs to the child: build a new ``Host`` and a new ``Group`` on it. A child that asks for the default host gets a fresh one, and it no longer inherits the parent's atexit cleanup. +* A host loop that cannot start now says why, immediately. It comes up on a thread + nobody is watching, so a ``trio.run`` that died at once left the caller waiting + out the full 30s start timeout and then raising something generic, with the + actual reason only on stderr. The failure is re-raised at the call site, and + when ``gevent.monkey`` is what broke it, the message says so. +* ``execnet.gevent`` requires a process that has **not** monkey-patched. The host + loop is a Trio program on its own OS thread and needs the real ``select`` (for + ``epoll``), ``socket``, ``thread`` and ``queue``; ``gevent.monkey`` replaces + those process-wide. Patching was never what made the namespace work -- its waits + park the calling greenlet because they wait on a gevent primitive -- but the + documentation implied patching was fine, and it is not. +* ``execnet.gevent``'s first ``makegateway`` no longer blocks the hub. Starting the + group's async side took the blocking portal call that every other management + operation on this facade deliberately avoids; it is short enough that the timing + test stayed green, which is why it survived. * A ``makegateway`` that fails after the worker answered its handshake no longer leaves that worker running. There is a seam between the connect helpers, which each clean up after themselves, and the group taking ownership of the process; diff --git a/ROADMAP-3.0.md b/ROADMAP-3.0.md index dcb5153a..919bbfaf 100644 --- a/ROADMAP-3.0.md +++ b/ROADMAP-3.0.md @@ -97,7 +97,35 @@ the routing layer `_trio_host`/`_trio_worker` drive. Underscore both (every caller is ours), keep `open_channel` as the async `newchannel()`, and add a namespace test pinning the public method set. -### 4. Stale wording +### 4. `execnet.gevent` and monkey-patching — decide what we claim + +The facade works in a process that uses gevent *without* monkey-patching, +and its own promise holds there: blocking waits park the calling greenlet. +It does **not** work once `gevent.monkey` has patched the modules trio +reaches for from a side thread — verified in every variant: + +| patched | where it dies | +|---|---| +| `patch_all()` | `select.epoll` is removed; trio's IO manager cannot be built | +| `patch_all(select=False)` | trio's wakeup socketpair is a gevent socket -> `EBADF` | +| `patch_all(thread=False, socket=False, select=False)` | `queue.SimpleQueue` is gevent's; `from_thread.run` -> `LoopExit` | + +Which is a problem, because a real gevent application usually *does* +monkey-patch. The failure is now immediate and names gevent +(`TrioHost.start` -> `_startup_hint`) rather than hanging for 30s, and the +docs no longer imply patching is fine, so nothing is silently broken. But +"supported for gevent apps" is a bigger claim than "works if you drive +gevent explicitly", and only one of them is true today. + +Three ways out, in increasing order of ambition: keep the honest +limitation and document it (where we are); give the host loop the +originals (`monkey.get_original`) everywhere trio touches the stdlib, +which means fighting a global patch from inside a library and is likely +unmaintainable; or let a gevent process drive gateways over a transport +that needs no trio loop in-process at all. Decide before 3.0, because it +is what the namespace promises. + +### 5. Stale wording `_execmodel.ExecModel`'s docstring still describes the `loop=`/`exec=`/ `wait=` axes, which were dropped before they shipped. `_shim.REMOVED_IN` diff --git a/doc/api.rst b/doc/api.rst index c70a935a..afb67965 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -83,12 +83,19 @@ longer stalls the whole hub:: channel = gateway.remote_exec("channel.send(6 * 7)") print(channel.receive()) # parks this greenlet, not the hub -Requires ``execnet[gevent]``. Importing it does not monkey-patch anything --- do that yourself, as early as usual. ``Group``, ``default_group`` and +Requires ``execnet[gevent]``. ``Group``, ``default_group`` and ``makegateway`` are this module's own; the remaining names (``Channel``, ``Gateway``, ``RSync``, the error types) are the ones from :mod:`execnet.sync`. +Importing it monkey-patches nothing, and the process must not have +monkey-patched either: protocol IO is a Trio loop on its own OS thread and +needs the real ``select`` (for ``epoll``), ``socket``, ``thread`` and +``queue``, which ``gevent.monkey`` replaces process-wide. You do not need +patching here -- the waits above park the calling greenlet because they +wait on a gevent primitive. A host that cannot start in a patched process +raises and names gevent. + This is about the *caller*. Whether the worker itself runs greenlets is the independent ``profile=gevent`` spec key -- see :ref:`worker profiles `. diff --git a/src/execnet/_errors.py b/src/execnet/_errors.py index b8763f4d..ecefcbcc 100644 --- a/src/execnet/_errors.py +++ b/src/execnet/_errors.py @@ -12,6 +12,7 @@ import os import sys import traceback +from contextlib import suppress #: exceptions that must never be swallowed by a broad ``except`` sysex = (KeyboardInterrupt, SystemExit) @@ -89,8 +90,14 @@ def __repr__(self) -> str: def warn(self) -> None: if self.formatted != INTERRUPT_TEXT: - # XXX do this better - sys.stderr.write(f"[{os.getpid()}] Warning: unhandled {self!r}\n") + # A best-effort diagnostic that must not raise: it runs for a + # channel nobody kept a reference to, which can be on the host + # loop (a close replayed to a late-bound consumer) and as late + # as interpreter shutdown, where stderr may already be closed. + # An exception on the loop ends the run for every gateway. + with suppress(Exception): + # XXX do this better + sys.stderr.write(f"[{os.getpid()}] Warning: unhandled {self!r}\n") class TimeoutError(IOError): diff --git a/src/execnet/_multi.py b/src/execnet/_multi.py index a4f45a3a..08b9b09d 100644 --- a/src/execnet/_multi.py +++ b/src/execnet/_multi.py @@ -101,6 +101,23 @@ def host(self) -> Host: def _ensure_trio_host(self) -> Any: return self._host._ensure_started() + def host_call(self, trio_host: Any, async_fn: Any, *args: Any) -> Any: + """Run ``async_fn`` on the host, parking the way this facade parks. + + ``wait=thread`` keeps the KI-deferred ``portal.run`` path. Any other + backend implies the caller may not own its OS thread -- a gevent hub + runs every other greenlet on it -- so the work becomes a host task + and the wait happens on a OneShot with this facade's wakener. + """ + if self._wait_backend == "thread": + return trio_host.call(async_fn, *args) + from ._boundary import make_wakener + + pending = trio_host.call_pending( + async_fn, *args, wakener=make_wakener(self._wait_backend) + ) + return pending.wait() + def _ensure_async_group(self) -> Any: """The FacadeAsyncGroup owning the async side, running on the host.""" if self._async_group is None: @@ -112,7 +129,7 @@ async def _start() -> Any: async_group = _trio_host.FacadeAsyncGroup(self, host) return await host._nursery.start(async_group.run) - self._async_group = host.call(_start) + self._async_group = self.host_call(host, _start) return self._async_group @property @@ -324,23 +341,9 @@ def terminate(self, timeout: float | None = None) -> None: self._gateways_to_join[:] = [] def _host_terminate(self, timeout: float | None) -> None: - """Terminate the async group, parking correctly for the wait backend. - - A non-thread backend implies the caller may be a greenlet: wait on - a OneShot instead of blocking the OS thread (which would stall the - hub for the whole grace). - """ + """Terminate the async group, parking the way this facade parks.""" trio_host = self._host._ensure_started() - if self._wait_backend == "thread": - trio_host.call(self._async_group.terminate, timeout) - return - from ._boundary import make_wakener - - trio_host.call_pending( - self._async_group.terminate, - timeout, - wakener=make_wakener(self._wait_backend), - ).wait() + self.host_call(trio_host, self._async_group.terminate, timeout) def remote_exec( self, diff --git a/src/execnet/_trio_host.py b/src/execnet/_trio_host.py index 8cba3d7c..57fbe304 100644 --- a/src/execnet/_trio_host.py +++ b/src/execnet/_trio_host.py @@ -528,6 +528,33 @@ def is_alive(self) -> bool: return not self._done_sync.is_set() +#: modules trio's cross-thread machinery needs the real versions of +_GEVENT_SENSITIVE = ("select", "socket", "thread", "queue") + + +def _startup_hint() -> str: + """Name gevent when monkey-patching is what kept the loop from starting. + + The host loop is a trio program in a side thread, and trio reaches for + ``select.epoll``, real sockets and a real ``SimpleQueue`` to talk to it. + ``gevent.monkey`` replaces those process-wide, so a patched process + fails somewhere inside trio with an error that says nothing about + gevent. + """ + monkey = sys.modules.get("gevent.monkey") + if monkey is None: + return "" + patched = [name for name in _GEVENT_SENSITIVE if monkey.is_module_patched(name)] + if not patched: + return "" + return ( + f" -- gevent has monkey-patched {', '.join(patched)}, and the host loop" + " needs the real ones. execnet.gevent supports a process that uses" + " gevent without monkey-patching these modules; its blocking waits" + " park the calling greenlet either way." + ) + + class TrioHost: """Dedicated OS thread running ``trio.run`` for protocol IO.""" @@ -545,6 +572,7 @@ def __init__( self._shutdown: trio.Event | None = None self._started = False self._callback_limiter: trio.CapacityLimiter | None = None + self._startup_error: BaseException | None = None def start(self) -> None: if self._started: @@ -552,7 +580,12 @@ def start(self) -> None: self._thread = threading.Thread(target=self._run, name=self._name, daemon=True) self._thread.start() if not self._ready.wait(timeout=30): - raise RuntimeError("TrioHost failed to start") + raise RuntimeError("TrioHost failed to start within 30s") + error = self._startup_error + if error is not None: + raise RuntimeError( + f"the execnet host loop could not start: {error!r}{_startup_hint()}" + ) from error self._started = True @property @@ -572,7 +605,18 @@ def is_host_thread(self) -> bool: return self._portal is not None and self._portal.is_loop_thread() def _run(self) -> None: - trio.run(self._main) + try: + trio.run(self._main) + except BaseException as exc: + if self._ready.is_set(): + # the loop was up and died later: nobody is waiting on us, + # so let the thread report it the loud way + raise + # start() is blocked on _ready and would otherwise wait out the + # full timeout and raise something generic, with the actual + # reason only on stderr + self._startup_error = exc + self._ready.set() async def _main(self) -> None: self._portal = LoopPortal() @@ -758,16 +802,9 @@ def makegateway_trio(group: Group, spec: Any) -> Gateway: """Create a sync-facade Gateway for ``spec`` on the group's Trio host.""" host: TrioHost = group._ensure_trio_host() async_group: FacadeAsyncGroup = group._ensure_async_group() - if group._wait_backend == "thread": - bridge = host.call(async_group.makegateway, spec) - else: - # e.g. a gevent app: wait on a OneShot so only the calling - # greenlet parks while the gateway comes up, not the whole hub. - from ._boundary import make_wakener - - bridge = host.call_pending( - async_group.makegateway, spec, wakener=make_wakener(group._wait_backend) - ).wait() + # e.g. a gevent app: only the calling greenlet parks while the gateway + # comes up, not the whole hub. + bridge = group.host_call(host, async_group.makegateway, spec) assert isinstance(bridge, SyncBridgeGateway) gw: Gateway = bridge.sync_gateway # type: ignore[assignment] gw._io = SyncIOHandle( diff --git a/src/execnet/gevent.py b/src/execnet/gevent.py index df5aadfb..5d0d8cbf 100644 --- a/src/execnet/gevent.py +++ b/src/execnet/gevent.py @@ -16,9 +16,16 @@ gevent (``execnet[gevent]``). This is about the *caller*: the worker's own shape is the ``profile=`` -spec key, and ``profile=gevent`` is an independent choice. Importing this -module does not monkey-patch anything -- do that yourself, as early as -usual. +spec key, and ``profile=gevent`` is an independent choice. + +Importing this module monkey-patches nothing, and **the process it runs in +must not have monkey-patched either**: the host loop is a Trio program on +its own OS thread, and it needs the real ``select`` (for ``epoll``), +``socket``, ``thread`` and ``queue``, which ``gevent.monkey`` replaces +process-wide. Patching is not what makes this namespace work anyway -- +its waits park the calling greenlet because they wait on a gevent +primitive, not because the stdlib was swapped underneath them. A host +that cannot start in a patched process says so, and names gevent. """ from __future__ import annotations diff --git a/testing/test_gevent.py b/testing/test_gevent.py index 5d8a84c3..8bdc3441 100644 --- a/testing/test_gevent.py +++ b/testing/test_gevent.py @@ -1,9 +1,11 @@ """The execnet.gevent facade: greenlet-parking blocking waits. Opt-in: requires the ``gevent`` dependency group (``uv sync --group -gevent``); skipped when gevent is not installed. No monkey-patching is +gevent``); skipped when gevent is not installed. Monkey-patching is not needed -- the wakener parks the waiting greenlet while the trio host -thread keeps running the protocol. +thread keeps running the protocol -- and is not supported either: the host +loop needs the real ``select``/``socket``/``thread``/``queue``, so these +tests run in an unpatched process and so must the facade. """ from __future__ import annotations @@ -16,6 +18,7 @@ import execnet # noqa: E402 import execnet.gevent # noqa: E402 +from execnet import _trio_host # noqa: E402 from execnet._boundary import Flag # noqa: E402 from execnet._boundary import Mailbox # noqa: E402 from execnet._boundary import make_wakener # noqa: E402 @@ -108,6 +111,28 @@ def other() -> None: finally: gevent.spawn(group.terminate, 5.0).get(timeout=TESTTIMEOUT) + def test_no_management_op_takes_the_blocking_portal( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # the deterministic half of the test above. TrioHost.call waits for + # arbitrary host-side work with the calling OS thread parked, which + # for a gevent caller is the hub and every greenlet on it -- so no + # facade path may take it. Easy to miss for the lazily started async + # group, whose wait is short enough that a timing test stays green. + # (A bounded scheduling hop -- portal.run_sync, for the setcallback + # switch -- is a different thing and stays allowed.) + def forbidden(self: object, async_fn: object, *args: object) -> None: + raise AssertionError("the gevent facade used the blocking portal.run") + + monkeypatch.setattr(_trio_host.TrioHost, "call", forbidden) + group = execnet.gevent.Group() + try: + gateway = gevent.spawn(group.makegateway, "popen").get(timeout=TESTTIMEOUT) + channel = gateway.remote_exec("channel.send(42)") + assert gevent.spawn(channel.receive, TESTTIMEOUT).get(TESTTIMEOUT) == 42 + finally: + gevent.spawn(group.terminate, 5.0).get(timeout=TESTTIMEOUT) + class TestGeventWorkerProfile: """execmodel=gevent: exec'd code runs as greenlets on the main-thread hub.""" diff --git a/testing/test_host.py b/testing/test_host.py index 67f60d3c..1d1c34a4 100644 --- a/testing/test_host.py +++ b/testing/test_host.py @@ -20,6 +20,7 @@ import trio import execnet +from execnet import _trio_host from execnet._errors import ForkedResourceError from execnet._host import Host from execnet._host import default_host @@ -73,6 +74,21 @@ def test_host_context_manager_closes(self) -> None: group.terminate(timeout=5.0) assert not host.running + def test_a_loop_that_cannot_start_says_why( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # the loop comes up on a thread nobody is watching, so a trio.run + # that dies at once used to leave the caller waiting out the full + # 30s start timeout for a message that named nothing + async def boom(self: object) -> None: + raise RuntimeError("no event loop for you") + + monkeypatch.setattr(_trio_host.TrioHost, "_main", boom) + host = Host(name="execnet-host-doomed") + with pytest.raises(RuntimeError, match="could not start") as excinfo: + execnet.Group(host=host).makegateway("popen") + assert "no event loop for you" in str(excinfo.value) + def test_starting_is_lazy(self) -> None: host = Host(name="execnet-host-lazy") execnet.Group(host=host)