diff --git a/.github/workflows/fspy-sigsys-research.yml b/.github/workflows/fspy-sigsys-research.yml new file mode 100644 index 000000000..aa4c8ecbd --- /dev/null +++ b/.github/workflows/fspy-sigsys-research.yml @@ -0,0 +1,234 @@ +name: fspy SIGSYS research + +permissions: + contents: read + +on: + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/fspy-sigsys-research.yml' + - 'crates/vite_task_bin/tests/e2e_snapshots/fixtures/vitest_browser_cache/**' + - 'research/ptrace-exec-prototype/**' + - 'research/rust-injected-runtime/**' + - 'research/sigsys-prototype/**' + +defaults: + run: + shell: bash + +jobs: + native-x86: + name: Native x86-64 + runs-on: ubuntu-24.04 + steps: + - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 + + - uses: oxc-project/setup-rust@3d6fb132fbe7cdcb66bf8ec193911c2945369d12 # v1.0.17 + with: + components: llvm-tools + + - name: Build and audit the freestanding Rust handler blob + run: | + rustup target add x86_64-unknown-linux-musl aarch64-unknown-linux-musl + make -C research/rust-injected-runtime check + + - name: Run syscall and exec prototypes + run: | + set -euxo pipefail + test "$(uname -m)" = x86_64 + + probe_dir=$(mktemp -d) + cp -a research/sigsys-prototype "$probe_dir/sigsys" + cp -a research/ptrace-exec-prototype "$probe_dir/ptrace" + + cd "$probe_dir/sigsys" + cc -O2 -Wall -Wextra -Werror -pthread trap_bench.c -o trap_bench + ./trap_bench + cc -O2 -Wall -Wextra -Werror unotify_bench.c -o unotify_bench + ./unotify_bench + cc -O2 -Wall -Wextra -Werror reexec_bootstrap.c -o reexec_bootstrap + ./reexec_bootstrap + + cd "$probe_dir/ptrace" + make check + timeout 30s ./recursive_injector /bin/true + cc -O2 -Wall -Wextra -Werror -pthread -std=gnu11 \ + nonleader_exec.c -o nonleader_exec + ./nonleader_exec /bin/true + + vitest-browser-x86: + name: Chromium sandbox compatibility boundary + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 + + - uses: oxc-project/setup-node@4c588e9266bd930b6ddc34307df0659ed511d187 # v1.3.1 + + - name: Install Chromium system dependencies + run: pnpm --filter vite-task-tools exec playwright install --with-deps chromium + + - name: Enable Chromium's user-namespace sandbox + run: | + set -euxo pipefail + sysctl kernel.unprivileged_userns_clone + sysctl kernel.apparmor_restrict_unprivileged_userns + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + - name: Verify the CI host can launch sandboxed Chromium + run: | + set -euxo pipefail + sandbox_control=' + const { chromium } = await import("playwright"); + const browser = await chromium.launch({ + headless: true, + chromiumSandbox: true, + }); + const page = await browser.newPage(); + await page.setContent("

sandbox control

"); + await browser.close(); + ' + pnpm --filter vite-task-tools exec node --input-type=module \ + --eval "$sandbox_control" + setpriv --no-new-privs \ + pnpm --filter vite-task-tools exec node --input-type=module \ + --eval "$sandbox_control" + + - name: Validate Vitest browser modes through the exec bridge + run: | + set -euxo pipefail + test "$(uname -m)" = x86_64 + + make -C research/ptrace-exec-prototype recursive_injector + probe_dir=$(mktemp -d) + cp -a crates/vite_task_bin/tests/e2e_snapshots/fixtures/vitest_browser_cache \ + "$probe_dir/default" + cp -a crates/vite_task_bin/tests/e2e_snapshots/fixtures/vitest_browser_cache \ + "$probe_dir/sandboxed" + ln -s "$PWD/packages/tools/node_modules" "$probe_dir/node_modules" + + injector="$PWD/research/ptrace-exec-prototype/recursive_injector" + vitest="$PWD/packages/tools/node_modules/.bin/vitest" + + cd "$probe_dir/default" + VITEST_CHROMIUM_SANDBOX=false \ + DEBUG=vitest:browser:playwright,pw:browser \ + timeout 120s "$injector" "$vitest" run \ + 2>&1 | tee "$probe_dir/default.log" + test -s dist/result.json + grep -F '"success":true' dist/result.json + grep -F 'chromiumSandbox: false' "$probe_dir/default.log" + grep -E 'bridge: injected exec .*exe=.*/(chrome|headless_shell)' \ + "$probe_dir/default.log" + grep -E 'bridge: summary injected_execs=[2-9][0-9]* failed_execs=0' \ + "$probe_dir/default.log" + + cd "$probe_dir/sandboxed" + + sandbox_record="$probe_dir/chromium-sandbox.txt" + no_sandbox_record="$probe_dir/chromium-no-sandbox.txt" + ( + set +x + while :; do + for cmdline_file in /proc/[0-9]*/cmdline; do + test -r "$cmdline_file" || continue + command_line=$(tr '\0' ' ' < "$cmdline_file" 2>/dev/null || true) + case "$command_line" in + *chrome-headless-shell*) ;; + *) continue ;; + esac + case "$command_line" in + *--no-sandbox*) + printf '%s\n' "$command_line" > "$no_sandbox_record" + ;; + esac + case "$command_line" in + *--type=renderer*) ;; + *) continue ;; + esac + + pid=${cmdline_file#/proc/} + pid=${pid%/cmdline} + filters=$(awk '/^Seccomp_filters:/ { print $2 }' \ + "/proc/$pid/status" 2>/dev/null || true) + if test "${filters:-0}" -ge 2; then + { + printf 'pid=%s\ncmdline=%s\n' "$pid" "$command_line" + awk '/^(NoNewPrivs|Seccomp|Seccomp_filters):/ { print }' \ + "/proc/$pid/status" 2>/dev/null || true + } > "$sandbox_record" + exit 0 + fi + done + sleep 0.02 + done + ) & + observer_pid=$! + + set +e + VITEST_CHROMIUM_SANDBOX=true \ + DEBUG=vitest:browser:playwright,pw:browser \ + timeout 120s "$injector" "$vitest" run \ + 2>&1 | tee "$probe_dir/bridge.log" + run_status=${PIPESTATUS[0]} + set -e + + kill "$observer_pid" 2>/dev/null || true + wait "$observer_pid" 2>/dev/null || true + test "$run_status" -ne 0 + + test -s dist/result.json + grep -F 'chromiumSandbox: true' "$probe_dir/bridge.log" + test ! -s "$no_sandbox_record" + test ! -s "$sandbox_record" + grep -E 'bridge: exec argv .* --type=zygote ' "$probe_dir/bridge.log" + grep -F 'bridge: zygote fd 3=socket:' "$probe_dir/bridge.log" + grep -F 'fspy: recvmsg returned EOF' "$probe_dir/bridge.log" + grep -F 'FATAL:content/browser/zygote_host/zygote_host_impl_linux.cc:207' \ + "$probe_dir/bridge.log" + grep -F 'ZygoteMain: initializing' "$probe_dir/bridge.log" + grep -F 'write: Broken pipe (32)' "$probe_dir/bridge.log" + grep -E 'bridge: summary injected_execs=[2-9][0-9]* failed_execs=0' \ + "$probe_dir/bridge.log" + + docker-x86: + name: Docker default seccomp + runs-on: ubuntu-24.04 + steps: + - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 + + - name: Run under an existing filter with no-new-privileges + run: | + set -euxo pipefail + docker run --rm \ + --security-opt no-new-privileges \ + -v "$PWD/research:/src:ro" \ + gcc:14-bookworm \ + sh -eux -c ' + grep Seccomp /proc/self/status + grep NoNewPrivs /proc/self/status + cat /proc/self/attr/current || true + + mkdir /tmp/sigsys /tmp/ptrace + cp /src/sigsys-prototype/trap_bench.c /tmp/sigsys/ + cp /src/sigsys-prototype/unotify_bench.c /tmp/sigsys/ + cp /src/sigsys-prototype/reexec_bootstrap.c /tmp/sigsys/ + cp /src/ptrace-exec-prototype/injector.c /tmp/ptrace/ + cp /src/ptrace-exec-prototype/nested_trap.c /tmp/ptrace/ + cp /src/ptrace-exec-prototype/recursive_injector.c /tmp/ptrace/ + cp /src/ptrace-exec-prototype/target.c /tmp/ptrace/ + cp /src/ptrace-exec-prototype/Makefile /tmp/ptrace/ + + cd /tmp/sigsys + cc -O2 -Wall -Wextra -Werror -pthread trap_bench.c -o trap_bench + ./trap_bench + cc -O2 -Wall -Wextra -Werror unotify_bench.c -o unotify_bench + ./unotify_bench + cc -O2 -Wall -Wextra -Werror reexec_bootstrap.c -o reexec_bootstrap + ./reexec_bootstrap + + cd /tmp/ptrace + make check + timeout 30s ./recursive_injector /bin/true + ' diff --git a/.typos.toml b/.typos.toml index c97dbfdec..3f17d0d6b 100644 --- a/.typos.toml +++ b/.typos.toml @@ -2,6 +2,9 @@ ratatui = "ratatui" PUNICODE = "PUNICODE" +[default.extend-identifiers] +msg_controllen = "msg_controllen" + [files] extend-exclude = [ "crates/fspy_detours_sys/detours", diff --git a/Cargo.lock b/Cargo.lock index 351ca9422..9fecd10e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1206,6 +1206,7 @@ dependencies = [ "anyhow", "bstr", "bumpalo", + "cc", "csv-async", "ctor", "derive_more", @@ -1213,7 +1214,6 @@ dependencies = [ "fspy_detours_sys", "fspy_preload_unix", "fspy_preload_windows", - "fspy_seccomp_unotify", "fspy_shared", "fspy_shared_unix", "fspy_test_bin", @@ -1367,7 +1367,6 @@ dependencies = [ "base64", "bstr", "elf", - "fspy_seccomp_unotify", "fspy_shared", "memmap2", "nix 0.31.2", diff --git a/crates/fspy/Cargo.toml b/crates/fspy/Cargo.toml index 38b9ffb51..d23b5e1c6 100644 --- a/crates/fspy/Cargo.toml +++ b/crates/fspy/Cargo.toml @@ -23,7 +23,6 @@ tokio-util = { workspace = true } which = { workspace = true, features = ["tracing"] } [target.'cfg(target_os = "linux")'.dependencies] -fspy_seccomp_unotify = { workspace = true, features = ["supervisor"] } nix = { workspace = true, features = ["uio"] } tokio = { workspace = true, features = ["bytes"] } @@ -61,6 +60,7 @@ fspy_test_bin = { path = "../fspy_test_bin", artifact = "bin", target = "x86_64- # builds are cheap. [build-dependencies] anyhow = { workspace = true } +cc = { workspace = true } materialized_artifact_build = { workspace = true } flate2 = { workspace = true } fspy_preload_unix = { workspace = true } diff --git a/crates/fspy/build.rs b/crates/fspy/build.rs index 90b7bbcf4..05ca6d204 100644 --- a/crates/fspy/build.rs +++ b/crates/fspy/build.rs @@ -146,6 +146,7 @@ fn fetch_macos_binaries(out_dir: &Path) -> anyhow::Result<()> { fn register_preload_cdylib() -> anyhow::Result<()> { let env_name = match env::var("CARGO_CFG_TARGET_OS").unwrap().as_str() { "windows" => "CARGO_CDYLIB_FILE_FSPY_PRELOAD_WINDOWS", + "linux" => return Ok(()), _ if env::var("CARGO_CFG_TARGET_ENV").unwrap() == "musl" => return Ok(()), _ => "CARGO_CDYLIB_FILE_FSPY_PRELOAD_UNIX", }; @@ -157,8 +158,24 @@ fn register_preload_cdylib() -> anyhow::Result<()> { Ok(()) } +fn build_linux_sigsys_injector() { + if env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("linux") + || env::var("CARGO_CFG_TARGET_ARCH").as_deref() != Ok("x86_64") + { + return; + } + + println!("cargo:rerun-if-changed=src/unix/sigsys_x86_64.c"); + cc::Build::new() + .file("src/unix/sigsys_x86_64.c") + .flag_if_supported("-std=c11") + .warnings(true) + .compile("fspy_sigsys_x86_64"); +} + fn main() -> anyhow::Result<()> { println!("cargo:rerun-if-changed=build.rs"); + build_linux_sigsys_injector(); let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); fetch_macos_binaries(&out_dir).context("Failed to fetch macOS binaries")?; register_preload_cdylib().context("Failed to register preload cdylib")?; diff --git a/crates/fspy/src/lib.rs b/crates/fspy/src/lib.rs index 6c89414ba..dfd5647e8 100644 --- a/crates/fspy/src/lib.rs +++ b/crates/fspy/src/lib.rs @@ -2,7 +2,6 @@ pub mod error; -#[cfg(not(target_env = "musl"))] mod ipc; #[cfg(unix)] diff --git a/crates/fspy/src/unix/mod.rs b/crates/fspy/src/unix/mod.rs index f01f63b5d..f4bb86dcd 100644 --- a/crates/fspy/src/unix/mod.rs +++ b/crates/fspy/src/unix/mod.rs @@ -1,16 +1,16 @@ #[cfg(target_os = "linux")] -mod syscall_handler; +mod sigsys; #[cfg(target_os = "macos")] mod macos_artifacts; +#[cfg(target_os = "linux")] +use std::os::fd::AsRawFd as _; use std::{io, path::Path}; -#[cfg(target_os = "linux")] -use fspy_seccomp_unotify::supervisor::supervise; -use fspy_shared::ipc::PathAccess; -#[cfg(not(target_env = "musl"))] -use fspy_shared::ipc::{NativeStr, channel::channel}; +#[cfg(target_os = "macos")] +use fspy_shared::ipc::NativeStr; +use fspy_shared::ipc::{PathAccess, channel::channel}; #[cfg(target_os = "macos")] use fspy_shared_unix::payload::Artifacts; use fspy_shared_unix::{ @@ -19,31 +19,39 @@ use fspy_shared_unix::{ spawn::handle_exec, }; use futures_util::FutureExt; -#[cfg(target_os = "linux")] -use syscall_handler::SyscallHandler; use tokio::task::spawn_blocking; use tokio_util::sync::CancellationToken; -#[cfg(not(target_env = "musl"))] -use crate::ipc::{OwnedReceiverLockGuard, SHM_CAPACITY}; -use crate::{ChildTermination, Command, TrackedChild, arena::PathAccessArena, error::SpawnError}; +use crate::{ + ChildTermination, Command, TrackedChild, + arena::PathAccessArena, + error::SpawnError, + ipc::{OwnedReceiverLockGuard, SHM_CAPACITY}, +}; #[derive(Debug)] pub struct SpyImpl { #[cfg(target_os = "macos")] artifacts: Artifacts, - #[cfg(not(target_env = "musl"))] + #[cfg(target_os = "macos")] preload_path: Box, } impl SpyImpl { - /// Initialize the fs access spy by writing the preload library on disk. - /// - /// On musl targets, we don't build a preload library — - /// only seccomp-based tracking is used. - pub fn init_in(#[cfg_attr(target_env = "musl", allow(unused))] dir: &Path) -> io::Result { - #[cfg(not(target_env = "musl"))] + /// Initializes platform artifacts. Linux injects its handler at exec and + /// does not materialize a preload library. + #[cfg(target_os = "linux")] + #[expect( + clippy::unnecessary_wraps, + reason = "keeps initialization uniform with the fallible macOS backend" + )] + pub const fn init_in(_dir: &Path) -> io::Result { + Ok(Self {}) + } + + #[cfg(target_os = "macos")] + pub fn init_in(dir: &Path) -> io::Result { let preload_path = { use materialized_artifact::{Artifact, artifact}; @@ -54,7 +62,7 @@ impl SpyImpl { }; Ok(Self { - #[cfg(not(target_env = "musl"))] + #[cfg(target_os = "macos")] preload_path, #[cfg(target_os = "macos")] artifacts: { @@ -74,25 +82,21 @@ impl SpyImpl { mut command: Command, cancellation_token: CancellationToken, ) -> Result { - #[cfg(target_os = "linux")] - let supervisor = supervise::().map_err(SpawnError::Supervisor)?; - - #[cfg(not(target_env = "musl"))] + #[cfg(target_os = "macos")] let (ipc_channel_conf, ipc_receiver) = channel(SHM_CAPACITY).map_err(SpawnError::ChannelCreation)?; + #[cfg(target_os = "linux")] + let (_, ipc_receiver) = channel(SHM_CAPACITY).map_err(SpawnError::ChannelCreation)?; let payload = Payload { - #[cfg(not(target_env = "musl"))] + #[cfg(target_os = "macos")] ipc_channel_conf, #[cfg(target_os = "macos")] artifacts: self.artifacts.clone(), - #[cfg(not(target_env = "musl"))] + #[cfg(target_os = "macos")] preload_path: self.preload_path.clone(), - - #[cfg(target_os = "linux")] - seccomp_payload: supervisor.payload().clone(), }; let encoded_payload = encode_payload(payload); @@ -108,14 +112,25 @@ impl SpyImpl { }, ) .map_err(|err| SpawnError::Injection(err.into()))?; + #[cfg(target_os = "linux")] + debug_assert!(pre_exec.is_none()); command.set_exec(exec); command.env("FSPY", "1"); let mut tokio_command = command.into_tokio_command(); - // SAFETY: the pre_exec closure only calls pre_exec.run() which is safe to call in a fork context + #[cfg(target_os = "linux")] + let shm_fd = ipc_receiver.shm_fd().as_raw_fd(); + #[cfg(target_os = "linux")] + let shm_len = ipc_receiver.shm_len(); + + // SAFETY: both platform hooks are restricted to async-signal-safe raw + // operations in the post-fork child. unsafe { tokio_command.pre_exec(move || { + #[cfg(target_os = "linux")] + sigsys::prepare(shm_fd)?; + #[cfg(target_os = "macos")] if let Some(pre_exec) = pre_exec.as_ref() { pre_exec.run()?; } @@ -123,13 +138,25 @@ impl SpyImpl { }); } - // tokio_command.spawn blocks while executing the `pre_exec` closure. - // Run it inside spawn_blocking to avoid blocking the tokio runtime, especially the supervisor loop, - // which needs to accept incoming connections while `pre_exec` is connecting to it. - let mut child = spawn_blocking(move || tokio_command.spawn()) - .await - .map_err(|err| SpawnError::OsSpawn(err.into()))? - .map_err(SpawnError::OsSpawn)?; + // Spawn and the post-exec ptrace handshake are blocking operations. + let mut child = spawn_blocking(move || { + let child = tokio_command.spawn().map_err(SpawnError::OsSpawn)?; + #[cfg(target_os = "linux")] + let child = { + let mut child = child; + let pid = child.id().ok_or_else(|| { + SpawnError::Injection(io::Error::other("spawned child has no process id")) + })?; + if let Err(error) = sigsys::inject(pid, shm_fd, shm_len) { + let _ = child.start_kill(); + return Err(SpawnError::Injection(error)); + } + child + }; + Ok(child) + }) + .await + .map_err(|err| SpawnError::OsSpawn(err.into()))??; Ok(TrackedChild { stdin: child.stdin.take(), @@ -146,28 +173,13 @@ impl SpyImpl { } }; - let arenas = std::iter::once(exec_resolve_accesses); - // Stop the supervisor and collect path accesses from it. - #[cfg(target_os = "linux")] - let arenas = arenas.chain( - supervisor - .stop() - .await? - .into_iter() - .map(syscall_handler::SyscallHandler::into_arena), - ); - let arenas = arenas.collect::>(); + let arenas = vec![exec_resolve_accesses]; // Lock the ipc channel after the child has exited. // We are not interested in path accesses from descendants after the main child has exited. - #[cfg(not(target_env = "musl"))] let ipc_receiver_lock_guard = OwnedReceiverLockGuard::lock_async(ipc_receiver).await?; - let path_accesses = PathAccessIterable { - arenas, - #[cfg(not(target_env = "musl"))] - ipc_receiver_lock_guard, - }; + let path_accesses = PathAccessIterable { arenas, ipc_receiver_lock_guard }; io::Result::Ok(ChildTermination { status, path_accesses }) }) @@ -179,7 +191,6 @@ impl SpyImpl { pub struct PathAccessIterable { arenas: Vec, - #[cfg(not(target_env = "musl"))] ipc_receiver_lock_guard: OwnedReceiverLockGuard, } @@ -188,14 +199,7 @@ impl PathAccessIterable { let accesses_in_arena = self.arenas.iter().flat_map(|arena| arena.borrow_accesses().iter()).copied(); - #[cfg(not(target_env = "musl"))] - { - let accesses_in_shm = self.ipc_receiver_lock_guard.iter_path_accesses(); - accesses_in_shm.chain(accesses_in_arena) - } - #[cfg(target_env = "musl")] - { - accesses_in_arena - } + let accesses_in_shm = self.ipc_receiver_lock_guard.iter_path_accesses(); + accesses_in_shm.chain(accesses_in_arena) } } diff --git a/crates/fspy/src/unix/sigsys.rs b/crates/fspy/src/unix/sigsys.rs new file mode 100644 index 000000000..3650fb2e0 --- /dev/null +++ b/crates/fspy/src/unix/sigsys.rs @@ -0,0 +1,64 @@ +use std::{io, os::fd::RawFd}; + +#[cfg(target_arch = "x86_64")] +unsafe extern "C" { + fn fspy_sigsys_prepare(shm_fd: libc::c_int) -> libc::c_int; + fn fspy_sigsys_inject( + pid: libc::pid_t, + shm_fd: libc::c_int, + shm_len: libc::size_t, + ) -> libc::c_int; +} + +/// Marks the child traceable and installs the selective TRAP filter. +/// +/// # Safety +/// +/// This must only run in the post-fork child immediately before exec. +pub unsafe fn prepare(shm_fd: RawFd) -> io::Result<()> { + #[cfg(target_arch = "x86_64")] + { + // SAFETY: the caller guarantees the pre-exec child context and the fd + // is the live channel memfd inherited from the parent. + if unsafe { fspy_sigsys_prepare(shm_fd) } == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } + } + + #[cfg(not(target_arch = "x86_64"))] + { + let _ = shm_fd; + Err(io::Error::new( + io::ErrorKind::Unsupported, + "the experimental SIGSYS injector supports Linux x86-64 only", + )) + } +} + +/// Waits for the child's post-exec ptrace stop, maps the existing IPC shared +/// memory, installs the in-process handler, and detaches. +pub fn inject(pid: u32, shm_fd: RawFd, shm_len: usize) -> io::Result<()> { + #[cfg(target_arch = "x86_64")] + { + let pid = libc::pid_t::try_from(pid) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "child pid exceeds pid_t"))?; + // SAFETY: `pid` is the freshly spawned TRACEME child and the fd/length + // identify the channel mapping inherited across its exec. + if unsafe { fspy_sigsys_inject(pid, shm_fd, shm_len) } == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } + } + + #[cfg(not(target_arch = "x86_64"))] + { + let _ = (pid, shm_fd, shm_len); + Err(io::Error::new( + io::ErrorKind::Unsupported, + "the experimental SIGSYS injector supports Linux x86-64 only", + )) + } +} diff --git a/crates/fspy/src/unix/sigsys_x86_64.c b/crates/fspy/src/unix/sigsys_x86_64.c new file mode 100644 index 000000000..94595082a --- /dev/null +++ b/crates/fspy/src/unix/sigsys_x86_64.c @@ -0,0 +1,499 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(__linux__) || !defined(__x86_64__) +#error "the experimental fspy SIGSYS injector supports Linux x86-64 only" +#endif + +#define ARRAY_LEN(values) (sizeof(values) / sizeof((values)[0])) +#define GATEWAY_MAGIC UINT64_C(0x4653505947415445) +#define GATEWAY_MAGIC_LOW UINT32_C(0x47415445) +#define FILTER_TAG UINT32_C(0x4653) +#define PATH_LIMIT 4096U +#define INSTALL_ACTION_OFFSET 0U + +/* + * This blob is copied into the tracee at its post-exec SIGTRAP stop. It has + * no relocations, GOT/PLT references, TLS, libc calls, or writable code data. + * The injector patches the slots at its tail before making the mapping RX. + * + * The record writer intentionally mirrors fspy_shared::ipc::channel::ShmWriter: + * + * usize committed_end + * repeated { i32 frame_size; frame bytes; padding to 4-byte alignment } + * + * A negative frame size commits a complete frame. The frame bytes are the + * existing wincode PathAccess encoding on 64-bit little-endian Unix: + * AccessMode(u8), path length(u64), path bytes. + */ +__asm__( + ".pushsection .text.fspy_sigsys_injected,\"ax\",@progbits\n" + ".balign 16\n" + ".global fspy_sigsys_blob_start\n" + ".global fspy_sigsys_handler\n" + ".global fspy_sigsys_restorer\n" + ".global fspy_sigsys_slot_rax\n" + ".global fspy_sigsys_slot_rdi\n" + ".global fspy_sigsys_slot_rsi\n" + ".global fspy_sigsys_slot_rdx\n" + ".global fspy_sigsys_slot_r10\n" + ".global fspy_sigsys_slot_r8\n" + ".global fspy_sigsys_slot_shm\n" + ".global fspy_sigsys_slot_shm_len\n" + ".global fspy_sigsys_slot_magic\n" + ".global fspy_sigsys_blob_end\n" + "fspy_sigsys_blob_start:\n" + "fspy_sigsys_handler:\n" + " push %rbp\n" + " mov %rsp, %rbp\n" + " push %rbx\n" + " push %r12\n" + " push %r13\n" + " push %r14\n" + " push %r15\n" + " mov %rsi, %r12\n" /* siginfo_t * */ + " mov %rdx, %r13\n" /* ucontext_t * */ + + /* Only dispatch traps produced by fspy's tagged filter. */ + " cmpl $0x4653, 4(%r12)\n" + " jne .Lfspy_sigsys_return\n" + " mov 24(%r12), %eax\n" + " cmp $2, %eax\n" /* __NR_open */ + " je .Lfspy_sigsys_open\n" + " cmp $257, %eax\n" /* __NR_openat */ + " je .Lfspy_sigsys_openat\n" + " mov $-38, %rax\n" /* -ENOSYS */ + " jmp .Lfspy_sigsys_store_result\n" + + ".Lfspy_sigsys_open:\n" + " mov fspy_sigsys_slot_rdi(%rip), %rcx\n" + " mov (%r13,%rcx), %r15\n" /* path */ + " mov fspy_sigsys_slot_rsi(%rip), %rcx\n" + " mov (%r13,%rcx), %rbx\n" /* flags */ + " jmp .Lfspy_sigsys_record\n" + + ".Lfspy_sigsys_openat:\n" + " mov fspy_sigsys_slot_rsi(%rip), %rcx\n" + " mov (%r13,%rcx), %r15\n" /* path */ + " mov fspy_sigsys_slot_rdx(%rip), %rcx\n" + " mov (%r13,%rcx), %rbx\n" /* flags */ + + ".Lfspy_sigsys_record:\n" + /* Convert O_ACCMODE into fspy's AccessMode bits. */ + " and $3, %ebx\n" + " cmp $1, %ebx\n" + " je .Lfspy_sigsys_write_mode\n" + " cmp $2, %ebx\n" + " je .Lfspy_sigsys_read_write_mode\n" + " mov $1, %ebx\n" + " jmp .Lfspy_sigsys_scan_path\n" + ".Lfspy_sigsys_write_mode:\n" + " mov $2, %ebx\n" + " jmp .Lfspy_sigsys_scan_path\n" + ".Lfspy_sigsys_read_write_mode:\n" + " mov $3, %ebx\n" + + /* Minimal experiment: paths are valid C strings, capped at PATH_MAX. */ + ".Lfspy_sigsys_scan_path:\n" + " xor %ecx, %ecx\n" + ".Lfspy_sigsys_scan_path_loop:\n" + " cmp $4096, %ecx\n" + " jae .Lfspy_sigsys_passthrough\n" + " cmpb $0, (%r15,%rcx)\n" + " je .Lfspy_sigsys_path_ready\n" + " inc %rcx\n" + " jmp .Lfspy_sigsys_scan_path_loop\n" + + ".Lfspy_sigsys_path_ready:\n" + " mov fspy_sigsys_slot_shm(%rip), %r14\n" + " lea 9(%rcx), %r8\n" /* encoded PathAccess size */ + " mov (%r14), %rax\n" /* current committed_end */ + ".Lfspy_sigsys_claim_loop:\n" + " lea 7(%rax,%r8), %rdx\n" /* old + 4-byte header + frame + 3 */ + " and $-4, %rdx\n" + " mov fspy_sigsys_slot_shm_len(%rip), %rsi\n" + " sub $8, %rsi\n" + " cmp %rsi, %rdx\n" + " ja .Lfspy_sigsys_passthrough\n" + " lock cmpxchgq %rdx, (%r14)\n" + " jne .Lfspy_sigsys_claim_loop\n" + + " lea 8(%r14,%rax), %rdx\n" /* claimed frame header */ + " mov %r8d, (%rdx)\n" /* positive: write in progress */ + " mov %bl, 4(%rdx)\n" + " mov %rcx, 5(%rdx)\n" + " lea 13(%rdx), %rdi\n" + " mov %r15, %rsi\n" + " rep movsb\n" + /* x86 TSO publishes the bytes before this aligned atomic-sized store. */ + " neg %r8d\n" + " mov %r8d, (%rdx)\n" /* negative: committed */ + + ".Lfspy_sigsys_passthrough:\n" + " mov 24(%r12), %ebx\n" + " mov fspy_sigsys_slot_rdi(%rip), %rcx\n" + " mov (%r13,%rcx), %rdi\n" + " mov fspy_sigsys_slot_rsi(%rip), %rcx\n" + " mov (%r13,%rcx), %rsi\n" + " mov fspy_sigsys_slot_rdx(%rip), %rcx\n" + " mov (%r13,%rcx), %rdx\n" + " mov fspy_sigsys_slot_r10(%rip), %rcx\n" + " mov (%r13,%rcx), %r10\n" + " mov fspy_sigsys_slot_r8(%rip), %rcx\n" + " mov (%r13,%rcx), %r8\n" + " mov fspy_sigsys_slot_magic(%rip), %r9\n" + " mov %ebx, %eax\n" + " syscall\n" + + ".Lfspy_sigsys_store_result:\n" + " mov fspy_sigsys_slot_rax(%rip), %rcx\n" + " mov %rax, (%r13,%rcx)\n" + ".Lfspy_sigsys_return:\n" + " pop %r15\n" + " pop %r14\n" + " pop %r13\n" + " pop %r12\n" + " pop %rbx\n" + " pop %rbp\n" + " ret\n" + + ".balign 8\n" + "fspy_sigsys_restorer:\n" + " mov $15, %eax\n" /* __NR_rt_sigreturn */ + " syscall\n" + " ud2\n" + ".balign 8\n" + "fspy_sigsys_slot_rax: .quad 0\n" + "fspy_sigsys_slot_rdi: .quad 0\n" + "fspy_sigsys_slot_rsi: .quad 0\n" + "fspy_sigsys_slot_rdx: .quad 0\n" + "fspy_sigsys_slot_r10: .quad 0\n" + "fspy_sigsys_slot_r8: .quad 0\n" + "fspy_sigsys_slot_shm: .quad 0\n" + "fspy_sigsys_slot_shm_len: .quad 0\n" + "fspy_sigsys_slot_magic: .quad 0\n" + "fspy_sigsys_blob_end:\n" + ".popsection\n"); + +extern const unsigned char fspy_sigsys_blob_start[]; +extern const unsigned char fspy_sigsys_handler[]; +extern const unsigned char fspy_sigsys_restorer[]; +extern const unsigned char fspy_sigsys_slot_rax[]; +extern const unsigned char fspy_sigsys_slot_rdi[]; +extern const unsigned char fspy_sigsys_slot_rsi[]; +extern const unsigned char fspy_sigsys_slot_rdx[]; +extern const unsigned char fspy_sigsys_slot_r10[]; +extern const unsigned char fspy_sigsys_slot_r8[]; +extern const unsigned char fspy_sigsys_slot_shm[]; +extern const unsigned char fspy_sigsys_slot_shm_len[]; +extern const unsigned char fspy_sigsys_slot_magic[]; +extern const unsigned char fspy_sigsys_blob_end[]; + +struct kernel_sigaction_wire { + uint64_t handler; + uint64_t flags; + uint64_t restorer; + uint64_t mask; +}; + +static size_t blob_offset(const unsigned char *symbol) +{ + return (size_t)(symbol - fspy_sigsys_blob_start); +} + +static void patch_u64(unsigned char *blob, const unsigned char *slot, + uint64_t value) +{ + memcpy(blob + blob_offset(slot), &value, sizeof(value)); +} + +static int ptrace_get_regs(pid_t pid, struct user_regs_struct *regs) +{ + struct iovec iov = {.iov_base = regs, .iov_len = sizeof(*regs)}; + return (int)ptrace(PTRACE_GETREGSET, pid, (void *)(uintptr_t)NT_PRSTATUS, + &iov); +} + +static int ptrace_set_regs(pid_t pid, const struct user_regs_struct *regs) +{ + struct iovec iov = {.iov_base = (void *)regs, .iov_len = sizeof(*regs)}; + return (int)ptrace(PTRACE_SETREGSET, pid, (void *)(uintptr_t)NT_PRSTATUS, + &iov); +} + +static int wait_for_breakpoint(pid_t pid) +{ + int status; + + if (waitpid(pid, &status, __WALL) < 0) + return -1; + if (!WIFSTOPPED(status) || WSTOPSIG(status) != SIGTRAP) { + errno = EPROTO; + return -1; + } + return 0; +} + +static int remote_syscall(pid_t pid, long number, uint64_t a0, uint64_t a1, + uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5, + long *result_out) +{ + struct user_regs_struct saved; + struct user_regs_struct call_regs; + struct user_regs_struct stopped; + unsigned long original_word = 0; + unsigned long patched_word; + uintptr_t pc; + int saved_errno; + int have_regs = 0; + int have_word = 0; + + if (ptrace_get_regs(pid, &saved) < 0) + return -1; + have_regs = 1; + call_regs = saved; + pc = (uintptr_t)saved.rip; + errno = 0; + original_word = + (unsigned long)ptrace(PTRACE_PEEKTEXT, pid, (void *)pc, NULL); + if (original_word == (unsigned long)-1 && errno != 0) + goto fail; + have_word = 1; + patched_word = original_word; + ((unsigned char *)&patched_word)[0] = 0x0f; /* syscall */ + ((unsigned char *)&patched_word)[1] = 0x05; + ((unsigned char *)&patched_word)[2] = 0xcc; /* int3 */ + if (ptrace(PTRACE_POKETEXT, pid, (void *)pc, (void *)patched_word) < 0) + goto fail; + call_regs.rax = (uint64_t)number; + call_regs.orig_rax = UINT64_MAX; + call_regs.rdi = a0; + call_regs.rsi = a1; + call_regs.rdx = a2; + call_regs.r10 = a3; + call_regs.r8 = a4; + call_regs.r9 = a5; + if (ptrace_set_regs(pid, &call_regs) < 0) + goto fail; + if (ptrace(PTRACE_CONT, pid, NULL, NULL) < 0) + goto fail; + if (wait_for_breakpoint(pid) < 0) + goto fail; + if (ptrace_get_regs(pid, &stopped) < 0) + goto fail; + *result_out = (long)stopped.rax; + if (ptrace(PTRACE_POKETEXT, pid, (void *)pc, (void *)original_word) < 0) + goto fail; + have_word = 0; + if (ptrace_set_regs(pid, &saved) < 0) + goto fail; + have_regs = 0; + return 0; + +fail: + saved_errno = errno; + if (have_word) + (void)ptrace(PTRACE_POKETEXT, pid, (void *)pc, + (void *)original_word); + if (have_regs) + (void)ptrace_set_regs(pid, &saved); + errno = saved_errno; + return -1; +} + +static int remote_write(pid_t pid, uintptr_t destination, const void *source, + size_t length) +{ + const unsigned char *bytes = source; + size_t offset = 0; + + while (offset < length) { + unsigned long word = 0; + size_t chunk = length - offset; + if (chunk > sizeof(word)) + chunk = sizeof(word); + memcpy(&word, bytes + offset, chunk); + if (ptrace(PTRACE_POKEDATA, pid, (void *)(destination + offset), + (void *)word) < 0) + return -1; + offset += chunk; + } + return 0; +} + +static int raw_syscall_failed(long result) +{ + return result < 0 && result >= -4095; +} + +static int raw_syscall_ok(long result) +{ + if (!raw_syscall_failed(result)) + return 0; + errno = (int)-result; + return -1; +} + +#define FILTER_GATEWAY_BLOCK(syscall_number) \ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, (syscall_number), 0, 4), \ + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, \ + offsetof(struct seccomp_data, args[5])), \ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, GATEWAY_MAGIC_LOW, 1, 0), \ + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_TRAP | FILTER_TAG), \ + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW) + +int fspy_sigsys_prepare(int shm_fd) +{ + int descriptor_flags = fcntl(shm_fd, F_GETFD); + struct sock_filter instructions[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, arch)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, AUDIT_ARCH_X86_64, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, nr)), + FILTER_GATEWAY_BLOCK(SYS_open), + FILTER_GATEWAY_BLOCK(SYS_openat), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + }; + struct sock_fprog program = { + .len = (unsigned short)ARRAY_LEN(instructions), + .filter = instructions, + }; + + if (descriptor_flags < 0) + return -1; + if (fcntl(shm_fd, F_SETFD, descriptor_flags & ~FD_CLOEXEC) < 0) + return -1; + if (ptrace(PTRACE_TRACEME, 0, NULL, NULL) < 0) + return -1; + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) < 0) + return -1; + if (syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, 0, &program) < 0) + return -1; + return 0; +} + +int fspy_sigsys_inject(pid_t pid, int shm_fd, size_t shm_len) +{ + const size_t page_size = (size_t)sysconf(_SC_PAGESIZE); + const size_t blob_size = + (size_t)(fspy_sigsys_blob_end - fspy_sigsys_blob_start); + unsigned char *blob = NULL; + struct kernel_sigaction_wire action = {0}; + uintptr_t remote_code; + uintptr_t remote_action; + uintptr_t remote_shm; + long result; + int status; + int saved_errno; + + if (page_size == 0 || blob_size > page_size || + INSTALL_ACTION_OFFSET + sizeof(action) > page_size || shm_len < 8) { + errno = EINVAL; + return -1; + } + + if (waitpid(pid, &status, __WALL) < 0) + return -1; + if (!WIFSTOPPED(status) || WSTOPSIG(status) != SIGTRAP) { + errno = EPROTO; + return -1; + } + + if (remote_syscall(pid, SYS_mmap, 0, shm_len, PROT_READ | PROT_WRITE, + MAP_SHARED, (uint64_t)shm_fd, 0, &result) < 0 || + raw_syscall_ok(result) < 0) + goto fail; + remote_shm = (uintptr_t)result; + + if (remote_syscall(pid, SYS_mmap, 0, page_size * 2, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, UINT64_MAX, 0, &result) < + 0 || + raw_syscall_ok(result) < 0) + goto fail; + remote_code = (uintptr_t)result; + remote_action = remote_code + page_size + INSTALL_ACTION_OFFSET; + + blob = malloc(blob_size); + if (blob == NULL) + goto fail; + memcpy(blob, fspy_sigsys_blob_start, blob_size); + patch_u64(blob, fspy_sigsys_slot_rax, + offsetof(ucontext_t, uc_mcontext) + + offsetof(mcontext_t, gregs[REG_RAX])); + patch_u64(blob, fspy_sigsys_slot_rdi, + offsetof(ucontext_t, uc_mcontext) + + offsetof(mcontext_t, gregs[REG_RDI])); + patch_u64(blob, fspy_sigsys_slot_rsi, + offsetof(ucontext_t, uc_mcontext) + + offsetof(mcontext_t, gregs[REG_RSI])); + patch_u64(blob, fspy_sigsys_slot_rdx, + offsetof(ucontext_t, uc_mcontext) + + offsetof(mcontext_t, gregs[REG_RDX])); + patch_u64(blob, fspy_sigsys_slot_r10, + offsetof(ucontext_t, uc_mcontext) + + offsetof(mcontext_t, gregs[REG_R10])); + patch_u64(blob, fspy_sigsys_slot_r8, + offsetof(ucontext_t, uc_mcontext) + + offsetof(mcontext_t, gregs[REG_R8])); + patch_u64(blob, fspy_sigsys_slot_shm, remote_shm); + patch_u64(blob, fspy_sigsys_slot_shm_len, shm_len); + patch_u64(blob, fspy_sigsys_slot_magic, GATEWAY_MAGIC); + if (remote_write(pid, remote_code, blob, blob_size) < 0) + goto fail; + free(blob); + blob = NULL; + + action.handler = remote_code + blob_offset(fspy_sigsys_handler); + action.flags = SA_SIGINFO | SA_NODEFER | UINT64_C(0x04000000); + action.restorer = remote_code + blob_offset(fspy_sigsys_restorer); + if (remote_write(pid, remote_action, &action, sizeof(action)) < 0) + goto fail; + + if (remote_syscall(pid, SYS_mprotect, remote_code, page_size, + PROT_READ | PROT_EXEC, 0, 0, 0, &result) < 0 || + raw_syscall_ok(result) < 0 || result != 0) + goto fail; + if (remote_syscall(pid, SYS_rt_sigaction, SIGSYS, remote_action, 0, 8, 0, + GATEWAY_MAGIC, &result) < 0 || + raw_syscall_ok(result) < 0 || result != 0) + goto fail; + if (remote_syscall(pid, SYS_close, (uint64_t)shm_fd, 0, 0, 0, 0, 0, + &result) < 0 || + raw_syscall_ok(result) < 0 || result != 0) + goto fail; + + if (ptrace(PTRACE_DETACH, pid, NULL, NULL) < 0) + return -1; + return 0; + +fail: + saved_errno = errno; + free(blob); + (void)ptrace(PTRACE_KILL, pid, NULL, NULL); + errno = saved_errno; + return -1; +} diff --git a/crates/fspy/src/unix/syscall_handler/execve.rs b/crates/fspy/src/unix/syscall_handler/execve.rs deleted file mode 100644 index d34ac8c2d..000000000 --- a/crates/fspy/src/unix/syscall_handler/execve.rs +++ /dev/null @@ -1,24 +0,0 @@ -use std::io; - -use fspy_seccomp_unotify::supervisor::handler::arg::{CStrPtr, Caller, Fd}; - -use super::SyscallHandler; - -impl SyscallHandler { - fn handle_execve(&mut self, caller: Caller, fd: Fd, path_ptr: CStrPtr) -> io::Result<()> { - // TODO: parse shebangs to track reading interpreters - self.handle_open(caller, fd, path_ptr, libc::O_RDONLY) - } - - pub(super) fn execveat( - &mut self, - caller: Caller, - (fd, path_ptr): (Fd, CStrPtr), - ) -> io::Result<()> { - self.handle_execve(caller, fd, path_ptr) - } - - pub(super) fn execve(&mut self, caller: Caller, (path_ptr,): (CStrPtr,)) -> io::Result<()> { - self.handle_execve(caller, Fd::cwd(), path_ptr) - } -} diff --git a/crates/fspy/src/unix/syscall_handler/getdents.rs b/crates/fspy/src/unix/syscall_handler/getdents.rs deleted file mode 100644 index 45eec5320..000000000 --- a/crates/fspy/src/unix/syscall_handler/getdents.rs +++ /dev/null @@ -1,16 +0,0 @@ -use std::io; - -use fspy_seccomp_unotify::supervisor::handler::arg::{Caller, Fd}; - -use super::SyscallHandler; - -impl SyscallHandler { - #[cfg(target_arch = "x86_64")] - pub(super) fn getdents(&mut self, caller: Caller, (fd,): (Fd,)) -> io::Result<()> { - self.handle_open_dir(caller, fd) - } - - pub(super) fn getdents64(&mut self, caller: Caller, (fd,): (Fd,)) -> io::Result<()> { - self.handle_open_dir(caller, fd) - } -} diff --git a/crates/fspy/src/unix/syscall_handler/mod.rs b/crates/fspy/src/unix/syscall_handler/mod.rs deleted file mode 100644 index 4b6f7947e..000000000 --- a/crates/fspy/src/unix/syscall_handler/mod.rs +++ /dev/null @@ -1,103 +0,0 @@ -mod execve; -mod getdents; -mod open; -mod stat; - -use std::{ - borrow::Cow, - ffi::{OsStr, c_int}, - io, - os::unix::ffi::OsStrExt, - path::{Path, PathBuf}, -}; - -use fspy_seccomp_unotify::{ - impl_handler, - supervisor::handler::arg::{CStrPtr, Caller, Fd}, -}; -use fspy_shared::ipc::{AccessMode, PathAccess}; - -use crate::arena::PathAccessArena; - -const PATH_MAX: usize = libc::PATH_MAX as usize; - -#[derive(Debug)] -pub struct SyscallHandler { - arena: PathAccessArena, - path_read_buf: [u8; PATH_MAX], -} - -impl Default for SyscallHandler { - fn default() -> Self { - Self { arena: PathAccessArena::default(), path_read_buf: [0; PATH_MAX] } - } -} - -impl SyscallHandler { - pub fn into_arena(self) -> PathAccessArena { - self.arena - } - - fn handle_open( - &mut self, - caller: Caller, - dir_fd: Fd, - path_ptr: CStrPtr, - flags: c_int, - ) -> io::Result<()> { - let Some(path_len) = path_ptr.read(caller, &mut self.path_read_buf)? else { - // Ignore paths that are too long to fit in PATH_MAX - return Ok(()); - }; - let mut path = Cow::Borrowed(Path::new(OsStr::from_bytes(&self.path_read_buf[..path_len]))); - if !path.is_absolute() { - let mut resolved_path = PathBuf::from(dir_fd.get_path(caller)?); - if !nix::NixPath::is_empty(path.as_ref()) { - resolved_path.push(&path); - } - path = Cow::Owned(resolved_path); - } - self.arena.add(PathAccess { - mode: match flags & libc::O_ACCMODE { - libc::O_RDWR => AccessMode::READ | AccessMode::WRITE, - libc::O_WRONLY => AccessMode::WRITE, - _ => AccessMode::READ, - }, - path: path.as_os_str().into(), - }); - Ok(()) - } - - fn handle_open_dir(&mut self, caller: Caller, fd: Fd) -> io::Result<()> { - let path = fd.get_path(caller)?; - self.arena.add(PathAccess { - mode: AccessMode::READ_DIR, - path: OsStr::from_bytes(path.as_bytes()).into(), - }); - Ok(()) - } -} - -impl_handler!( - SyscallHandler: - - #[cfg(target_arch = "x86_64")] open, - openat, - openat2, - - #[cfg(target_arch = "x86_64")] getdents, - getdents64, - - #[cfg(target_arch = "x86_64")] stat, - #[cfg(target_arch = "x86_64")] lstat, - #[cfg(target_arch = "x86_64")] newfstatat, - #[cfg(target_arch = "aarch64")] fstatat, - statx, - - #[cfg(target_arch = "x86_64")] access, - faccessat, - faccessat2, - - execve, - execveat, -); diff --git a/crates/fspy/src/unix/syscall_handler/open.rs b/crates/fspy/src/unix/syscall_handler/open.rs deleted file mode 100644 index be7ae157e..000000000 --- a/crates/fspy/src/unix/syscall_handler/open.rs +++ /dev/null @@ -1,35 +0,0 @@ -use std::{ffi::c_int, io}; - -use fspy_seccomp_unotify::supervisor::handler::arg::{CStrPtr, Caller, Fd, Ptr}; - -use super::SyscallHandler; - -impl SyscallHandler { - #[cfg(target_arch = "x86_64")] - pub(super) fn open( - &mut self, - caller: Caller, - (path, flags): (CStrPtr, c_int), - ) -> io::Result<()> { - self.handle_open(caller, Fd::cwd(), path, flags) - } - - pub(super) fn openat( - &mut self, - caller: Caller, - (dir_fd, path, flags): (Fd, CStrPtr, c_int), - ) -> io::Result<()> { - self.handle_open(caller, dir_fd, path, flags) - } - - pub(super) fn openat2( - &mut self, - caller: Caller, - // open_how is a pointer to struct `open_how`, but we only care about flags here, so use `Ptr` - (dir_fd, path, open_how): (Fd, CStrPtr, Ptr), - ) -> io::Result<()> { - // SAFETY: open_how is a valid pointer to struct `open_how` in the target process, which has `flags` as the first field of type `u64` - let flags = unsafe { open_how.read(caller) }?; - self.handle_open(caller, dir_fd, path, c_int::try_from(flags).unwrap_or(libc::O_RDWR)) - } -} diff --git a/crates/fspy/src/unix/syscall_handler/stat.rs b/crates/fspy/src/unix/syscall_handler/stat.rs deleted file mode 100644 index 40d9f76f1..000000000 --- a/crates/fspy/src/unix/syscall_handler/stat.rs +++ /dev/null @@ -1,68 +0,0 @@ -use std::io; - -use fspy_seccomp_unotify::supervisor::handler::arg::{CStrPtr, Caller, Fd}; - -use super::SyscallHandler; - -impl SyscallHandler { - #[cfg(target_arch = "x86_64")] - pub(super) fn stat(&mut self, caller: Caller, (path,): (CStrPtr,)) -> io::Result<()> { - self.handle_open(caller, Fd::cwd(), path, libc::O_RDONLY) - } - - #[cfg(target_arch = "x86_64")] - pub(super) fn lstat(&mut self, caller: Caller, (path,): (CStrPtr,)) -> io::Result<()> { - self.handle_open(caller, Fd::cwd(), path, libc::O_RDONLY) - } - - #[cfg(target_arch = "aarch64")] - pub(super) fn fstatat( - &mut self, - caller: Caller, - (dir_fd, path_ptr): (Fd, CStrPtr), - ) -> io::Result<()> { - self.handle_open(caller, dir_fd, path_ptr, libc::O_RDONLY) - } - - #[cfg(target_arch = "x86_64")] - pub(super) fn newfstatat( - &mut self, - caller: Caller, - (dir_fd, path_ptr): (Fd, CStrPtr), - ) -> io::Result<()> { - self.handle_open(caller, dir_fd, path_ptr, libc::O_RDONLY) - } - - /// statx(2) — modern replacement for stat/fstatat used by newer glibc. - pub(super) fn statx( - &mut self, - caller: Caller, - (dir_fd, path_ptr): (Fd, CStrPtr), - ) -> io::Result<()> { - self.handle_open(caller, dir_fd, path_ptr, libc::O_RDONLY) - } - - /// access(2) — check file accessibility (e.g. existsSync in Node.js). - #[cfg(target_arch = "x86_64")] - pub(super) fn access(&mut self, caller: Caller, (path,): (CStrPtr,)) -> io::Result<()> { - self.handle_open(caller, Fd::cwd(), path, libc::O_RDONLY) - } - - /// faccessat(2) — check file accessibility relative to directory fd. - pub(super) fn faccessat( - &mut self, - caller: Caller, - (dir_fd, path_ptr): (Fd, CStrPtr), - ) -> io::Result<()> { - self.handle_open(caller, dir_fd, path_ptr, libc::O_RDONLY) - } - - /// faccessat2(2) — extended faccessat with flags parameter. - pub(super) fn faccessat2( - &mut self, - caller: Caller, - (dir_fd, path_ptr): (Fd, CStrPtr), - ) -> io::Result<()> { - self.handle_open(caller, dir_fd, path_ptr, libc::O_RDONLY) - } -} diff --git a/crates/fspy_benchmark/README.md b/crates/fspy_benchmark/README.md index 7d722b214..de74de285 100644 --- a/crates/fspy_benchmark/README.md +++ b/crates/fspy_benchmark/README.md @@ -5,7 +5,7 @@ Measures what fspy costs a process it tracks, and whether a change to fspy moved - `launch`: the wall clock of a tracked launch that opens nothing. This is the cost of starting a process under tracking: injection, session setup, and teardown. - `access`: how long a batch of opens takes, timed by two threads that each open their own path. This is the cost of interception itself, under the concurrency a tracked process normally has. -Linux measures a dynamically linked target (`LD_PRELOAD`) and a `x86_64-unknown-linux-musl` target (seccomp user notification). macOS measures `DYLD_INSERT_LIBRARIES`. Windows measures Detours injection. +Linux measures both a dynamically linked target and a `x86_64-unknown-linux-musl` target. Those labels describe the workload, not the interception backend, so the same rows can compare backend changes. macOS measures `DYLD_INSERT_LIBRARIES`. Windows measures Detours injection. Run the overhead report locally with: diff --git a/crates/fspy_preload_unix/src/lib.rs b/crates/fspy_preload_unix/src/lib.rs index 42bf9e9cb..cd5c207ed 100644 --- a/crates/fspy_preload_unix/src/lib.rs +++ b/crates/fspy_preload_unix/src/lib.rs @@ -1,13 +1,11 @@ -// Compile as an empty crate on non-unix targets and on musl (where seccomp -// alone handles access tracking). Guarding the feature gate keeps rustc from -// warning about unused features on those targets. -#![cfg_attr(all(unix, not(target_env = "musl")), feature(c_variadic))] +// Linux uses the ptrace/SIGSYS backend. Keep this cdylib only for macOS. +#![cfg_attr(target_os = "macos", feature(c_variadic))] -#[cfg(all(unix, not(target_env = "musl")))] +#[cfg(target_os = "macos")] mod client; -#[cfg(all(unix, not(target_env = "musl")))] +#[cfg(target_os = "macos")] mod interceptions; -#[cfg(all(unix, not(target_env = "musl")))] +#[cfg(target_os = "macos")] mod libc; -#[cfg(all(unix, not(target_env = "musl")))] +#[cfg(target_os = "macos")] mod macros; diff --git a/crates/fspy_shared/src/ipc/channel/mod.rs b/crates/fspy_shared/src/ipc/channel/mod.rs index ae34c8ca6..6558e9419 100644 --- a/crates/fspy_shared/src/ipc/channel/mod.rs +++ b/crates/fspy_shared/src/ipc/channel/mod.rs @@ -2,6 +2,8 @@ mod shm_io; +#[cfg(target_os = "linux")] +use std::os::fd::{AsFd, BorrowedFd}; use std::{env::temp_dir, fs::File, io, mem::MaybeUninit, ops::Deref, path::PathBuf, sync::Arc}; use fspy_shm::Shm; @@ -175,6 +177,20 @@ impl Receiver { let reader = ShmReader::new(unsafe { self.shm.as_slice() }); Ok(ReceiverLockGuard { reader, lock_file: &self.lock_file }) } + + /// Returns the channel's backing memfd for mapping into a Linux tracee. + #[cfg(target_os = "linux")] + #[must_use] + pub fn shm_fd(&self) -> BorrowedFd<'_> { + self.shm.as_fd() + } + + /// Returns the channel's mapped capacity. + #[cfg(target_os = "linux")] + #[must_use] + pub fn shm_len(&self) -> usize { + self.shm.len() + } } pub struct ReceiverLockGuard<'a> { diff --git a/crates/fspy_shared/src/ipc/mod.rs b/crates/fspy_shared/src/ipc/mod.rs index c7236e5d6..e272e1261 100644 --- a/crates/fspy_shared/src/ipc/mod.rs +++ b/crates/fspy_shared/src/ipc/mod.rs @@ -1,4 +1,3 @@ -#[cfg(not(target_env = "musl"))] pub mod channel; mod native_path; use std::fmt::Debug; diff --git a/crates/fspy_shared_unix/Cargo.toml b/crates/fspy_shared_unix/Cargo.toml index 38b9301bb..99ba43a77 100644 --- a/crates/fspy_shared_unix/Cargo.toml +++ b/crates/fspy_shared_unix/Cargo.toml @@ -18,7 +18,6 @@ stackalloc = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] elf = { workspace = true } -fspy_seccomp_unotify = { workspace = true, features = ["target"] } memmap2 = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] diff --git a/crates/fspy_shared_unix/src/payload.rs b/crates/fspy_shared_unix/src/payload.rs index 5267bd42e..7bd5f2438 100644 --- a/crates/fspy_shared_unix/src/payload.rs +++ b/crates/fspy_shared_unix/src/payload.rs @@ -2,29 +2,22 @@ use std::os::unix::ffi::OsStringExt; use base64::{Engine as _, prelude::BASE64_STANDARD_NO_PAD}; use bstr::BString; -#[cfg(not(target_env = "musl"))] +#[cfg(target_os = "macos")] use fspy_shared::ipc::NativeStr; -#[cfg(not(target_env = "musl"))] +#[cfg(target_os = "macos")] use fspy_shared::ipc::channel::ChannelConf; use wincode::{SchemaRead, SchemaWrite}; #[derive(Debug, SchemaWrite, SchemaRead)] pub struct Payload { - #[cfg(not(target_env = "musl"))] + #[cfg(target_os = "macos")] pub ipc_channel_conf: ChannelConf, - #[cfg(not(target_env = "musl"))] + #[cfg(target_os = "macos")] pub preload_path: Box, #[cfg(target_os = "macos")] pub artifacts: Artifacts, - - #[cfg(target_os = "linux")] - #[cfg_attr( - not(target_env = "musl"), - expect(clippy::struct_field_names, reason = "descriptive field name for clarity") - )] - pub seccomp_payload: fspy_seccomp_unotify::payload::SeccompPayload, } #[cfg(target_os = "macos")] diff --git a/crates/fspy_shared_unix/src/spawn/linux/mod.rs b/crates/fspy_shared_unix/src/spawn/linux/mod.rs index d3197da00..40ec4b7dc 100644 --- a/crates/fspy_shared_unix/src/spawn/linux/mod.rs +++ b/crates/fspy_shared_unix/src/spawn/linux/mod.rs @@ -1,63 +1,32 @@ -#[cfg(not(target_env = "musl"))] -use std::{ffi::OsStr, os::unix::ffi::OsStrExt as _, path::Path}; +use std::convert::Infallible; -use fspy_seccomp_unotify::{payload::SeccompPayload, target::install_target}; -#[cfg(not(target_env = "musl"))] -use memmap2::Mmap; - -#[cfg(not(target_env = "musl"))] -use crate::{ - elf, - exec::{append_path_env, ensure_env}, - open_exec::open_executable, -}; use crate::{ exec::Exec, payload::{EncodedPayload, PAYLOAD_ENV_NAME}, }; -const LD_PRELOAD: &str = "LD_PRELOAD"; - -pub struct PreExec(SeccompPayload); +pub struct PreExec(Infallible); impl PreExec { - /// Installs the seccomp unotify filter for the current process. + /// Linux command preparation is performed by fspy's ptrace/SIGSYS path. /// /// # Errors /// - /// Returns an error if the seccomp filter installation fails. - pub fn run(&self) -> nix::Result<()> { - install_target(&self.0) + /// This function is unreachable because Linux never constructs `PreExec`. + pub const fn run(&self) -> nix::Result<()> { + match self.0 {} } } +#[expect( + clippy::unnecessary_wraps, + reason = "keeps the platform-specific command preparation signature uniform" +)] pub fn handle_exec( command: &mut Exec, - encoded_payload: &EncodedPayload, + _encoded_payload: &EncodedPayload, ) -> nix::Result> { - // On musl targets, LD_PRELOAD is not available (cdylib not supported). - // Always use seccomp-based tracking instead. - #[cfg(not(target_env = "musl"))] - { - let executable_fd = open_executable(Path::new(OsStr::from_bytes(&command.program)))?; - // SAFETY: The file descriptor is valid and we only read from the mapping. - let executable_mmap = unsafe { Mmap::map(&executable_fd) }.map_err(|io_error| { - nix::Error::try_from(io_error).unwrap_or(nix::Error::UnknownErrno) - })?; - if elf::is_dynamically_linked_to_libc(executable_mmap)? { - // Append (don't overwrite) so a user-provided LD_PRELOAD keeps - // working. fspy's shim goes last so user preloads that - // short-circuit a libc call stay invisible to fspy — what the - // OS actually executed is what we want to record. - append_path_env( - &mut command.envs, - LD_PRELOAD, - encoded_payload.payload.preload_path.as_os_str().as_bytes(), - ); - ensure_env(&mut command.envs, PAYLOAD_ENV_NAME, &encoded_payload.encoded_string)?; - return Ok(None); - } - } - - command.envs.retain(|(name, _)| name != LD_PRELOAD && name != PAYLOAD_ENV_NAME); - Ok(Some(PreExec(encoded_payload.payload.seccomp_payload.clone()))) + // Do not leak a payload from an outer fspy session into this command. A + // user-supplied LD_PRELOAD is left untouched; this backend never adds one. + command.envs.retain(|(name, _)| name != PAYLOAD_ENV_NAME); + Ok(None) } diff --git a/crates/fspy_shm/src/linux/mod.rs b/crates/fspy_shm/src/linux/mod.rs index f0abf020c..ac6405c55 100644 --- a/crates/fspy_shm/src/linux/mod.rs +++ b/crates/fspy_shm/src/linux/mod.rs @@ -2,7 +2,11 @@ mod broker; -use std::{io, os::fd::OwnedFd, slice}; +use std::{ + io, + os::fd::{AsFd, BorrowedFd, OwnedFd}, + slice, +}; use memfd::{FileSeal, MemfdOptions}; use memmap2::{MmapOptions, MmapRaw}; @@ -13,6 +17,9 @@ use tokio_util::sync::DropGuard; pub struct Shm { id: String, mapping: MmapRaw, + /// Kept open so an owner may pass the mapping into a process stopped at + /// exec. The broker owns a second descriptor. + memfd: OwnedFd, /// Stops the owner's broker on drop. `None` for opened views. _service: Option, } @@ -47,11 +54,13 @@ pub fn create(size: usize) -> io::Result { .add_seals(&[FileSeal::SealGrow, FileSeal::SealShrink, FileSeal::SealSeal]) .map_err(memfd_error)?; let mapping = MmapOptions::new().len(size).map_raw(memfd.as_file())?; - let memfd: OwnedFd = memfd.into_file().into(); - let (id, service, guard) = broker::new(memfd)?; + let memfd = memfd.into_file(); + let broker_memfd: OwnedFd = memfd.try_clone()?.into(); + let memfd: OwnedFd = memfd.into(); + let (id, service, guard) = broker::new(broker_memfd)?; runtime.spawn(service); - Ok(Shm { id, mapping, _service: Some(guard) }) + Ok(Shm { id, mapping, memfd, _service: Some(guard) }) } /// Opens a view of the memfd mapping identified by `id` through its broker. @@ -73,7 +82,7 @@ pub fn open(id: &str) -> io::Result { return Err(io::Error::new(io::ErrorKind::InvalidData, "shared-memory size is zero")); } let mapping = MmapOptions::new().len(size).map_raw(&memfd)?; - Ok(Shm { id: id.to_owned(), mapping, _service: None }) + Ok(Shm { id: id.to_owned(), mapping, memfd, _service: None }) } fn valid_size(size: usize) -> io::Result { @@ -128,3 +137,9 @@ impl Shm { unsafe { slice::from_raw_parts(self.mapping.as_ptr(), self.mapping.len()) } } } + +impl AsFd for Shm { + fn as_fd(&self) -> BorrowedFd<'_> { + self.memfd.as_fd() + } +} diff --git a/crates/vite_task_bin/tests/e2e_snapshots/fixtures/vitest_browser_cache/vitest.config.js b/crates/vite_task_bin/tests/e2e_snapshots/fixtures/vitest_browser_cache/vitest.config.js index 1604a770d..864dcc0d6 100644 --- a/crates/vite_task_bin/tests/e2e_snapshots/fixtures/vitest_browser_cache/vitest.config.js +++ b/crates/vite_task_bin/tests/e2e_snapshots/fixtures/vitest_browser_cache/vitest.config.js @@ -6,6 +6,8 @@ class NoTestSummaryReporter extends DefaultReporter { reportTestSummary() {} } +const chromiumSandbox = process.env.VITEST_CHROMIUM_SANDBOX === 'true'; + export default defineConfig({ test: { reporters: [new NoTestSummaryReporter({ summary: false }), 'json'], @@ -13,7 +15,12 @@ export default defineConfig({ browser: { enabled: true, headless: true, - provider: playwright(), + provider: playwright({ + launchOptions: { + chromiumSandbox, + args: chromiumSandbox ? ['--enable-logging=stderr', '--v=1'] : [], + }, + }), instances: [{ browser: 'chromium' }], }, }, diff --git a/docs/fspy-linux-sigsys-research.md b/docs/fspy-linux-sigsys-research.md new file mode 100644 index 000000000..3057c52c9 --- /dev/null +++ b/docs/fspy-linux-sigsys-research.md @@ -0,0 +1,400 @@ +# Linux SIGSYS interception for fspy + +Research date: 2026-08-02 + +Status: feasibility proven on native Linux AArch64 and x86-64, including default Vitest browser mode and Docker's default seccomp and AppArmor profiles. Playwright's `chromiumSandbox: true` is a confirmed compatibility boundary for the current ptrace exec bridge. + +Primary audience: fspy maintainers deciding whether to replace the Linux `LD_PRELOAD` and seccomp user-notification backends. + +## Decision + +The design is feasible for fspy's unprivileged build-tool workload. Use `SECCOMP_RET_TRAP` and a freestanding in-process `SIGSYS` handler for Linux file-system interception. It observes libc calls and direct syscalls without the per-access process switch required by seccomp user notification. + +Use two exec bridges, selected by a startup probe: + +1. Prefer a temporary ptrace attachment around a real target exec. This preserves kernel ELF, script, identity, and failure semantics. +2. Do not use that bridge for Chromium's namespace-sandbox zygote exec. The current bridge breaks Chromium's synchronous zygote boot handshake before Chromium installs its seccomp policy. +3. If ptrace is denied, already owned, or incompatible with a namespace-sandbox launch, real-exec a static `fspy_host` and load the target in user space. This path works for the tested frontend workload but has a narrower, documented exec contract. Sandboxed Chromium still needs a dedicated userland-loader validation. + +The preferred ptrace bridge is: + +1. The in-process handler traps `execve` or `execveat`. +2. The handler asks the existing fspy supervisor to attach to that thread with `PTRACE_SEIZE` and `PTRACE_O_TRACEEXEC`. +3. The handler explicitly unblocks physical `SIGSYS`, then reissues the original syscall through a trusted gateway. +4. Linux performs the requested exec and stops at `PTRACE_EVENT_EXEC` before target code runs. +5. The supervisor advances once to the pending exec syscall-exit stop. +6. The supervisor maps the handler island into the new address space, reinstalls the physical `SIGSYS` action, and detaches. +7. File-system syscalls run with no tracer attached. Their `SIGSYS` handling stays in process. + +Keep the custom loader in-house. The reference loaders were useful for finding requirements, but neither is suitable for production. Current esbuild 0.28.1, Node, shells, glibc, static musl, and static Go all passed a pure userland handoff after correcting reference-loader defects. + +Do not replace both backends yet. Signal virtualization and async-signal-safe event recording remain open engineering work; syscall trapping and frontend compatibility have working proofs. + +## Recommended architecture + +```mermaid +flowchart TD + A["Tracked file syscall"] --> B["seccomp RET_TRAP"] + B --> C["In-process SIGSYS handler"] + C --> D["Write raw event to shared memory"] + D --> E["Reissue syscall with gateway marker"] + E --> F["Return result through ucontext"] + + X["Tracked exec syscall"] --> B + C -->|"exec only"| G["Notify supervisor"] + G --> H["Temporary PTRACE_SEIZE"] + H --> I["Real kernel exec"] + I --> J["PTRACE_EVENT_EXEC before target entry"] + J --> K["Exec syscall-exit rendezvous"] + K --> L["Map handler, install SIGSYS action, detach"] + L --> P["Target starts with no active tracer"] + + G -->|"ptrace denied or occupied"| M["Real exec of static fspy_host"] + M --> N["Earliest-entry handler bootstrap"] + N --> O["In-house userland ELF handoff"] +``` + +The ptrace attachment must be temporary. A permanently traced process stops in the tracer on every signal delivery. That would turn every seccomp-generated `SIGSYS` back into a cross-process operation and remove the main performance benefit. See [`ptrace(2)`](https://man7.org/linux/man-pages/man2/ptrace.2.html). + +## What the prototypes established + +The primary experiments ran on Ubuntu 24.04 AArch64, Linux 6.8, in a four-vCPU Lima VM. The same syscall and injection probes passed on a native Ubuntu 24.04 x86-64 GitHub runner and inside both rootless containerd and Docker containers with an existing seccomp filter. The Docker case also had `no_new_privs=1` and `docker-default (enforce)` AppArmor. + +| Question | Result | +| ------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------- | +| Can `RET_TRAP` catch direct and libc syscalls? | Yes; dynamic glibc, static glibc, and static musl probes passed | +| Can the handler execute the denied syscall? | Yes; the sixth-argument gateway passed natively on AArch64 and x86-64 | +| Can it return `EFAULT` instead of crashing on a bad pointer? | Yes, using self `process_vm_readv`/`process_vm_writev` | +| Can nested and concurrent traps work? | Yes; `SA_NODEFER`, a nested trap, and 200,000 calls from four threads passed | +| Can Go replace or block `SIGSYS` without breaking fspy? | The prototype virtualized the tested `rt_sigaction` and `rt_sigprocmask` operations; esbuild passed | +| Can a real exec regain the handler before target entry? | Yes; post-exec ptrace injection passed for dynamic, static, non-leader, and esbuild execs | +| Can a recursive on-demand ptrace bridge run Vitest browser mode? | Yes with Playwright's default `--no-sandbox`; nine exec reinjections passed on native x86-64 | +| Does the same bridge support `chromiumSandbox: true`? | No; the namespace zygote handshake reaches EOF at its ptrace exec boundary before Chromium seccomp | +| Can the proposed static-host cycle bootstrap under the inherited filter? | Yes; a real exec into a static-musl second stage reinstalled the handler through the trusted gateway | +| Can a pure handoff run frontend tools? | Yes; Node and esbuild 0.28.1 CLI/API paths, static Go, static musl, shells, and coreutils passed | + +The latest-esbuild ptrace experiment is the strongest combined result. It real-execed the static AArch64 esbuild 0.28.1 binary, injected a 360-byte prototype handler, detached before entry, survived Go's signal initialization and threads, intercepted the input `openat`, and produced a working bundle. The handler used an RWX page and omitted several signal cases. The result proves ordering and compatibility, while W^X mapping and complete virtualization remain production work. + +## Why RET_TRAP works + +For `SECCOMP_RET_TRAP`, Linux does not execute the original syscall. It sends a thread-directed `SIGSYS` with the syscall number, architecture, instruction address, and the filter's 16-bit data value. Execution resumes after the syscall instruction, so the handler must write the raw syscall result into the saved architecture return register. See the [kernel seccomp filter documentation](https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html), [`seccomp(2)`](https://man7.org/linux/man-pages/man2/seccomp.2.html), and the [kernel implementation](https://github.com/torvalds/linux/blob/master/kernel/seccomp.c#L1259). + +The filter is inherited through `fork` and `clone` and preserved across `execve`. Unlike `SECCOMP_FILTER_FLAG_NEW_LISTENER`, an ordinary `RET_TRAP` filter can be stacked with an existing filter. + +### Use the unused sixth argument as the trusted gateway marker + +Every syscall in fspy's present interception set uses at most five arguments. The handler can place a random per-session 64-bit marker in the sixth raw argument register, issue the syscall from handwritten assembly, and let the BPF filter allow a tracked syscall when that marker matches. + +On x86-64 the marker is placed in `r9`. On AArch64 it is placed in `x5`. + +This design has three useful properties: + +- It is independent of ASLR and the gateway instruction address. +- It continues to work after `fork` and after a handler island is injected into a new exec image. +- The marker remains inside the handler call. `rt_sigreturn` restores the target's original registers, so the marker does not leak into the target's next syscall. + +The marker is an accidental-bypass guard, not a security boundary. Target code can discover and reuse it. fspy observes accesses; it does not sandbox the process. + +Six-argument syscalls need a different gateway rule. The current file and exec syscall set does not have this problem. Future interception of `mmap`, `pselect6`, or similar calls would require an instruction-address exception or another protocol. + +## SIGSYS virtualization is required + +A physical fspy handler requires virtualization to coexist with arbitrary target signal state. + +Linux force-delivers a seccomp `SIGSYS`. If the signal is blocked or ignored, the kernel changes its disposition to `SIG_DFL`, unblocks it, and delivers it. The process dies instead of leaving the signal pending. This behavior appears in [`force_sig_info_to_task`](https://github.com/torvalds/linux/blob/master/kernel/signal.c#L1280) and [`force_sig_seccomp`](https://github.com/torvalds/linux/blob/master/kernel/signal.c#L1811). + +Go exercises this behavior during normal startup. Its runtime installs signal actions whose masks include `SIGSYS`, and it masks signals around thread creation. A backend intended to run esbuild must virtualize those operations. + +The handler island must implement at least this contract: + +- Install the physical handler with `SA_SIGINFO | SA_NODEFER`. `SA_NODEFER` is necessary because a gateway call can trigger a second `SIGSYS` from an outer or target-installed seccomp filter. +- Intercept every `rt_sigaction` call. Keep a logical target action for `SIGSYS`, keep the physical fspy action in the kernel, and remove `SIGSYS` from every physical handler mask. A different target handler can otherwise block physical `SIGSYS` while it runs. +- Intercept `rt_sigprocmask`. Keep the logical `SIGSYS` mask per thread, but never block it in the kernel. +- Return logical values from `rt_sigaction(..., oldact)` and `rt_sigprocmask(..., oldset)`. +- Identify fspy traps with `si_code == SYS_SECCOMP` and a dedicated `SECCOMP_RET_DATA` tag. +- Dispatch non-fspy `SIGSYS` events to the logical target action. +- Leave `rt_sigreturn` untrapped. + +The prototype used one logical mask for the process. Production needs a per-TID mask table and inheritance rules for `clone`/`clone3`; Go changes masks around thread creation. Lifecycle syscalls may need their own rare, supervisor-assisted slow path because reissuing a thread-creating `clone` from a C signal frame is not generally safe when the child receives a new stack. + +The first implementation can declare narrower behavior for `signalfd`, `sigwait`, temporary-mask syscalls such as `pselect6` and `epoll_pwait`, target-written `ucontext.uc_sigmask`, direct `rt_sigreturn`, `CLONE_CLEAR_SIGHAND`, and `SIGSYS` queued by another process. These cases require deeper signal emulation. + +### Do not take ownership of the target alternate signal stack + +Linux provides one alternate signal stack per thread. Reserving it for fspy breaks target handlers that use `SA_ONSTACK`, including runtimes that use an alternate stack for overflow or fault handling. + +The production handler should run on the interrupted target stack and keep its stack use bounded. A userland loader must transfer control only after the target stack is valid. If a dedicated fspy alternate stack is retained as an optional hardening mode, the implementation must virtualize `sigaltstack` and target `SA_ONSTACK` delivery. + +## Async-signal-safe event recording + +The handler cannot call the current preload client. It must avoid libc, allocation, TLS, locks, unwinding, and callbacks into the target runtime. + +The injected artifact can be written in freestanding Rust. The [Rust injected-runtime design](fspy-rust-injected-runtime.md) includes a cross-compiled relocation-free blob, raw syscall and restorer assembly, a separate state ABI, and the fixed-capacity lock-free allocator decision. Allocation remains prohibited in the `SIGSYS` fast path. + +Use a target-independent shared-memory ABI with fixed-size records and a bounded byte area for paths. Reserve records with lock-free atomics; wake the supervisor with raw `futex` or `eventfd` operations. If the ring fills, block on a raw primitive and never drop a cache-relevant access. Partitioning record lanes by TID reduces contention and simplifies nested delivery. + +Copy target pointers with bounded self `process_vm_readv`. Direct loads can turn a target `EFAULT` into a recursive fault in the handler. The full `openat` prototype copied the path this way and reissued the syscall, but it did not normalize the path or record an event. Production must resolve cwd/dirfd state before returning to the target or capture enough stable fd identity for the supervisor; deferring a raw pointer or fd introduces close/reuse races. + +## Real exec with a temporary ptrace attachment + +`PTRACE_EVENT_EXEC` occurs after Linux has installed the new image and reset exec-owned state, but before the new program executes an instruction. It also occurs before the pending exec syscall finishes returning. The ordering is visible in [`fs/exec.c`](https://github.com/torvalds/linux/blob/master/fs/exec.c#L1747) and the [x86-64 syscall return path](https://github.com/torvalds/linux/blob/master/arch/x86/entry/syscall_64.c). + +The exec handler can use this sequence: + +1. Write an exec request containing the current TID and logical signal state to a preinitialized channel. +2. Wait for the supervisor to call `PTRACE_SEIZE` with `PTRACE_O_TRACEEXEC`. +3. Unblock physical `SIGSYS`, then reissue the original `execve` or `execveat` with the sixth-argument gateway marker. Preserve the original path, argv, environment, fd, and flags. +4. On success, handle the exec event under the post-exec thread-group-leader TID. `PTRACE_GETEVENTMSG` reports the former TID for a nonleader exec. +5. Resume once with `PTRACE_SYSCALL` and `PTRACE_O_TRACESYSGOOD`, then require the pending exec syscall-exit stop. This prevents the late exec return from overwriting registers prepared for the first injected syscall; x86-64 uses `rax` for both the syscall number and return value. +6. Remote-map a sealed, position-independent handler artifact and its state mapping. +7. Install the physical `SIGSYS` action and force the physical mask to unblock `SIGSYS`. +8. Restore the target's entry registers and any instruction bytes overwritten for injection. +9. Detach with no delivered signal. + +Remote injection syscalls are also evaluated by inherited seccomp filters. Set the gateway marker on remote `mmap`, `mprotect`, and `rt_sigaction` calls that fspy itself traps. A stronger target or outer filter can still reject an operation. + +If exec fails, the old address space and signal frame remain. The handler reports the failure, the supervisor uses `PTRACE_INTERRUPT`, detaches at the resulting stop, and wakes the handler to return the raw exec error. This path must not leave a failed exec caller traced. + +The AArch64 proof passed dynamic and static targets, 20 repeated launches, sanitizers, and a non-leader pthread exec. At non-leader exec, Linux reported the event under the thread-group leader TID and `PTRACE_GETEVENTMSG` returned the former worker TID, so the supervisor must re-key per-thread state. The latest-esbuild proof measured the interval from `PTRACE_EVENT_EXEC` to detach over 30 runs: 78.7 microseconds p50, 101.8 microseconds p95, and 84.7 microseconds mean. This is the exec-only cost; no tracer remained for file syscalls. + +The recursive x86-64 proof now exercises the production-shaped success path end to end. The handler sends a queued real-time signal containing its TID and a release-word address, waits in a futex, and lets an ancestor supervisor attach with on-demand `PTRACE_SEIZE`. The supervisor releases the handler, observes `PTRACE_EVENT_EXEC`, injects an RX code page plus a separate RW state page, and detaches. No inherited control fd is required, so `posix_spawn` close actions cannot sever the bridge. Failed-exec behavior is implemented but still needs a dedicated compatibility matrix. + +A subtle signal-mask rule is mandatory. `SIGSYS` is automatically blocked while its handler runs. If that handler successfully calls `execve`, there is no later `rt_sigreturn` to restore the old mask, and the new image inherits `SIGSYS` blocked. Its first trapped syscall is then fatal. The handler must explicitly unblock physical `SIGSYS` immediately before the gateway exec. This was the only ptrace-protocol defect exposed by the shell-to-Node startup chain. + +The native x86-64 [Vitest browser validation](https://github.com/voidzero-dev/vite-task/actions/runs/30735989574) ran the repository's real fixture with Node 22.19.0, Vitest 4.1.10, `@vitest/browser-playwright` 4.1.10, Playwright 1.61.1, and Chrome Headless Shell 149.0.7827.55. It reinjected nine successful images—dash, sed, dirname, uname, Node, and four Chromium processes—with zero failed execs. The browser test and JSON report passed while the inherited filter trapped and reissued the representative file-syscall set: `openat`, `openat2`, `newfstatat`, `statx`, `getdents64`, `faccessat`, and `faccessat2`. + +This proves compatibility with Vitest's default Playwright Chromium launch, which disables Chromium's sandbox unless `chromiumSandbox: true` is requested. + +### `chromiumSandbox: true` fails at the namespace-zygote exec boundary + +The current ptrace exec bridge does not support Playwright's `chromiumSandbox: true`. This is a confirmed negative result, not a missing host prerequisite. + +The native Ubuntu 24.04 validation established these controls: + +- A direct Playwright launch with `chromiumSandbox: true` passed. +- The same direct launch under `setpriv --no-new-privs` passed. +- The full bridge continued to pass Vitest with Playwright's default `chromiumSandbox: false`. +- The sandboxed launch used no `--no-sandbox` argument. + +Ubuntu 24.04's AppArmor policy restricts unprivileged user namespaces by default on the GitHub runner. The validation explicitly set `kernel.apparmor_restrict_unprivileged_userns=0`. Without that host setup, Chromium fails earlier with `No usable sandbox`; that is a separate environment failure. + +With the host prerequisite satisfied, the bridged launch fails in this order: + +1. The bridge injects the main Chrome image successfully. +2. Chrome creates its namespace zygote and the bridge injects that exec successfully. `/proc//fd/3` still reports the inherited Unix socket. +3. The browser's blocking `recvmsg` returns zero at the zygote exec boundary. Chromium fails `ReceiveFixedMessage` at `zygote_host_impl_linux.cc:207` before receiving `ZYGOTE_BOOT`. +4. The zygote reaches `ZygoteMain` after detach. Its later control-socket write reports `EPIPE` because the browser has already abandoned the handshake. + +The [full-filter diagnostic run](https://github.com/voidzero-dev/vite-task/actions/runs/30737025514) captured the zero-length receive, the preserved fd 3, and the later broken pipe. An [exec-and-signal-only filter run](https://github.com/voidzero-dev/vite-task/actions/runs/30737143580) failed at the same point. The representative file-syscall traps are therefore not the cause. + +Chromium's own seccomp policy is not active at this failure point. Forwarding a foreign `SECCOMP_RET_TRAP` to the target's logical `SIGSYS` handler and installing fspy's physical handler with `SA_NODEFER` are still required. The standalone nested-filter test validates those mechanics, but they do not fix this earlier zygote handshake. + +Treat namespace-sandbox zygote exec as a ptrace incompatibility until a different rendezvous proves otherwise. A production hybrid should route this exec through the static-host userland loader or decline tracing with an actionable error. It must make that choice before attempting the ptrace exec because Chromium treats the failed boot handshake as fatal. + +This path preserves the parts of exec that frontend tools depend on: + +- kernel ELF, script, and binfmt loading; +- static and dynamic executables, including Go binaries; +- destruction of sibling threads; +- `vfork` parent release; +- close-on-exec and file-table unsharing; +- target `/proc/self/exe`, auxv, comm, memory layout, and brk state; +- kernel handling of ELF properties and the platform dynamic linker. + +Installing an unprivileged seccomp filter requires `no_new_privs`, which puts target set-user-ID, file-capability elevation, and similar privileged exec transitions outside the supported contract. + +### Ptrace limitations + +Use a startup capability probe and choose the fallback before doing substantial work. Temporary attachment can fail when: + +- another debugger, `strace`, or `rr` owns the one ptrace relationship; +- Yama is in a restrictive mode; +- `PR_SET_DUMPABLE(0)` or a credential transition prevents attachment; +- AppArmor, SELinux, a container profile, or a sandbox denies ptrace; +- a descendant is no longer in an allowed ancestor relationship with the supervisor. + +For normal Yama mode 1, the fspy supervisor is an ancestor of the tracked process. `PR_SET_PTRACER` can cover descendants whose ancestry changes, subject to the surrounding policy. + +## Userland exec fallback + +An in-house loader is feasible for ordinary build tools. Existing projects should be used as references, not adopted without modification. + +Each logical exec must first perform a trusted real exec of a fresh static `fspy_host`. That kernel transition kills sibling threads, releases a `vfork` parent, closes `CLOEXEC` descriptors, clears the alternate stack, resets caught signal dispositions, and discards the old address space. These generic effects match the requested target exec. Keep the host single-threaded until handoff. + +The inherited filter is active while the kernel has reset the handler. A normal dynamic loader or unaudited C runtime can die before `main`. Give the static PIE host a custom earliest entry that installs `SIGSYS` with a raw, marker-bypassed `rt_sigaction` before any intercepted syscall. The static-musl bootstrap experiment passed this exec-filter-handler cycle; production should not depend on the observed musl startup sequence. + +Preserve the logical argv and environment exactly when real-execing the host. Pass target metadata through a reserved inherited fd or a hidden entry that is removed before handoff. Do not prefix argv with `fspy_host` and the target. This keeps `/proc/self/cmdline`, `argv[0]`, and language-level argv behavior closer to a real exec. + +The fallback should start with these supported forms on x86-64 and AArch64: + +- dynamic PIE and dynamic non-PIE; +- static `ET_EXEC`; +- static PIE with documented relocation types; +- glibc and musl startup; +- Go executables such as esbuild; +- Linux shebang recursion and argument construction. + +The loader must use a bounded ELF parser, reserve the full image before mapping, map target segments from the target fd where possible, zero BSS correctly, enforce final W^X permissions, support `PT_INTERP`, and construct an accurate owned initial stack and auxv. It must keep the handler, restorer, gateway, and state in a collision-checked survivor island, then guard that island from target `MAP_FIXED`, `mremap`, `munmap`, and `mprotect` calls. + +The compatibility matrix passed dynamic PIE/non-PIE glibc, static musl, static Go, Node workers and children, shells, coreutils, esbuild 0.28.1 CLI, and the Node esbuild API service path. Neither reference loader accepted a shebang directly, but expanding the same script to its interpreter passed. Script parsing is an implementable host feature. + +One useful failure illustrates why the loader should be in-house. The Anvil reference placed `AT_RANDOM` at a word index treated as a byte offset. Go 1.23 and newer overwrite the 16-byte seed after reading it, corrupting adjacent argv data. Supplying owned random bytes fixed current esbuild. Libreflect passed a broader matrix but still has unchecked placement and incorrect auxv entries. + +The fallback retains target-specific differences: + +- `/proc/self/exe` and external process identity refer to the host; +- target credential, LSM, IMA, and audit exec hooks do not run; +- kernel auxv and mm metadata describe the host unless individual queries are virtualized; +- generic binfmt and target-specific ELF-property behavior is incomplete; +- the host's kernel brk, executable-file accounting, dumpability, and other mm-owned state can differ from the target's expected exec image; +- after kernel exec commits to the host, a later target parse or mapping error cannot return the original exec error to the old image. + +Virtualize in-process `/proc/self/exe`, `/proc/self/auxv`, and `PR_GET_AUXV` queries, and recognize logical self-reexec. This should address the Node and Go self-reexec failures observed with unmodified reference hosts. It cannot change what external observers, audit, ptrace, or the kernel see. + +Reject set-user-ID, set-group-ID, file-capability, target-specific LSM, mandatory-map collision, and exact external-identity cases with an actionable error. `no_new_privs` already prevents privilege elevation, but an explicit contract is better than accidental behavior. Preflight the target before committing to the host exec to reduce, but not eliminate, post-commit failures. + +The static host also needs a target-independent payload ABI. The current `Payload` and shared-memory channel schemas differ between glibc and musl builds because fields are compiled out under `target_env = "musl"`. A new host protocol must not use target-conditional serialization. + +## Environment compatibility + +| Environment | RET_TRAP syscall path | Temporary ptrace exec | Evidence or required handling | +| ---------------------------- | --------------------- | ------------------------- | --------------------------------------------------------------------- | +| Native Linux AArch64 | Passed | Passed | Ubuntu 24.04/Linux 6.8; dynamic, static, esbuild, and non-leader exec | +| Native Linux x86-64 | Passed | Passed with boundary | Ubuntu 24.04; default Chromium passes, namespace-sandbox zygote fails | +| WSL2 | Expected | Expected, untested | Avoid `NEW_LISTENER`; test mirrored networking and ptrace policy | +| Rootless containerd | Passed | Passed | Existing seccomp filter; also passed with `no_new_privs=1` | +| Docker default on Linux | Passed | Passed | Existing filter, `no_new_privs=1`, enforced default AppArmor | +| Docker Desktop amd64/Rosetta | Unsupported | Not reached | Local `PR_SET_SECCOMP` returned `EINVAL`; use a native-arch image | +| Kubernetes | Runtime-dependent | Runtime and LSM-dependent | Probe and fall back; test containerd/CRI-O RuntimeDefault profiles | +| Hosted CI | Passed on GitHub | Passed with boundary | GitHub passes except sandboxed Chromium; other providers need a probe | +| Custom sandbox | Policy-dependent | Often denied | Use the userland fallback or report an actionable error | + +The default-browser native x86-64 and Docker evidence is recorded in [GitHub Actions run 30735989574](https://github.com/voidzero-dev/vite-task/actions/runs/30735989574). The Docker recursive bridge passed with `Seccomp: 2`, `NoNewPrivs: 1`, and `docker-default (enforce)`. The sandboxed-Chromium boundary is recorded separately in [run 30737025514](https://github.com/voidzero-dev/vite-task/actions/runs/30737025514). The Rosetta result is an emulation limitation, not a failure of native x86-64 Docker. + +The open [WSL issue about seccomp notification](https://github.com/microsoft/WSL/issues/9548) concerns the single `NEW_LISTENER` restriction when WSL mirrored networking already owns a listener. It does not prevent stacking a normal `RET_TRAP` filter. The current WSL kernel configuration enables seccomp filtering. + +Docker's [default seccomp profile](https://github.com/moby/profiles/blob/main/seccomp/default.json) allows `seccomp`, `prctl`, exec, signal operations, and ptrace/process-vm operations on kernels at least 4.8. The current [Docker AppArmor template](https://github.com/moby/profiles/blob/main/apparmor/template.go) allows tracing between processes in the same container profile, subject to Yama and other LSM rules. Kubernetes `RuntimeDefault` is selected by the runtime, so it is not a portable guarantee. See the [Kubernetes kernel security constraints](https://kubernetes.io/docs/concepts/security/linux-kernel-security-constraints/). + +## Performance expectations and measurements + +The expected ordering is: + +```text +LD_PRELOAD < in-process RET_TRAP < seccomp user notification +``` + +`RET_TRAP` constructs and restores a full signal frame, runs the handler, and often issues the real syscall a second time. The measurements below quantify its advantage over waking a supervisor, reading target memory from another process, recording the event there, and sending a notification response. + +Five pinned runs of the controlled AArch64 prototype produced these medians: + +| Path | Median | Relative to matching baseline | +| --------------------------------------------------- | ----------: | -------------------------------------: | +| Direct `getpid`, no filter | 115.6 ns | 1.00x | +| Trap and set the result register | 565.8 ns | 4.93x | +| Trap and reissue through the gateway | 713.7 ns | 6.17x | +| `openat("/dev/null")` plus `close`, no filter | 531.7 ns | 1.00x | +| Trap, safe path copy, `openat` reissue, and `close` | 1,451.7 ns | 2.72x | +| User notification, emulated result | 13,922.7 ns | about 122x the direct-syscall baseline | +| User notification with `CONTINUE` | 13,931.4 ns | about 123x the direct-syscall baseline | + +The representative filesystem trap was about 9.6 times faster than the user-notification round trip. It excludes absolute-path normalization and shared-memory recording, so it is a lower bound for the new backend. The user-notification probe also excludes fspy path processing, making the process-boundary comparison conservative. + +One native x86-64 GitHub runner produced the same ordering: 133.0 ns for direct `getpid`, 2,782.5 ns for trap and reissue, and 29,798.4 ns for user notification with `CONTINUE`. On that runner the in-process path was 10.7 times faster than user notification. Trapped `openat` plus `close` took 6,956.1 ns versus a 2,611.5 ns unfiltered baseline, or 2.66x. These are single-run CI observations rather than pinned-run medians. + +The minimal preload interposer was indistinguishable from the untracked `openat` baseline, about 0.53 microseconds. That is only dispatch cost; it does not record an event and direct syscalls bypass it. `LD_PRELOAD` remains the performance floor. + +The [current main-branch x86-64 fspy benchmark](https://github.com/voidzero-dev/vite-task/actions/runs/30639476712) from 2026-07-31 reports: + +| Current backend | Launch overhead | Access overhead | +| -------------------------------- | --------------: | --------------: | +| Dynamic `LD_PRELOAD` | +60.07% | +53.43% | +| Static seccomp user notification | +154.49% | +965.35% | + +These percentages use different dynamic and static target binaries. They establish the current cost range, not a controlled three-way comparison. + +The persistent `SECCOMP_RET_TRACE` experiment in [draft PR #575](https://github.com/voidzero-dev/vite-task/pull/575) reinforces the design choice. Its [benchmark run](https://github.com/voidzero-dev/vite-task/actions/runs/30353491853) kept a tracer attached and measured static access at +938% over untracked, 11.49% slower than the user-notification base in that run. Ptrace is useful only as the short exec bridge; keeping it attached does not solve the process-switch cost. + +Post-exec handler injection measured 78.7 microseconds p50 and 101.8 microseconds p95 in the AArch64 VM. This is paid once per successful exec, not per file access. + +Before rollout, extend the benchmark to report: + +- a raw syscall baseline; +- trap plus register emulation; +- trap plus shared-memory recording and absolute-path resolution; +- 1, 2, 8, and 32 concurrent threads; +- open/stat/readdir and exec-heavy workloads; +- p50, p95, and p99, plus frontend-tool wall time and CPU time. + +Add forced backend selection to the existing benchmark so the same dynamic and static workloads compare untracked, preload, user notification, and SIGSYS on one machine. + +## Implementation map + +The current backend decision is in `crates/fspy_shared_unix/src/spawn/linux/mod.rs`. It inspects the ELF interpreter and chooses either `LD_PRELOAD` or a seccomp user-notification `pre_exec` hook. The new backend removes that per-executable choice. + +A staged implementation should use these boundaries: + +1. Add a Linux-only `fspy_sigsys` crate with filter generation, architecture register access, the raw gateway, and a freestanding handler artifact. +2. Define a stable `repr(C)` or manually encoded host/handler configuration that is identical for glibc, musl, and freestanding artifacts. +3. Add a raw lock-free shared-memory event format. Do not call the current preload `Client` from the signal handler because it uses runtime facilities and target-conditional types. +4. Implement process-wide virtual `SIGSYS` action state, per-TID masks, action-mask sanitization, and the declared compatibility boundary for the remaining signal APIs. +5. Add an exec coordination service beside the current supervisor lifecycle in `crates/fspy/src/unix/mod.rs`. +6. Implement post-exec injection for x86-64 and AArch64, including the x86-64 `SA_RESTORER` trampoline, TID re-keying, W^X mappings, and exec-failure detach. +7. Build the custom-entry static host and in-house loader as the ptrace-denied fallback. Keep target metadata out of logical argv and environment. +8. Replace preload and user-notification selection only after forced-backend differential tests pass. Keep forced legacy modes for bisecting regressions during rollout. + +## Validation gates + +Do not make SIGSYS the default until all of these gates pass: + +1. Every syscall in the tracked set produces the same cache-relevant event for libc and direct-syscall callers. +2. Invalid pointers return `EFAULT` without crashing the handler. +3. Go and esbuild survive signal installation, thread creation, direct file syscalls, subprocesses, and repeated exec. +4. Node, npm/pnpm/yarn, Vite, Vitest, Rolldown, oxlint, Bun, Deno, and Playwright complete representative workloads with output and exit-status parity. +5. Shell scripts, `posix_spawn`, `fork`, `vfork`, multithreaded exec, `execveat(AT_EMPTY_PATH)`, and failed exec preserve observable behavior. +6. Ubuntu glibc, Alpine musl, WSL2, Docker default/rootless, and representative Kubernetes profiles either run or select a documented fallback. +7. x86-64 and AArch64 run the same functional suite. Compat x32/i386 must be rejected with a clear error or implemented. +8. Performance is measured with forced backends on identical target binaries and real frontend workloads. + +## Remaining risks + +- Full logical SIGSYS behavior is a small signal-compatibility layer, not a single intercepted `sigaction` call. +- Per-thread logical mask inheritance across `clone` is not solved by the current prototype. +- Fault-safe in-process pointer reads add work. `process_vm_readv` works for the prototype but can be denied by policy; direct reads turn an `EFAULT` case into a potential crash. +- Correct event recording needs bounded backpressure and stable cwd/dirfd resolution without target runtime locks. +- An outer seccomp filter can return a stronger action. `KILL` remains fatal; an outer `TRAP` requires correct nested dispatch. +- The handler mapping must be position-independent, W^X-clean, collision-checked, and independent of libc, TLS, allocation, locks, and unwinding. +- The sixth-argument gateway is not a security boundary. +- Ptrace injection conflicts with another tracer and can be denied after startup, so fallback selection must also handle a later capability loss. +- The userland fallback cannot reproduce target kernel identity/security hooks, and target-loading failures after host exec are irreversible. + +## Sources and verification + +This decision is grounded in the repository's current fspy implementation, Linux kernel documentation and source, the supplied userland-exec evaluation, and these reproducible artifacts: + +- [`research/sigsys-prototype/README.md`](../research/sigsys-prototype/README.md): trap semantics, signal virtualization, static-host bootstrap, esbuild, and controlled timings +- [`research/ptrace-exec-prototype/README.md`](../research/ptrace-exec-prototype/README.md): post-exec handler injection +- [`research/ptrace-exec-prototype/ESBUILD_RESULTS.md`](../research/ptrace-exec-prototype/ESBUILD_RESULTS.md): latest esbuild, non-leader exec, and injection latency +- [`research/userland-exec-compat/RESULTS.md`](../research/userland-exec-compat/RESULTS.md): frontend and ELF compatibility matrix + +Key repository entry points: + +- `crates/fspy/src/unix/mod.rs` +- `crates/fspy_shared_unix/src/spawn/linux/mod.rs` +- `crates/fspy_preload_unix/src/client/mod.rs` +- `crates/fspy_seccomp_unotify/src/supervisor/mod.rs` +- `crates/fspy/src/unix/syscall_handler/mod.rs` +- `crates/fspy_benchmark/README.md` + +Primary external references: + +- [Linux seccomp filter documentation](https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html) +- [`seccomp(2)`](https://man7.org/linux/man-pages/man2/seccomp.2.html) +- [`execve(2)`](https://man7.org/linux/man-pages/man2/execve.2.html) +- [`signal(7)`](https://man7.org/linux/man-pages/man7/signal.7.html) +- [`sigaction(2)`](https://man7.org/linux/man-pages/man2/sigaction.2.html) +- [`ptrace(2)`](https://man7.org/linux/man-pages/man2/ptrace.2.html) +- [Linux syscall user dispatch](https://www.kernel.org/doc/html/latest/admin-guide/syscall-user-dispatch.html) +- [WSL seccomp-notify issue](https://github.com/microsoft/WSL/issues/9548) +- [Docker default seccomp profile](https://github.com/moby/profiles/blob/main/seccomp/default.json) + +Syscall user dispatch was considered and rejected for the main design. It is per-thread and is reset by `fork`, `clone`, and `exec`, which makes transparent thread/process coverage harder than an inherited seccomp filter. diff --git a/docs/fspy-rust-injected-runtime.md b/docs/fspy-rust-injected-runtime.md new file mode 100644 index 000000000..173dfa89d --- /dev/null +++ b/docs/fspy-rust-injected-runtime.md @@ -0,0 +1,166 @@ +# Rust injected runtime for fspy + +Research date: 2026-08-02 + +Status: architecture, cross-compiled artifact audit, and native x86-64 execution proof complete. The proof is not yet a production syscall dispatcher. + +Primary audience: fspy maintainers implementing the Linux `SIGSYS` handler island and its ptrace injection protocol. + +## Decision + +Write the injected runtime in freestanding Rust, with small assembly fragments for the raw syscall gateway and `rt_sigreturn` restorer. + +Use two mappings: + +1. An RX mapping containing a relocation-free Rust code blob, read-only constants, the restorer, and one pointer-sized slot patched before injection. +2. A separate RW mapping containing versioned runtime state, fixed event and scratch storage, and an optional fixed-capacity allocator arena. + +Do not allocate in the `SIGSYS` fast path. Use stack records, a preallocated event ring, and an atomic scratch-slot pool there. For initialization and other bounded slow paths, use the in-house monotonic `AtomicUsize` bump allocator in the [proof source](../research/rust-injected-runtime/src/lib.rs). It is lock-free, syscall-free, signal-reentrant, and backed by 64 KiB of fixed memory in the RW state mapping. + +The cross-build currently produces these complete probe blobs: + +| Target | Raw blob size | Contents | +| ------- | ------------: | -------------------------------------------------------------------------- | +| x86-64 | 240 bytes | handler, restorer, raw six-argument syscall, allocator, state-pointer slot | +| AArch64 | 304 bytes | handler, restorer, raw six-argument syscall, allocator, state-pointer slot | + +Both outputs have no runtime relocations, undefined symbols, writable sections, GOT, PLT, TLS, or dynamic-loader dependency. The [linker script](../research/rust-injected-runtime/blob.ld) and [artifact verifier](../research/rust-injected-runtime/verify.sh) make those properties build failures. + +The native [CI execution proof](https://github.com/voidzero-dev/vite-task/actions/runs/30737941238) mapped the x86-64 blob RX, installed its handler and restorer through raw `rt_sigaction`, triggered a seccomp `SIGSYS`, returned through the Rust restorer, updated the RW state, and exercised the raw syscall gateway and allocator. + +## How Rust code is injected + +The supervisor does not inject a Rust ELF executable or start a Rust runtime. Rust is only the source language for a raw machine-code artifact. + +At `PTRACE_EVENT_EXEC`, before the target executes its entry instruction, the supervisor performs this sequence: + +1. Remote-`mmap` a private RW state region and initialize its versioned ABI header, rings, scratch slots, allocator cursor, and fixed arena. +2. Read the architecture-specific raw blob produced by `llvm-objcopy`. +3. Patch the `FSPY_STATE_PTR` slot in that local byte buffer with the remote state address. +4. Remote-`mmap` a code region as RW and copy the blob with `PTRACE_POKEDATA` or `process_vm_writev`. +5. Remote-`mprotect` the code region to RX. Never leave the production handler RWX. +6. Install its physical `SIGSYS` action with the relocated handler and restorer addresses. +7. Restore target registers and overwritten instruction bytes, then detach. + +All code and constant references inside the artifact are PC-relative. The only process-specific value is the state pointer slot. The slot is part of the raw blob and is patched before the remote mapping becomes executable. + +An initialized Rust `static` is the wrong representation for this slot. LLVM can constant-fold a private static, while an externally visible PIC static can introduce a GOT lookup. The linker script instead defines the bytes directly: + +```ld +. = ALIGN(8); +HIDDEN(FSPY_STATE_PTR = .); +QUAD(0); +``` + +Rust finds the slot using `lea [rip + FSPY_STATE_PTR]` on x86-64 or `adr FSPY_STATE_PTR` on AArch64. The build asserts that the AArch64 blob remains within the `adr` range. If the artifact grows beyond that bound, change it to `adrp` plus `add` and retain the relocation audit. + +## Freestanding Rust contract + +The crate uses `#![no_std]`, `panic=abort`, and no dependencies. It links with `rust-lld -nostdlib` rather than musl, despite using a `*-unknown-linux-musl` target for the Linux ABI. The runtime must not depend on: + +- libc, `errno`, pthreads, TLS, dynamic linking, or target-owned callbacks; +- formatting, unwinding, or a panic path that returns; +- compiler-generated stack probes or large stack allocations; +- target CPU features newer than the deployment baseline; +- hidden `memcpy` or `memset` imports. + +The few operations that require a stable raw ABI stay explicit: + +- The syscall gateway is inline assembly. On x86-64 it marks `rcx` and `r11` clobbered and uses `r10`, `r8`, and `r9` for arguments four through six. It deliberately omits the `nomem` option because the kernel can access pointed-to memory. +- The `rt_sigreturn` restorer is a standalone assembly symbol. Its syscall number is 15 on x86-64 and 139 on AArch64. +- The `SIGSYS` handler accepts the kernel's three `SA_SIGINFO` arguments and mutates the architecture return register in the supplied `ucontext`. + +Hard-coded `siginfo` and `ucontext` offsets are part of the Linux UAPI contract, not the Rust or libc ABI. Keep native C `offsetof` assertions for glibc and musl in CI, then compare them with Rust constants. The proof currently uses: + +| Field | x86-64 offset | AArch64 offset | +| -------------------------------- | ------------: | -------------: | +| `siginfo.si_code` | 8 | 8 | +| `siginfo.si_syscall` | 24 | 24 | +| `ucontext` syscall return result | 144 | 184 | + +These values follow the Linux [x86-64 signal context](https://github.com/torvalds/linux/blob/master/arch/x86/include/uapi/asm/sigcontext.h), [AArch64 signal context](https://github.com/torvalds/linux/blob/master/arch/arm64/include/uapi/asm/sigcontext.h), and [AArch64 ucontext](https://github.com/torvalds/linux/blob/master/arch/arm64/include/uapi/asm/ucontext.h). Do not infer additional offsets from a Rust `libc` struct without the native checks. + +## Fixed-capacity global allocator + +No existing crate is a better fit than the small in-house allocator. + +| Candidate | Assessment | +| ------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Rust's [`GlobalAlloc` example](https://doc.rust-lang.org/core/alloc/trait.GlobalAlloc.html) | Uses exactly the required fixed arena plus atomic monotonic cursor. This is the basis of the proof. | +| [`lock_free_buddy_allocator`](https://github.com/pskrgag/lock_free_buddy_allocator) | Lock-free, but page-granular, requires a backend allocator for internal metadata, requires a CPU-ID provider, and implements the nightly `Allocator` API rather than a ready global allocator. | +| [`slaballoc`](https://github.com/DrChat/slaballoc) | Lockless and fixed-memory, but allocates only one `Sized` type, is not `GlobalAlloc`, uses nightly features, and retains an alignment FIXME. | +| [`atomic-pool`](https://github.com/embassy-rs/atomic-pool) | Good model for typed scratch slots, but intentionally exposes a pool-specific `Box`, not arbitrary `Layout` allocation through `GlobalAlloc`. | + +The implemented allocator reserves an aligned interval with a CAS loop: + +```rust +let mut current = state.arena_next.load(Relaxed); +loop { + let aligned = current.checked_add(align - 1)? & !(align - 1); + let end = aligned.checked_add(size)?; + if end > ARENA_LEN { + return null_mut(); + } + match state.arena_next.compare_exchange_weak(current, end, Relaxed, Relaxed) { + Ok(_) => return state.arena_base().add(aligned), + Err(observed) => current = observed, + } +} +``` + +The actual implementation returns null rather than using `?`, validates `Layout`, rejects alignments above 4096, and uses checked arithmetic. `dealloc` is a no-op. + +Rust guarantees that available atomics in `core::sync::atomic` are [lock-free but not necessarily wait-free](https://doc.rust-lang.org/core/sync/atomic/index.html#portability). For the supported x86-64 and AArch64 targets, the audited output is a native compare-and-swap loop. AArch64 builds with `-C target-feature=-outline-atomics`; otherwise LLVM can emit an external outline-atomic helper, breaking the self-contained blob contract. + +`Relaxed` ordering is sufficient to reserve disjoint byte ranges. It does not publish initialized objects to another thread. Any cross-thread object handoff needs its own release and acquire operation. + +This allocator has deliberate limitations: + +- It is lock-free, not wait-free. A contending caller can retry. +- It never reclaims memory. Logical exec replaces the whole state mapping. +- Exhaustion is deterministic. Infallible `Box::new` or `Vec::push` can still abort on a null result, so runtime code must use fallible allocation APIs. +- Being signal-reentrant does not make allocation desirable in a handler. A nested signal cannot deadlock the allocator, but it can consume the remaining arena or starve an interrupted CAS loop. + +## Keep the SIGSYS path allocation-free + +Use three bounded structures instead: + +1. Build the immediate syscall description in fixed stack storage. +2. Reserve an event-ring record with a monotonically increasing atomic sequence. Publish the completed record with a separate release store so the supervisor never reads a partial record. +3. For bounded path-copy or nested-handler scratch space, claim a fixed slot with an atomic bitmap. Release the bitmap bit only after the slot is no longer referenced. + +Nested `SIGSYS` delivery under `SA_NODEFER` must claim a different scratch slot. When the pool is exhausted, block or use a documented supervisor slow path. Do not reuse an in-flight slot and do not drop a cache-relevant event. + +Represent virtualized target `SIGSYS` actions as immutable snapshots. Publish a new snapshot with an atomic pointer swap and do not reclaim old snapshots until logical exec. That avoids use-after-free if a nested handler still observes the previous action. + +## Build and audit + +Run the proof from its directory: + +```sh +make check +``` + +The build performs these steps for `x86_64-unknown-linux-musl` and `aarch64-unknown-linux-musl`: + +1. Compile one `staticlib` with PIC, no red zone, no unwind tables, and aborting panics. +2. Link only selected sections with `rust-lld -nostdlib --gc-sections --no-undefined`. +3. Reject `.data`, `.bss`, GOT, PLT, TLS, dynamic, and initialization sections in the linker script. +4. Extract `.fspy_blob` with `llvm-objcopy`. +5. Assert that the final ELF has no relocations or undefined symbols and report the state-pointer patch offset. +6. On native Linux x86-64, map the blob RX, install its handler and restorer, trigger a seccomp trap, and exercise its allocator and raw syscall gateway. + +Production CI should additionally disassemble and allowlist every syscall instruction, enforce a maximum blob and stack-frame size, run the C/Rust layout assertions, inject at randomized addresses, stress nested and concurrent signals, and execute on native AArch64. Cross-compilation verifies the AArch64 artifact shape, but only native execution can validate instruction-cache coherency after remote writes. + +## Remaining work + +The Rust language and artifact format are no longer feasibility risks. The production risks are in runtime semantics: + +- implement the complete syscall dispatcher and fault-safe target-memory copying; +- build the lossless shared event ring and bounded scratch pool; +- complete per-thread signal-mask and logical `SIGSYS` action virtualization; +- protect the code and state mappings from target `mmap`, `mprotect`, `mremap`, and `munmap` operations; +- test post-injection AArch64 instruction-cache synchronization; +- integrate the same runtime with both the ptrace bridge and the static-host userland loader. + +The ptrace bridge remains unsuitable for Chromium's namespace-sandbox zygote exec. Rust changes the handler implementation, not that exec-handshake boundary. Route that exec through the static-host loader before attempting ptrace injection. diff --git a/research/ptrace-exec-prototype/ESBUILD_RESULTS.md b/research/ptrace-exec-prototype/ESBUILD_RESULTS.md new file mode 100644 index 000000000..ce0cf9627 --- /dev/null +++ b/research/ptrace-exec-prototype/ESBUILD_RESULTS.md @@ -0,0 +1,133 @@ +# Latest-esbuild compatibility experiment + +Validation date: 2026-08-02 + +The npm registry and upstream GitHub release API both reported esbuild 0.28.1 +as latest. The GitHub release was published on 2026-06-11: + +- +- +- + +## Environment + +```text +Ubuntu 24.04.4 LTS +Linux 6.8.0-134-generic aarch64 +GCC 13.3.0 +Lima/VZ, 4 CPUs, 6 GiB RAM +esbuild 0.28.1, statically linked AArch64 ELF +``` + +## Interception configuration + +Before exec, `esbuild_injector.c` installs `SECCOMP_RET_TRAP` rules for: + +- `getpid` +- `openat` +- `rt_sigaction` +- `rt_sigprocmask` + +At `PTRACE_EVENT_EXEC`, it injects a 360-byte freestanding PIC handler. The +handler: + +- reissues `getpid` and `openat` with the sixth argument set to + `0xf5f05ec0dec0de55`; +- compares both 32-bit halves of that marker in classic BPF before allowing a + gateway call; +- shadows the target's logical `SIGSYS` action so Go can query and replace it + without removing the physical fspy handler; +- strips the physical SIGSYS bit from every action mask passed to the kernel; +- strips SIGSYS from every `rt_sigprocmask` input before reissuing it; +- returns raw kernel results through saved AArch64 `x0`. + +The tracer detaches before target entry. It does not remain attached for any +seccomp trap. + +## Results + +Version startup passed: + +```text +esbuild-injector: handler=0xf0ea72d07000 blob=360 bytes (RWX experiment) +esbuild-injector: exec-stop to detach 86.125 us +0.28.1 +esbuild-injector: target exit=0 +``` + +A real bundle, which requires `openat` to read `input.js`, also passed: + +```text +esbuild-injector: handler=0xe6ec97cc4000 blob=360 bytes (RWX experiment) +esbuild-injector: exec-stop to detach 110.125 us + + out.js 1.0kb + +⚡ Done in 1ms +esbuild-injector: target exit=0 +``` + +Executing the generated bundle printed `answer=42`. + +The version experiment also passed with the injector compiled under +AddressSanitizer and UndefinedBehaviorSanitizer. + +Thirty independent `esbuild --version` injections measured time directly from +receipt of `PTRACE_EVENT_EXEC` through successful `PTRACE_DETACH`: + +```text +runs=30 min=66.208 us p50=78.708 us p95=101.792 us max=217.375 us mean=84.669 us +``` + +This excludes launcher startup and esbuild runtime. It includes remote `mmap`, +45 word-sized `PTRACE_POKEDATA` writes, remote `rt_sigaction`, register and entry +instruction restoration, and detach. + +Run the recorded experiment with: + +```sh +./run_esbuild_experiment.sh +``` + +## Exact boundary of this proof + +This is compatibility evidence, not a production handler: + +- The handler page is RWX so its four-word SIGSYS shadow can share the PIC + mapping. Production should split RX code from RW state or use a file-backed + sealed mapping. +- `rt_sigaction` and `rt_sigprocmask` inputs are modified in place around the + gateway syscall. A concurrent reader can observe the temporary sanitized + value, and an invalid/read-only pointer can fault in the handler. +- SIGSYS action state is shadowed, but logical per-thread SIGSYS mask state is + not. Callers always observe the physical unblocked state in returned masks. +- A non-seccomp SIGSYS is not forwarded to the target's shadow handler. +- `SA_RESETHAND`, logical `SA_NODEFER`, `SA_ONSTACK`, synchronous signal waits, + `signalfd`, and target-edited `ucontext` masks are not emulated. +- Only native AArch64 and the four named syscall numbers are implemented. +- The gateway marker is an accidental-bypass guard, not a security boundary. + +Despite those limits, latest static Go/esbuild completed runtime signal setup, +created threads, opened its input, and produced a valid bundle after ptrace had +detached. + +## Non-leader exec observation + +`nonleader_exec.c` separately traced a pthread worker executing `/bin/true`. +The event was reported under the process leader's TID, while +`PTRACE_GETEVENTMSG` returned the worker's former TID: + +```sh +cc -O2 -g -Wall -Wextra -Werror -std=gnu11 -pthread \ + -o nonleader-exec nonleader_exec.c +./nonleader-exec /bin/true +``` + +```text +nonleader: clone event leader=66112 worker=66113 +nonleader: exec stop reported as tid=66112; former tid=66113 +PASS: PTRACE_GETEVENTMSG preserved the non-leader's former TID +``` + +This confirms that a production supervisor must re-key per-TID exec state at a +non-leader exec event. diff --git a/research/ptrace-exec-prototype/Makefile b/research/ptrace-exec-prototype/Makefile new file mode 100644 index 000000000..3aa2d34fd --- /dev/null +++ b/research/ptrace-exec-prototype/Makefile @@ -0,0 +1,25 @@ +CC ?= cc +CFLAGS ?= -O2 -g -Wall -Wextra -Werror -std=gnu11 + +.PHONY: all check clean + +all: injector recursive_injector target nested_trap + +injector: injector.c + $(CC) $(CFLAGS) -o $@ $< + +recursive_injector: recursive_injector.c + $(CC) $(CFLAGS) -pthread -o $@ $< + +target: target.c + $(CC) $(CFLAGS) -o $@ $< + +nested_trap: nested_trap.c + $(CC) $(CFLAGS) -o $@ $< + +check: all + ./injector "$$(realpath ./target)" + ./recursive_injector "$$(realpath ./nested_trap)" + +clean: + rm -f injector recursive_injector target nested_trap diff --git a/research/ptrace-exec-prototype/README.md b/research/ptrace-exec-prototype/README.md new file mode 100644 index 000000000..b2976c965 --- /dev/null +++ b/research/ptrace-exec-prototype/README.md @@ -0,0 +1,125 @@ +# Post-exec `SIGSYS` handler injection prototype + +This proves the kernel ordering needed by a hybrid fspy design: + +1. A child installs a seccomp filter that returns `SECCOMP_RET_TRAP` for + `getpid` and then performs a real `execve`. +2. Its parent catches `PTRACE_EVENT_EXEC`, after the new image exists but before + its first user-space instruction. +3. The parent advances once to the pending `execve` syscall-exit stop. +4. The parent executes remote `mmap`, `rt_sigaction`, and `mprotect` syscalls at + the stopped entry PC, copies in a freestanding handler, and detaches. +5. The target verifies `TracerPid: 0` and calls `getpid`. The injected in-process + handler changes the saved return register to `0x51515151`. + +The source has native AArch64 and x86-64 register/trampoline implementations. +It uses only libc/kernel headers and is suitable for a native Linux CI job. +The syscall-exit rendezvous is required because the exec event occurs before +the original syscall finishes returning. In particular, the x86-64 return path +would otherwise overwrite the first injected syscall number in `rax`. + +## Run + +```sh +make check +``` + +Expected output includes: + +```text +injector: caught PTRACE_EVENT_EXEC before target entry +injector: mapped handler at ..., handler=..., blob=... bytes +injector: detached; target's trapped syscall now has no tracer +target: TracerPid=0 before trapped getpid +target: trapped getpid returned 0x51515151 (expected 0x51515151) +PASS: post-exec handler ran entirely in-process after detach +injector: target exit status 0 +``` + +## Recursive `PTRACE_SEIZE` and Vitest browser proof + +`recursive_injector.c` implements the production-shaped recursive success path +on x86-64: + +1. The in-process handler traps `execve` and `execveat`, sends its TID and a + release-word address to the ancestor supervisor with a queued real-time + signal, and waits in a futex. It does not depend on an inherited control fd. +2. The supervisor attaches only to that thread with `PTRACE_SEIZE`, releases + the handler, and follows it to `PTRACE_EVENT_EXEC`. +3. The supervisor injects a freestanding handler into the new image, installs + the physical `SIGSYS` action, and detaches before target code runs. +4. Steady-state file syscalls are handled in process with no ptrace stop. The + prototype reissues `openat`, `openat2`, `newfstatat`, `statx`, `getdents64`, + `faccessat`, and `faccessat2` through the sixth-argument gateway. +5. `rt_sigaction(SIGSYS, ...)` and `rt_sigprocmask` are minimally virtualized so + shell, Node, and Chromium signal initialization cannot remove or block the + physical handler. + +Run a command with: + +```sh +make recursive_injector +./recursive_injector /bin/true +``` + +The repository CI runs its real Vitest browser fixture through this launcher. +The [native x86-64 result](https://github.com/voidzero-dev/vite-task/actions/runs/30735989574) +passed Node 22.19.0, Vitest 4.1.10, Playwright 1.61.1, and Chrome Headless Shell +149.0.7827.55. The bridge injected nine images, including four Chromium images, +reported zero failed execs, and produced a passing JSON test report. The same +bridge also passed inside Docker's default seccomp and AppArmor profiles with +`no_new_privs=1`. + +One non-obvious requirement came directly from this test. Linux automatically +blocks `SIGSYS` while the handler runs. A successful exec from that handler +never reaches `rt_sigreturn`, so the new image otherwise inherits `SIGSYS` +blocked and dies on its first trapped syscall. The handler must explicitly +unblock physical `SIGSYS` immediately before issuing the gateway exec. + +The default-browser result does not extend to Playwright's +`chromiumSandbox: true`. Native Ubuntu 24.04 controls launched sandboxed +Chromium both normally and under `no_new_privs`. Through the bridge, however, +the namespace zygote exec leaves fd 3 present while the browser side of the +boot socket observes EOF. Chromium aborts at +`zygote_host_impl_linux.cc:207`; the zygote later reaches `ZygoteMain` and gets +`EPIPE` after the browser closes its end. + +The [full-filter run](https://github.com/voidzero-dev/vite-task/actions/runs/30737025514) +captured that sequence. An +[exec-and-signal-only run](https://github.com/voidzero-dev/vite-task/actions/runs/30737143580) +failed at the same boundary, so the file-syscall passthrough is not the cause. +The failure precedes Chromium's own seccomp setup. The nested-filter test still +proves that fspy can identify its filter tag, forward a foreign seccomp trap to +the target's logical `SIGSYS` action, and permit reentrant delivery with +`SA_NODEFER`. + +Do not use this ptrace bridge for Chromium namespace-zygote execs. A production +hybrid must select the static-host userland loader before attempting that exec, +or reject the launch with an actionable compatibility error. + +## Prototype boundaries + +- This initial launcher uses `PTRACE_TRACEME`. A production chain can use the + trapped exec handler to coordinate a short-lived `PTRACE_SEIZE` with the + supervisor, then detach at the same post-exec stop. +- The filter traps only `getpid`; therefore the three remote injection syscalls + do not need the planned sixth-argument gateway marker. +- The injected handler only writes a constant return register. It does not yet + validate `siginfo_t`, reissue a syscall, log an event, virtualize signal state, + use an alternate stack, or protect its mapping from the target. +- The remote-syscall stub temporarily overwrites one machine word at the new + entry PC and restores both that word and the complete register set after each + call. +- The injected page is anonymous RX after installation. SELinux/AppArmor + policies that deny anonymous executable memory require a file-backed handler + mapping instead. +- Ptrace remains subject to one-tracer exclusivity, Yama, seccomp, and LSM + policy. The important performance property is that the tracer is detached + before any `SECCOMP_RET_TRAP` signal is delivered. +- The recursive prototype uses direct pointer loads while virtualizing signal + APIs and only reissues file syscalls; it does not yet perform fault-safe path + capture or write fspy events to shared memory. +- Its logical `SIGSYS` model is intentionally incomplete. The prototype + forwards a tagged foreign trap to the tested target handler, but per-thread + virtual masks, complete action semantics, and general coexistence with other + seccomp producers remain production work. diff --git a/research/ptrace-exec-prototype/RESULTS.md b/research/ptrace-exec-prototype/RESULTS.md new file mode 100644 index 000000000..9251158f1 --- /dev/null +++ b/research/ptrace-exec-prototype/RESULTS.md @@ -0,0 +1,94 @@ +# Validation results + +Validation date: 2026-08-02 + +## Native AArch64 + +Environment: + +```text +Ubuntu 24.04.4 LTS +Linux 6.8.0-134-generic aarch64 +GCC 13.3.0 +Lima/VZ, 4 CPUs, 6 GiB RAM +``` + +The host workspace is mounted read-only in the VM, so the sources were copied +to a disposable writable directory before building: + +```sh +probe_dir=$(mktemp -d) +repo_root=$(git rev-parse --show-toplevel) +cp "$repo_root"/research/ptrace-exec-prototype/* "$probe_dir"/ +cd "$probe_dir" +make check +``` + +Observed output: + +```text +cc -O2 -g -Wall -Wextra -Werror -std=gnu11 -o injector injector.c +cc -O2 -g -Wall -Wextra -Werror -std=gnu11 -o target target.c +./injector "$(realpath ./target)" +injector: caught PTRACE_EVENT_EXEC before target entry +injector: mapped handler at 0xe4e3b6551000, handler=0xe4e3b6551000, blob=32 bytes +injector: detached; target's trapped syscall now has no tracer +target: TracerPid=0 before trapped getpid +target: trapped getpid returned 0x51515151 (expected 0x51515151) +PASS: post-exec handler ran entirely in-process after detach +injector: target exit status 0 +``` + +The same injector passed with a fully static target: + +```sh +cc -O2 -static -Wall -Wextra -Werror -std=gnu11 -o target-static target.c +./injector "$(realpath ./target-static)" +``` + +Twenty additional dynamic-target runs completed successfully. + +The injector also passed the dynamic-target test when compiled with +AddressSanitizer and UndefinedBehaviorSanitizer: + +```sh +cc -O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer \ + -Wall -Wextra -Werror -std=gnu11 -o injector-asan injector.c +ASAN_OPTIONS=detect_leaks=1 ./injector-asan "$(realpath ./target)" +``` + +## Native x86-64 and Docker + +The x86-64 implementation was cross-compiled on macOS with Zig 0.15.2: + +```sh +zig cc -target x86_64-linux-gnu -O2 -g -Wall -Wextra -Werror \ + -std=gnu11 -o /tmp/injector-x86_64 injector.c +zig cc -target x86_64-linux-gnu -O2 -g -Wall -Wextra -Werror \ + -std=gnu11 -o /tmp/target-x86_64 target.c +``` + +Both outputs were valid dynamically linked x86-64 ELF executables. The native +Ubuntu 24.04 and Docker jobs in [GitHub Actions run +30734549943](https://github.com/voidzero-dev/vite-task/actions/runs/30734549943) +then ran `make check` successfully. The Docker process had one existing seccomp +filter, `no_new_privs=1`, and the enforced `docker-default` AppArmor profile. + +The first native x86-64 run caught an architecture-sensitive ordering bug. +`PTRACE_EVENT_EXEC` stopped before the pending `execve` return had written zero +to `rax`, overwriting the prepared `SYS_mmap` number. The corrected injector +uses `PTRACE_SYSCALL` once to rendezvous at that syscall's exit stop before +preparing any remote syscall. The same sequence now runs on both architectures. + +## What the result establishes + +- `PTRACE_EVENT_EXEC` occurs late enough that the new address space exists and + early enough to install a handler before the ELF entry point runs. +- Remote syscalls can allocate the island and register the handler while the + target remains at the exec stop. +- Detaching before resuming avoids ptrace signal-delivery stops for subsequent + seccomp-generated `SIGSYS` signals. +- The target observed `TracerPid=0` before entering its trapped syscall, so the + emulated result came from the injected handler rather than tracer mediation. +- The injection proof works on native AArch64 and x86-64 and under Docker's + default Linux security profiles. diff --git a/research/ptrace-exec-prototype/esbuild_injector.c b/research/ptrace-exec-prototype/esbuild_injector.c new file mode 100644 index 000000000..9859eeaef --- /dev/null +++ b/research/ptrace-exec-prototype/esbuild_injector.c @@ -0,0 +1,436 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(__aarch64__) +#error "The bounded esbuild experiment is AArch64-only" +#endif + +#define ARRAY_LEN(values) (sizeof(values) / sizeof((values)[0])) +#define GATEWAY_MAGIC UINT64_C(0xf5f05ec0dec0de55) +#define GATEWAY_MAGIC_LOW UINT32_C(0xdec0de55) +#define GATEWAY_MAGIC_HIGH UINT32_C(0xf5f05ec0) +#define SIGSYS_MASK_BIT UINT64_C(0x40000000) + +/* + * Freestanding PIC handler copied into the post-exec image. + * + * x9 points at saved x0 in ucontext. The filter traps getpid, openat, + * rt_sigaction, and rt_sigprocmask. The handler reissues the first two with a + * sixth-argument marker. It keeps the physical fspy SIGSYS handler installed, + * shadows the target's logical SIGSYS action, and strips SIGSYS from masks that + * reach the kernel. + * + * This intentionally mutates target action/mask words around the gateway + * syscall and uses an RWX page. Those shortcuts bound the compatibility + * experiment; they are not production design choices. + */ +__asm__( + ".pushsection .text.fspy_esbuild,\"ax\",@progbits\n" + ".balign 16\n" + ".global esbuild_blob_start\n" + ".global esbuild_handler\n" + ".global esbuild_return_offset_slot\n" + ".global esbuild_blob_end\n" + "esbuild_blob_start:\n" + "esbuild_handler:\n" + " adr x9, esbuild_return_offset_slot\n" + " ldr x9, [x9]\n" + " add x9, x2, x9\n" + " ldr x10, [x9, #64]\n" /* saved x8 / syscall number */ + " cmp x10, #172\n" /* getpid */ + " b.eq 1f\n" + " cmp x10, #56\n" /* openat */ + " b.eq 1f\n" + " cmp x10, #134\n" /* rt_sigaction */ + " b.eq 2f\n" + " cmp x10, #135\n" /* rt_sigprocmask */ + " b.eq 5f\n" + " mov x0, #-38\n" /* -ENOSYS */ + " str x0, [x9]\n" + " ret\n" + + /* Reissue getpid/openat using x5 as the seccomp gateway marker. */ + "1:\n" + " ldp x0, x1, [x9, #0]\n" + " ldp x2, x3, [x9, #16]\n" + " ldr x4, [x9, #32]\n" + " movz x5, #0xde55\n" + " movk x5, #0xdec0, lsl #16\n" + " movk x5, #0x5ec0, lsl #32\n" + " movk x5, #0xf5f0, lsl #48\n" + " mov x8, x10\n" + " svc #0\n" + " str x0, [x9]\n" + " ret\n" + + /* rt_sigaction: shadow SIGSYS; sanitize every other action mask. */ + "2:\n" + " ldr x11, [x9, #0]\n" /* signum */ + " cmp x11, #31\n" + " b.eq 4f\n" + " ldr x12, [x9, #8]\n" /* new action */ + " cbz x12, 3f\n" + " ldr x13, [x12, #24]\n" + " mov x14, #0x40000000\n" + " bic x15, x13, x14\n" + " str x15, [x12, #24]\n" + "3:\n" + " ldp x0, x1, [x9, #0]\n" + " ldp x2, x3, [x9, #16]\n" + " movz x5, #0xde55\n" + " movk x5, #0xdec0, lsl #16\n" + " movk x5, #0x5ec0, lsl #32\n" + " movk x5, #0xf5f0, lsl #48\n" + " mov x8, #134\n" + " svc #0\n" + " cbz x12, 30f\n" + " str x13, [x12, #24]\n" + "30:\n" + " str x0, [x9]\n" + " ret\n" + + /* Logical rt_sigaction(SIGSYS): copy a four-word kernel action. */ + "4:\n" + " adr x12, esbuild_sigsys_shadow\n" + " ldr x13, [x9, #16]\n" /* old action */ + " cbz x13, 40f\n" + " ldp x14, x15, [x12, #0]\n" + " stp x14, x15, [x13, #0]\n" + " ldp x14, x15, [x12, #16]\n" + " stp x14, x15, [x13, #16]\n" + "40:\n" + " ldr x13, [x9, #8]\n" /* new action */ + " cbz x13, 41f\n" + " ldp x14, x15, [x13, #0]\n" + " stp x14, x15, [x12, #0]\n" + " ldp x14, x15, [x13, #16]\n" + " stp x14, x15, [x12, #16]\n" + "41:\n" + " str xzr, [x9]\n" + " ret\n" + + /* rt_sigprocmask: temporarily clear SIGSYS in the supplied set. */ + "5:\n" + " ldr x12, [x9, #8]\n" /* new set */ + " cbz x12, 6f\n" + " ldr x13, [x12]\n" + " mov x14, #0x40000000\n" + " bic x15, x13, x14\n" + " str x15, [x12]\n" + "6:\n" + " ldp x0, x1, [x9, #0]\n" + " ldp x2, x3, [x9, #16]\n" + " movz x5, #0xde55\n" + " movk x5, #0xdec0, lsl #16\n" + " movk x5, #0x5ec0, lsl #32\n" + " movk x5, #0xf5f0, lsl #48\n" + " mov x8, #135\n" + " svc #0\n" + " cbz x12, 60f\n" + " str x13, [x12]\n" + "60:\n" + " str x0, [x9]\n" + " ret\n" + + ".balign 8\n" + "esbuild_sigsys_shadow:\n" + " .quad 0, 0, 0, 0\n" + "esbuild_return_offset_slot:\n" + " .quad 0\n" + "esbuild_blob_end:\n" + ".popsection\n"); + +extern const unsigned char esbuild_blob_start[]; +extern const unsigned char esbuild_handler[]; +extern const unsigned char esbuild_return_offset_slot[]; +extern const unsigned char esbuild_blob_end[]; +extern char **environ; + +struct kernel_sigaction_wire { + uint64_t handler; + uint64_t flags; + uint64_t restorer; + uint64_t mask; +}; + +static void fatal(const char *message) +{ + perror(message); + exit(EXIT_FAILURE); +} + +static void fatal_message(const char *message) +{ + fprintf(stderr, "%s\n", message); + exit(EXIT_FAILURE); +} + +static int get_regs(pid_t pid, struct user_regs_struct *regs) +{ + struct iovec iov = {.iov_base = regs, .iov_len = sizeof(*regs)}; + return (int)ptrace(PTRACE_GETREGSET, pid, (void *)(uintptr_t)NT_PRSTATUS, &iov); +} + +static int set_regs(pid_t pid, const struct user_regs_struct *regs) +{ + struct iovec iov = {.iov_base = (void *)regs, .iov_len = sizeof(*regs)}; + return (int)ptrace(PTRACE_SETREGSET, pid, (void *)(uintptr_t)NT_PRSTATUS, &iov); +} + +static int wait_for_trap(pid_t pid) +{ + int status; + if (waitpid(pid, &status, 0) < 0) + return -1; + if (!WIFSTOPPED(status) || WSTOPSIG(status) != SIGTRAP) { + fprintf(stderr, "unexpected remote-syscall status: %#x\n", status); + errno = EPROTO; + return -1; + } + return 0; +} + +static long remote_syscall(pid_t pid, long number, uint64_t a0, uint64_t a1, + uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) +{ + struct user_regs_struct saved; + struct user_regs_struct call; + struct user_regs_struct stopped; + unsigned long original; + const unsigned long stub = UINT64_C(0xd4200000d4000001); /* svc; brk */ + long result; + + if (get_regs(pid, &saved) < 0) + fatal("PTRACE_GETREGSET"); + call = saved; + errno = 0; + original = (unsigned long)ptrace(PTRACE_PEEKTEXT, pid, (void *)saved.pc, NULL); + if (original == (unsigned long)-1 && errno != 0) + fatal("PTRACE_PEEKTEXT"); + if (ptrace(PTRACE_POKETEXT, pid, (void *)saved.pc, (void *)stub) < 0) + fatal("PTRACE_POKETEXT stub"); + + call.regs[0] = a0; + call.regs[1] = a1; + call.regs[2] = a2; + call.regs[3] = a3; + call.regs[4] = a4; + call.regs[5] = a5; + call.regs[8] = (uint64_t)number; + if (set_regs(pid, &call) < 0) + fatal("PTRACE_SETREGSET call"); + if (ptrace(PTRACE_CONT, pid, NULL, NULL) < 0) + fatal("PTRACE_CONT remote syscall"); + if (wait_for_trap(pid) < 0) + fatal("wait remote syscall"); + if (get_regs(pid, &stopped) < 0) + fatal("PTRACE_GETREGSET result"); + result = (long)stopped.regs[0]; + + if (ptrace(PTRACE_POKETEXT, pid, (void *)saved.pc, (void *)original) < 0) + fatal("PTRACE_POKETEXT restore"); + if (set_regs(pid, &saved) < 0) + fatal("PTRACE_SETREGSET restore"); + return result; +} + +static void remote_write(pid_t pid, uintptr_t destination, const void *source, + size_t length) +{ + const unsigned char *bytes = source; + for (size_t offset = 0; offset < length; offset += sizeof(unsigned long)) { + unsigned long word = 0; + size_t chunk = length - offset; + if (chunk > sizeof(word)) + chunk = sizeof(word); + memcpy(&word, bytes + offset, chunk); + if (ptrace(PTRACE_POKEDATA, pid, (void *)(destination + offset), + (void *)word) < 0) + fatal("PTRACE_POKEDATA"); + } +} + +static void inject_handler(pid_t pid) +{ + const size_t page_size = (size_t)sysconf(_SC_PAGESIZE); + const size_t blob_size = (size_t)(esbuild_blob_end - esbuild_blob_start); + const size_t handler_offset = (size_t)(esbuild_handler - esbuild_blob_start); + const size_t slot_offset = + (size_t)(esbuild_return_offset_slot - esbuild_blob_start); + const size_t action_offset = (blob_size + 15U) & ~((size_t)15U); + const uint64_t context_offset = + offsetof(ucontext_t, uc_mcontext) + offsetof(mcontext_t, regs[0]); + struct kernel_sigaction_wire action = {0}; + unsigned char *blob; + long result; + uintptr_t remote_page; + + if (action_offset + sizeof(action) > page_size) + fatal_message("handler blob is larger than one page"); + result = remote_syscall(pid, SYS_mmap, 0, page_size, + PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_PRIVATE | MAP_ANONYMOUS, UINT64_MAX, 0); + if (result < 0 && result >= -4095) { + errno = (int)-result; + fatal("remote mmap RWX"); + } + remote_page = (uintptr_t)result; + + blob = malloc(blob_size); + if (blob == NULL) + fatal("malloc blob"); + memcpy(blob, esbuild_blob_start, blob_size); + memcpy(blob + slot_offset, &context_offset, sizeof(context_offset)); + remote_write(pid, remote_page, blob, blob_size); + free(blob); + + action.handler = remote_page + handler_offset; + action.flags = SA_SIGINFO; + remote_write(pid, remote_page + action_offset, &action, sizeof(action)); + result = remote_syscall(pid, SYS_rt_sigaction, SIGSYS, + remote_page + action_offset, 0, 8, 0, + GATEWAY_MAGIC); + if (result != 0) { + if (result < 0 && result >= -4095) + errno = (int)-result; + fatal("remote rt_sigaction"); + } + printf("esbuild-injector: handler=%#lx blob=%zu bytes (RWX experiment)\n", + (unsigned long)(remote_page + handler_offset), blob_size); +} + +static void install_filter(void) +{ + struct sock_filter insns[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_getpid, 4, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_openat, 3, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_rt_sigaction, 2, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_rt_sigprocmask, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, args[5])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, GATEWAY_MAGIC_LOW, 0, 3), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, args[5]) + 4), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, GATEWAY_MAGIC_HIGH, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_TRAP | 0x4653), + }; + struct sock_fprog program = { + .len = (unsigned short)ARRAY_LEN(insns), + .filter = insns, + }; + + if (syscall(SYS_prctl, 38, 1, 0, 0, 0) != 0) + fatal("PR_SET_NO_NEW_PRIVS"); + if (syscall(SYS_prctl, 22, SECCOMP_MODE_FILTER, &program, 0, 0) != 0) + fatal("PR_SET_SECCOMP"); +} + +static void child_main(char **target_argv) +{ + if (ptrace(PTRACE_TRACEME, 0, NULL, NULL) < 0) + fatal("PTRACE_TRACEME"); + if (raise(SIGSTOP) != 0) + fatal("raise SIGSTOP"); + install_filter(); + syscall(SYS_execve, target_argv[0], target_argv, environ); + fatal("execve target"); +} + +static void wait_for_exec(pid_t child) +{ + int status; + for (;;) { + if (waitpid(child, &status, 0) < 0) + fatal("waitpid exec"); + if (WIFEXITED(status) || WIFSIGNALED(status)) + fatal_message("child exited before PTRACE_EVENT_EXEC"); + if (!WIFSTOPPED(status)) + continue; + if ((unsigned int)status >> 16 == PTRACE_EVENT_EXEC) + return; + if (ptrace(PTRACE_CONT, child, NULL, + (void *)(uintptr_t)WSTOPSIG(status)) < 0) + fatal("PTRACE_CONT signal"); + } +} + +static uint64_t elapsed_ns(const struct timespec *start, const struct timespec *end) +{ + return (uint64_t)(end->tv_sec - start->tv_sec) * UINT64_C(1000000000) + + (uint64_t)(end->tv_nsec - start->tv_nsec); +} + +int main(int argc, char **argv) +{ + pid_t child; + int status; + struct timespec start; + struct timespec end; + + if (argc < 2) { + fprintf(stderr, "usage: %s /path/to/esbuild [arguments...]\n", argv[0]); + return EXIT_FAILURE; + } + child = fork(); + if (child < 0) + fatal("fork"); + if (child == 0) + child_main(&argv[1]); + + if (waitpid(child, &status, 0) < 0) + fatal("waitpid initial"); + if (!WIFSTOPPED(status) || WSTOPSIG(status) != SIGSTOP) + fatal_message("child did not stop before filter installation"); + if (ptrace(PTRACE_SETOPTIONS, child, NULL, + (void *)(PTRACE_O_TRACEEXEC | PTRACE_O_EXITKILL)) < 0) + fatal("PTRACE_SETOPTIONS"); + if (ptrace(PTRACE_CONT, child, NULL, NULL) < 0) + fatal("PTRACE_CONT exec"); + + wait_for_exec(child); + if (clock_gettime(CLOCK_MONOTONIC_RAW, &start) != 0) + fatal("clock_gettime start"); + inject_handler(child); + if (ptrace(PTRACE_DETACH, child, NULL, NULL) < 0) + fatal("PTRACE_DETACH"); + if (clock_gettime(CLOCK_MONOTONIC_RAW, &end) != 0) + fatal("clock_gettime end"); + printf("esbuild-injector: exec-stop to detach %.3f us\n", + (double)elapsed_ns(&start, &end) / 1000.0); + fflush(stdout); + + if (waitpid(child, &status, 0) < 0) + fatal("waitpid target"); + if (WIFEXITED(status)) { + printf("esbuild-injector: target exit=%d\n", WEXITSTATUS(status)); + return WEXITSTATUS(status) == 0 ? EXIT_SUCCESS : EXIT_FAILURE; + } + if (WIFSIGNALED(status)) + fprintf(stderr, "esbuild-injector: target signal=%d\n", WTERMSIG(status)); + else + fprintf(stderr, "esbuild-injector: target status=%#x\n", status); + return EXIT_FAILURE; +} diff --git a/research/ptrace-exec-prototype/esbuild_input.js b/research/ptrace-exec-prototype/esbuild_input.js new file mode 100644 index 000000000..a3e1dae32 --- /dev/null +++ b/research/ptrace-exec-prototype/esbuild_input.js @@ -0,0 +1,2 @@ +export const answer = 21 * 2; +console.log(`answer=${answer}`); diff --git a/research/ptrace-exec-prototype/injector.c b/research/ptrace-exec-prototype/injector.c new file mode 100644 index 000000000..6afd15305 --- /dev/null +++ b/research/ptrace-exec-prototype/injector.c @@ -0,0 +1,452 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(__aarch64__) && !defined(__x86_64__) +#error "This prototype supports native AArch64 and x86-64 only" +#endif + +#define TRAPPED_RESULT UINT64_C(0x51515151) +#define ARRAY_LEN(values) (sizeof(values) / sizeof((values)[0])) + +/* + * This entire section is copied into the post-exec address space. It must be + * position independent and must not reference the injector's GOT, PLT, TLS, + * stack protector, or libc. + * + * The ucontext return-register offset is patched in the local copy before it + * is written to the tracee. On x86-64 the raw kernel sigaction also points at + * the copied rt_sigreturn restorer. + */ +#if defined(__aarch64__) +__asm__( + ".pushsection .text.fspy_injected,\"ax\",@progbits\n" + ".balign 16\n" + ".global fspy_blob_start\n" + ".global fspy_handler\n" + ".global fspy_return_offset_slot\n" + ".global fspy_blob_end\n" + "fspy_blob_start:\n" + "fspy_handler:\n" + " adr x3, fspy_return_offset_slot\n" + " ldr x3, [x3]\n" + " movz x4, #0x5151\n" + " movk x4, #0x5151, lsl #16\n" + " str x4, [x2, x3]\n" + " ret\n" + ".balign 8\n" + "fspy_return_offset_slot:\n" + " .quad 0\n" + "fspy_blob_end:\n" + ".popsection\n"); +#elif defined(__x86_64__) +__asm__( + ".pushsection .text.fspy_injected,\"ax\",@progbits\n" + ".balign 16\n" + ".global fspy_blob_start\n" + ".global fspy_handler\n" + ".global fspy_restorer\n" + ".global fspy_return_offset_slot\n" + ".global fspy_blob_end\n" + "fspy_blob_start:\n" + "fspy_handler:\n" + " lea fspy_return_offset_slot(%rip), %rcx\n" + " mov (%rcx), %rcx\n" + " mov $0x51515151, %eax\n" + " mov %rax, (%rdx,%rcx)\n" + " ret\n" + ".balign 8\n" + "fspy_restorer:\n" + " mov $15, %rax\n" /* __NR_rt_sigreturn */ + " syscall\n" + " ud2\n" + ".balign 8\n" + "fspy_return_offset_slot:\n" + " .quad 0\n" + "fspy_blob_end:\n" + ".popsection\n"); +#endif + +extern const unsigned char fspy_blob_start[]; +extern const unsigned char fspy_handler[]; +extern const unsigned char fspy_return_offset_slot[]; +extern const unsigned char fspy_blob_end[]; +#if defined(__x86_64__) +extern const unsigned char fspy_restorer[]; +#endif + +struct kernel_sigaction_wire { + uint64_t handler; + uint64_t flags; + uint64_t restorer; + uint64_t mask; +}; + +static void fatal(const char *message) +{ + perror(message); + exit(EXIT_FAILURE); +} + +static void fatal_message(const char *message) +{ + fputs(message, stderr); + fputc('\n', stderr); + exit(EXIT_FAILURE); +} + +static int ptrace_get_regs(pid_t pid, struct user_regs_struct *regs) +{ + struct iovec iov = {.iov_base = regs, .iov_len = sizeof(*regs)}; + return (int)ptrace(PTRACE_GETREGSET, pid, (void *)(uintptr_t)NT_PRSTATUS, &iov); +} + +static int ptrace_set_regs(pid_t pid, const struct user_regs_struct *regs) +{ + struct iovec iov = {.iov_base = (void *)regs, .iov_len = sizeof(*regs)}; + return (int)ptrace(PTRACE_SETREGSET, pid, (void *)(uintptr_t)NT_PRSTATUS, &iov); +} + +static uintptr_t regs_pc(const struct user_regs_struct *regs) +{ +#if defined(__aarch64__) + return (uintptr_t)regs->pc; +#else + return (uintptr_t)regs->rip; +#endif +} + +static void prepare_remote_syscall(struct user_regs_struct *regs, long number, + const uint64_t args[6]) +{ +#if defined(__aarch64__) + regs->regs[0] = args[0]; + regs->regs[1] = args[1]; + regs->regs[2] = args[2]; + regs->regs[3] = args[3]; + regs->regs[4] = args[4]; + regs->regs[5] = args[5]; + regs->regs[8] = (uint64_t)number; +#else + regs->rax = (uint64_t)number; + regs->orig_rax = UINT64_MAX; + regs->rdi = args[0]; + regs->rsi = args[1]; + regs->rdx = args[2]; + regs->r10 = args[3]; + regs->r8 = args[4]; + regs->r9 = args[5]; +#endif +} + +static long remote_syscall_result(const struct user_regs_struct *regs) +{ +#if defined(__aarch64__) + return (long)regs->regs[0]; +#else + return (long)regs->rax; +#endif +} + +static unsigned long syscall_breakpoint_word(unsigned long original) +{ +#if defined(__aarch64__) + (void)original; + /* svc #0; brk #0, in little-endian instruction order. */ + return UINT64_C(0xd4200000d4000001); +#else + unsigned long patched = original; + unsigned char *bytes = (unsigned char *)&patched; + bytes[0] = 0x0f; /* syscall */ + bytes[1] = 0x05; + bytes[2] = 0xcc; /* int3 */ + return patched; +#endif +} + +static int wait_for_injected_breakpoint(pid_t pid) +{ + int status; + if (waitpid(pid, &status, 0) < 0) + return -1; + if (!WIFSTOPPED(status) || WSTOPSIG(status) != SIGTRAP) { + fprintf(stderr, "unexpected status during remote syscall: 0x%x\n", status); + errno = EPROTO; + return -1; + } + return 0; +} + +static long remote_syscall(pid_t pid, long number, uint64_t a0, uint64_t a1, + uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) +{ + struct user_regs_struct saved; + struct user_regs_struct call_regs; + struct user_regs_struct stopped; + const uint64_t args[6] = {a0, a1, a2, a3, a4, a5}; + unsigned long original_word; + uintptr_t pc; + long result; + + if (ptrace_get_regs(pid, &saved) < 0) + fatal("PTRACE_GETREGSET"); + call_regs = saved; + pc = regs_pc(&saved); + + errno = 0; + original_word = (unsigned long)ptrace(PTRACE_PEEKTEXT, pid, (void *)pc, NULL); + if (original_word == (unsigned long)-1 && errno != 0) + fatal("PTRACE_PEEKTEXT"); + + if (ptrace(PTRACE_POKETEXT, pid, (void *)pc, + (void *)syscall_breakpoint_word(original_word)) < 0) + fatal("PTRACE_POKETEXT syscall stub"); + + prepare_remote_syscall(&call_regs, number, args); + if (ptrace_set_regs(pid, &call_regs) < 0) + fatal("PTRACE_SETREGSET syscall arguments"); + if (ptrace(PTRACE_CONT, pid, NULL, NULL) < 0) + fatal("PTRACE_CONT remote syscall"); + if (wait_for_injected_breakpoint(pid) < 0) + fatal("waitpid remote syscall"); + if (ptrace_get_regs(pid, &stopped) < 0) + fatal("PTRACE_GETREGSET result"); + result = remote_syscall_result(&stopped); + + if (ptrace(PTRACE_POKETEXT, pid, (void *)pc, (void *)original_word) < 0) + fatal("PTRACE_POKETEXT restore"); + if (ptrace_set_regs(pid, &saved) < 0) + fatal("PTRACE_SETREGSET restore"); + return result; +} + +static void remote_write(pid_t pid, uintptr_t destination, const void *source, + size_t length) +{ + const unsigned char *bytes = source; + size_t offset = 0; + + while (offset < length) { + unsigned long word = 0; + size_t chunk = length - offset; + if (chunk > sizeof(word)) + chunk = sizeof(word); + memcpy(&word, bytes + offset, chunk); + if (ptrace(PTRACE_POKEDATA, pid, (void *)(destination + offset), + (void *)word) < 0) + fatal("PTRACE_POKEDATA"); + offset += chunk; + } +} + +static size_t return_register_offset(void) +{ +#if defined(__aarch64__) + return offsetof(ucontext_t, uc_mcontext) + offsetof(mcontext_t, regs[0]); +#else + return offsetof(ucontext_t, uc_mcontext) + + offsetof(mcontext_t, gregs[REG_RAX]); +#endif +} + +static void inject_sigsys_handler(pid_t pid) +{ + const size_t page_size = (size_t)sysconf(_SC_PAGESIZE); + const size_t blob_size = (size_t)(fspy_blob_end - fspy_blob_start); + const size_t offset_slot = + (size_t)(fspy_return_offset_slot - fspy_blob_start); + const size_t handler_offset = (size_t)(fspy_handler - fspy_blob_start); + const size_t action_offset = (blob_size + 15U) & ~((size_t)15U); + unsigned char *local_blob; + struct kernel_sigaction_wire action = {0}; + uintptr_t remote_page; + long result; + uint64_t context_offset; + + if (page_size == 0 || action_offset + sizeof(action) > page_size) + fatal_message("injected blob does not fit in one page"); + + result = remote_syscall(pid, SYS_mmap, 0, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, UINT64_MAX, 0); + if (result < 0 && result >= -4095) { + errno = (int)-result; + fatal("remote mmap"); + } + remote_page = (uintptr_t)result; + + local_blob = malloc(blob_size); + if (local_blob == NULL) + fatal("malloc local blob"); + memcpy(local_blob, fspy_blob_start, blob_size); + context_offset = return_register_offset(); + memcpy(local_blob + offset_slot, &context_offset, sizeof(context_offset)); + remote_write(pid, remote_page, local_blob, blob_size); + free(local_blob); + + action.handler = remote_page + handler_offset; + action.flags = SA_SIGINFO; +#if defined(__x86_64__) + action.flags |= 0x04000000UL; /* SA_RESTORER from Linux UAPI. */ + action.restorer = remote_page + (uintptr_t)(fspy_restorer - fspy_blob_start); +#endif + remote_write(pid, remote_page + action_offset, &action, sizeof(action)); + + result = remote_syscall(pid, SYS_rt_sigaction, SIGSYS, + remote_page + action_offset, 0, 8, 0, 0); + if (result != 0) { + if (result < 0 && result >= -4095) + errno = (int)-result; + fatal("remote rt_sigaction"); + } + + result = remote_syscall(pid, SYS_mprotect, remote_page, page_size, + PROT_READ | PROT_EXEC, 0, 0, 0); + if (result != 0) { + if (result < 0 && result >= -4095) + errno = (int)-result; + fatal("remote mprotect"); + } + + printf("injector: mapped handler at %#lx, handler=%#llx, blob=%zu bytes\n", + (unsigned long)remote_page, (unsigned long long)action.handler, blob_size); +} + +static void install_trap_filter(void) +{ + struct sock_filter instructions[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, nr)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_getpid, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_TRAP | 0x4653), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + }; + struct sock_fprog program = { + .len = (unsigned short)ARRAY_LEN(instructions), + .filter = instructions, + }; + + if (syscall(SYS_prctl, 38 /* PR_SET_NO_NEW_PRIVS */, 1, 0, 0, 0) != 0) + fatal("PR_SET_NO_NEW_PRIVS"); + if (syscall(SYS_prctl, 22 /* PR_SET_SECCOMP */, SECCOMP_MODE_FILTER, + &program, 0, 0) != 0) + fatal("PR_SET_SECCOMP"); +} + +static void child_main(const char *target) +{ + if (ptrace(PTRACE_TRACEME, 0, NULL, NULL) < 0) + fatal("PTRACE_TRACEME"); + if (raise(SIGSTOP) != 0) + fatal("raise SIGSTOP"); + + install_trap_filter(); + execl(target, target, NULL); + fatal("execl target"); +} + +static void wait_for_exec_stop(pid_t child) +{ + int status; + + for (;;) { + if (waitpid(child, &status, 0) < 0) + fatal("waitpid exec"); + if (WIFEXITED(status) || WIFSIGNALED(status)) + fatal_message("child exited before PTRACE_EVENT_EXEC"); + if (!WIFSTOPPED(status)) + continue; + + if ((unsigned int)status >> 16 == PTRACE_EVENT_EXEC) + return; + + if (ptrace(PTRACE_CONT, child, NULL, + (void *)(uintptr_t)WSTOPSIG(status)) < 0) + fatal("PTRACE_CONT forwarding signal"); + } +} + +static void finish_exec_syscall(pid_t child) +{ + int status; + + /* PTRACE_EVENT_EXEC happens before the original execve returns. Wait for + * its syscall-exit stop so that the kernel cannot overwrite registers + * prepared for the first injected syscall. */ + if (ptrace(PTRACE_SYSCALL, child, NULL, NULL) < 0) + fatal("PTRACE_SYSCALL after exec event"); + if (waitpid(child, &status, 0) < 0) + fatal("waitpid exec syscall exit"); + if (!WIFSTOPPED(status) || WSTOPSIG(status) != (SIGTRAP | 0x80) || + (unsigned int)status >> 16 != 0) + fatal_message("child did not reach the exec syscall-exit stop"); +} + +int main(int argc, char **argv) +{ + pid_t child; + int status; + unsigned long options = + PTRACE_O_TRACEEXEC | PTRACE_O_EXITKILL | PTRACE_O_TRACESYSGOOD; + + if (argc != 2) { + fprintf(stderr, "usage: %s /absolute/path/to/target\n", argv[0]); + return EXIT_FAILURE; + } + + child = fork(); + if (child < 0) + fatal("fork"); + if (child == 0) + child_main(argv[1]); + + if (waitpid(child, &status, 0) < 0) + fatal("waitpid initial stop"); + if (!WIFSTOPPED(status) || WSTOPSIG(status) != SIGSTOP) + fatal_message("child did not enter its initial SIGSTOP"); + + if (ptrace(PTRACE_SETOPTIONS, child, NULL, (void *)options) < 0) + fatal("PTRACE_SETOPTIONS"); + if (ptrace(PTRACE_CONT, child, NULL, NULL) < 0) + fatal("PTRACE_CONT to exec"); + + wait_for_exec_stop(child); + puts("injector: caught PTRACE_EVENT_EXEC before target entry"); + finish_exec_syscall(child); + inject_sigsys_handler(child); + + if (ptrace(PTRACE_DETACH, child, NULL, NULL) < 0) + fatal("PTRACE_DETACH"); + puts("injector: detached; target's trapped syscall now has no tracer"); + fflush(stdout); + + if (waitpid(child, &status, 0) < 0) + fatal("waitpid target"); + if (!WIFEXITED(status)) { + if (WIFSIGNALED(status)) + fprintf(stderr, "target died from signal %d\n", WTERMSIG(status)); + else + fprintf(stderr, "unexpected final status: 0x%x\n", status); + return EXIT_FAILURE; + } + printf("injector: target exit status %d\n", WEXITSTATUS(status)); + return WEXITSTATUS(status) == 0 ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/research/ptrace-exec-prototype/nested_trap.c b/research/ptrace-exec-prototype/nested_trap.c new file mode 100644 index 000000000..4ef98277d --- /dev/null +++ b/research/ptrace-exec-prototype/nested_trap.c @@ -0,0 +1,78 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define NESTED_TAG 0x1234 +#define NESTED_RESULT 0x12345678L +#define SECCOMP_SIGINFO_CODE 1 + +static volatile sig_atomic_t saw_nested_trap; + +static void logical_sigsys_handler(int signal_number, siginfo_t *info, + void *opaque_context) +{ + ucontext_t *context = opaque_context; + + if (signal_number != SIGSYS || info->si_code != SECCOMP_SIGINFO_CODE || + info->si_errno != NESTED_TAG || info->si_syscall != SYS_getppid) + _exit(120); + saw_nested_trap = 1; + context->uc_mcontext.gregs[REG_RAX] = NESTED_RESULT; +} + +static void install_nested_filter(void) +{ + struct sock_filter instructions[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, arch)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, AUDIT_ARCH_X86_64, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, nr)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_getppid, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_TRAP | NESTED_TAG), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + }; + struct sock_fprog program = { + .len = (unsigned short)(sizeof(instructions) / sizeof(instructions[0])), + .filter = instructions, + }; + + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) + _exit(121); + if (syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, 0, &program) != 0) + _exit(122); +} + +int main(void) +{ + struct sigaction action = {0}; + long result; + + sigemptyset(&action.sa_mask); + action.sa_sigaction = logical_sigsys_handler; + action.sa_flags = SA_SIGINFO; + if (sigaction(SIGSYS, &action, NULL) != 0) { + perror("sigaction logical SIGSYS"); + return EXIT_FAILURE; + } + install_nested_filter(); + result = syscall(SYS_getppid); + printf("nested: logical SIGSYS result=%#lx seen=%d\n", result, + saw_nested_trap); + if (result != NESTED_RESULT || !saw_nested_trap) + return EXIT_FAILURE; + puts("PASS: foreign seccomp TRAP reached the target's logical handler"); + return EXIT_SUCCESS; +} diff --git a/research/ptrace-exec-prototype/nonleader_exec.c b/research/ptrace-exec-prototype/nonleader_exec.c new file mode 100644 index 000000000..7ade8ba87 --- /dev/null +++ b/research/ptrace-exec-prototype/nonleader_exec.c @@ -0,0 +1,134 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern char **environ; + +static const char *exec_target; + +static void fatal(const char *message) +{ + perror(message); + exit(EXIT_FAILURE); +} + +static void *worker_exec(void *unused) +{ + char *const argv[] = {(char *)exec_target, NULL}; + (void)unused; + syscall(SYS_execve, exec_target, argv, environ); + _exit(111); +} + +static void child_main(void) +{ + pthread_t worker; + + if (ptrace(PTRACE_TRACEME, 0, NULL, NULL) < 0) + fatal("PTRACE_TRACEME"); + if (raise(SIGSTOP) != 0) + fatal("raise SIGSTOP"); + if (pthread_create(&worker, NULL, worker_exec, NULL) != 0) + fatal("pthread_create"); + + /* A successful exec by the worker destroys this thread. */ + for (;;) + pause(); +} + +int main(int argc, char **argv) +{ + const unsigned long options = + PTRACE_O_TRACECLONE | PTRACE_O_TRACEEXEC | PTRACE_O_EXITKILL; + pid_t child; + int status; + pid_t worker_tid = -1; + + if (argc != 2) { + fprintf(stderr, "usage: %s /absolute/path/to/target\n", argv[0]); + return EXIT_FAILURE; + } + exec_target = argv[1]; + child = fork(); + if (child < 0) + fatal("fork"); + if (child == 0) + child_main(); + + if (waitpid(child, &status, 0) < 0) + fatal("initial waitpid"); + if (!WIFSTOPPED(status) || WSTOPSIG(status) != SIGSTOP) { + fprintf(stderr, "unexpected initial status %#x\n", status); + return EXIT_FAILURE; + } + if (ptrace(PTRACE_SETOPTIONS, child, NULL, (void *)options) < 0) + fatal("PTRACE_SETOPTIONS"); + if (ptrace(PTRACE_CONT, child, NULL, NULL) < 0) + fatal("PTRACE_CONT initial"); + + for (;;) { + unsigned int event; + pid_t stopped_tid = waitpid(-1, &status, __WALL); + if (stopped_tid < 0) + fatal("waitpid trace event"); + if (WIFEXITED(status) || WIFSIGNALED(status)) + continue; + if (!WIFSTOPPED(status)) + continue; + + event = (unsigned int)status >> 16; + if (event == PTRACE_EVENT_CLONE) { + unsigned long message = 0; + if (ptrace(PTRACE_GETEVENTMSG, stopped_tid, NULL, &message) < 0) + fatal("PTRACE_GETEVENTMSG clone"); + worker_tid = (pid_t)message; + printf("nonleader: clone event leader=%d worker=%d\n", child, + worker_tid); + if (ptrace(PTRACE_CONT, stopped_tid, NULL, NULL) < 0) + fatal("PTRACE_CONT clone parent"); + continue; + } + if (event == PTRACE_EVENT_EXEC) { + unsigned long former_tid = 0; + if (ptrace(PTRACE_GETEVENTMSG, stopped_tid, NULL, &former_tid) < 0) + fatal("PTRACE_GETEVENTMSG exec"); + printf("nonleader: exec stop reported as tid=%d; former tid=%lu\n", + stopped_tid, former_tid); + if (stopped_tid != child || former_tid != (unsigned long)worker_tid || + former_tid == (unsigned long)child) { + fputs("FAIL: non-leader exec TID transition was unexpected\n", + stderr); + return EXIT_FAILURE; + } + if (ptrace(PTRACE_DETACH, stopped_tid, NULL, NULL) < 0) + fatal("PTRACE_DETACH exec"); + break; + } + + /* Automatically attached clone children initially report SIGSTOP. */ + if (ptrace(PTRACE_CONT, stopped_tid, NULL, + WSTOPSIG(status) == SIGSTOP ? NULL + : (void *)(uintptr_t)WSTOPSIG(status)) < 0 && + errno != ESRCH) + fatal("PTRACE_CONT other stop"); + } + + if (waitpid(child, &status, 0) < 0) + fatal("waitpid final target"); + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + fprintf(stderr, "FAIL: target final status %#x\n", status); + return EXIT_FAILURE; + } + puts("PASS: PTRACE_GETEVENTMSG preserved the non-leader's former TID"); + return EXIT_SUCCESS; +} diff --git a/research/ptrace-exec-prototype/recursive_injector.c b/research/ptrace-exec-prototype/recursive_injector.c new file mode 100644 index 000000000..562c1d25a --- /dev/null +++ b/research/ptrace-exec-prototype/recursive_injector.c @@ -0,0 +1,1029 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(__x86_64__) +#error "The recursive browser prototype currently supports native x86-64 only" +#endif + +extern char **environ; + +#define ARRAY_LEN(values) (sizeof(values) / sizeof((values)[0])) +#define GATEWAY_MAGIC UINT64_C(0x4653505947415445) +#define GATEWAY_MAGIC_LOW UINT32_C(0x47415445) +#define FILTER_TAG UINT32_C(0x4653) +#define BRIDGE_SIGNAL (SIGRTMIN + 6) +#define VIRTUAL_ACTION_OFFSET 0U +#define INSTALL_ACTION_OFFSET 64U + +/* + * The copied handler contains no relocations, GOT/PLT references, TLS access, + * libc calls, or stack-protector references. Runtime-specific values are + * patched into the slots at the end of the blob before it is copied. + */ +__asm__( + ".pushsection .text.fspy_recursive_injected,\"ax\",@progbits\n" + ".balign 16\n" + ".global fspy_recursive_blob_start\n" + ".global fspy_recursive_handler\n" + ".global fspy_recursive_restorer\n" + ".global fspy_recursive_slot_rax\n" + ".global fspy_recursive_slot_rdi\n" + ".global fspy_recursive_slot_rsi\n" + ".global fspy_recursive_slot_rdx\n" + ".global fspy_recursive_slot_r10\n" + ".global fspy_recursive_slot_r8\n" + ".global fspy_recursive_slot_r9\n" + ".global fspy_recursive_slot_sigmask\n" + ".global fspy_recursive_slot_supervisor\n" + ".global fspy_recursive_slot_signal\n" + ".global fspy_recursive_slot_state\n" + ".global fspy_recursive_slot_magic\n" + ".global fspy_recursive_blob_end\n" + "fspy_recursive_blob_start:\n" + "fspy_recursive_handler:\n" + " push %rbp\n" + " mov %rsp, %rbp\n" + " push %rbx\n" + " push %r12\n" + " push %r13\n" + " push %r14\n" + " push %r15\n" + " sub $168, %rsp\n" /* keep the stack aligned for a logical handler */ + " mov %rsi, %r12\n" /* siginfo_t * */ + " mov %rdx, %r13\n" /* ucontext_t * */ + " mov fspy_recursive_slot_state(%rip), %r14\n" + /* A stacked filter may also return TRAP. Only fspy's filter tag belongs to + * this dispatcher; all other traps belong to the target's logical action. */ + " cmpl $0x4653, 4(%r12)\n" /* siginfo_t.si_errno */ + " jne .Lfspy_logical_sigsys\n" + " mov 24(%r12), %eax\n" /* siginfo_t.si_syscall */ + " cmp $39, %eax\n" /* __NR_getpid */ + " je .Lfspy_getpid\n" + " cmp $59, %eax\n" /* __NR_execve */ + " je .Lfspy_exec\n" + " cmp $322, %eax\n" /* __NR_execveat */ + " je .Lfspy_exec\n" + " cmp $13, %eax\n" /* __NR_rt_sigaction */ + " je .Lfspy_sigaction\n" + " cmp $14, %eax\n" /* __NR_rt_sigprocmask */ + " je .Lfspy_sigprocmask\n" + " cmp $47, %eax\n" /* __NR_recvmsg; diagnostic for Chromium zygote */ + " je .Lfspy_recvmsg\n" + " cmp $217, %eax\n" /* __NR_getdents64 */ + " je .Lfspy_passthrough\n" + " cmp $257, %eax\n" /* __NR_openat */ + " je .Lfspy_passthrough\n" + " cmp $262, %eax\n" /* __NR_newfstatat */ + " je .Lfspy_passthrough\n" + " cmp $269, %eax\n" /* __NR_faccessat */ + " je .Lfspy_passthrough\n" + " cmp $332, %eax\n" /* __NR_statx */ + " je .Lfspy_passthrough\n" + " cmp $437, %eax\n" /* __NR_openat2 */ + " je .Lfspy_passthrough\n" + " cmp $439, %eax\n" /* __NR_faccessat2 */ + " je .Lfspy_passthrough\n" + " mov $-38, %rax\n" /* -ENOSYS */ + " jmp .Lfspy_store_result\n" + + ".Lfspy_logical_sigsys:\n" + " mov 0(%r14), %rax\n" /* virtual sa_handler/sa_sigaction */ + " test %rax, %rax\n" /* SIG_DFL */ + " je .Lfspy_logical_default\n" + " cmp $1, %rax\n" /* SIG_IGN */ + " je .Lfspy_return\n" + " mov 8(%r14), %rcx\n" /* virtual sa_flags */ + " test $4, %ecx\n" /* SA_SIGINFO */ + " je .Lfspy_logical_one_arg\n" + " mov $31, %edi\n" + " mov %r12, %rsi\n" + " mov %r13, %rdx\n" + " call *%rax\n" + " jmp .Lfspy_return\n" + ".Lfspy_logical_one_arg:\n" + " mov $31, %edi\n" + " call *%rax\n" + " jmp .Lfspy_return\n" + ".Lfspy_logical_default:\n" + " mov $159, %edi\n" /* prototype equivalent of default SIGSYS death */ + " mov $231, %eax\n" /* __NR_exit_group */ + " syscall\n" + " ud2\n" + + ".Lfspy_getpid:\n" + " mov $39, %eax\n" + " mov fspy_recursive_slot_magic(%rip), %r9\n" + " syscall\n" + " jmp .Lfspy_store_result\n" + + /* Chromium's namespace sandbox performs a blocking recvmsg immediately + * after launching its zygote. Keep this diagnostic in the research proof + * until the transient-ptrace compatibility question is resolved. */ + ".Lfspy_recvmsg:\n" + " mov fspy_recursive_slot_rdi(%rip), %rcx\n" + " mov (%r13,%rcx), %rdi\n" + " mov fspy_recursive_slot_rsi(%rip), %rcx\n" + " mov (%r13,%rcx), %rsi\n" + " mov fspy_recursive_slot_rdx(%rip), %rcx\n" + " mov (%r13,%rcx), %rdx\n" + " mov fspy_recursive_slot_magic(%rip), %r9\n" + " mov $47, %eax\n" /* __NR_recvmsg */ + " syscall\n" + " mov %rax, %r15\n" + " test %rax, %rax\n" + " js .Lfspy_recvmsg_error\n" + " jz .Lfspy_recvmsg_eof\n" + " lea .Lfspy_recvmsg_positive_message(%rip), %rsi\n" + " mov $32, %edx\n" + " jmp .Lfspy_recvmsg_log\n" + ".Lfspy_recvmsg_eof:\n" + " lea .Lfspy_recvmsg_eof_message(%rip), %rsi\n" + " mov $27, %edx\n" + " jmp .Lfspy_recvmsg_log\n" + ".Lfspy_recvmsg_error:\n" + " cmp $-4, %rax\n" /* -EINTR */ + " je .Lfspy_recvmsg_eintr\n" + " cmp $-2, %rax\n" /* -ENOENT */ + " je .Lfspy_recvmsg_enoent\n" + " lea .Lfspy_recvmsg_error_message(%rip), %rsi\n" + " mov $29, %edx\n" + " jmp .Lfspy_recvmsg_log\n" + ".Lfspy_recvmsg_eintr:\n" + " lea .Lfspy_recvmsg_eintr_message(%rip), %rsi\n" + " mov $29, %edx\n" + " jmp .Lfspy_recvmsg_log\n" + ".Lfspy_recvmsg_enoent:\n" + " lea .Lfspy_recvmsg_enoent_message(%rip), %rsi\n" + " mov $30, %edx\n" + ".Lfspy_recvmsg_log:\n" + " mov $2, %edi\n" + " mov $1, %eax\n" /* __NR_write */ + " syscall\n" + " mov %r15, %rax\n" + " jmp .Lfspy_store_result\n" + + ".Lfspy_passthrough:\n" + " mov %eax, %ebx\n" + " mov fspy_recursive_slot_rdi(%rip), %rcx\n" + " mov (%r13,%rcx), %rdi\n" + " mov fspy_recursive_slot_rsi(%rip), %rcx\n" + " mov (%r13,%rcx), %rsi\n" + " mov fspy_recursive_slot_rdx(%rip), %rcx\n" + " mov (%r13,%rcx), %rdx\n" + " mov fspy_recursive_slot_r10(%rip), %rcx\n" + " mov (%r13,%rcx), %r10\n" + " mov fspy_recursive_slot_r8(%rip), %rcx\n" + " mov (%r13,%rcx), %r8\n" + " mov fspy_recursive_slot_magic(%rip), %r9\n" + " mov %ebx, %eax\n" + " syscall\n" + " jmp .Lfspy_store_result\n" + + ".Lfspy_sigaction:\n" + " mov fspy_recursive_slot_rdi(%rip), %rcx\n" + " mov (%r13,%rcx), %rdi\n" + " cmp $31, %edi\n" /* SIGSYS */ + " jne .Lfspy_sigaction_passthrough\n" + " mov fspy_recursive_slot_rdx(%rip), %rcx\n" + " mov (%r13,%rcx), %rdi\n" /* oldact */ + " test %rdi, %rdi\n" + " je .Lfspy_sigaction_no_old\n" + " mov 0(%r14), %rax\n" + " mov %rax, 0(%rdi)\n" + " mov 8(%r14), %rax\n" + " mov %rax, 8(%rdi)\n" + " mov 16(%r14), %rax\n" + " mov %rax, 16(%rdi)\n" + " mov 24(%r14), %rax\n" + " mov %rax, 24(%rdi)\n" + ".Lfspy_sigaction_no_old:\n" + " mov fspy_recursive_slot_rsi(%rip), %rcx\n" + " mov (%r13,%rcx), %rsi\n" /* act */ + " test %rsi, %rsi\n" + " je .Lfspy_sigaction_done\n" + " mov 0(%rsi), %rax\n" + " mov %rax, 0(%r14)\n" + " mov 8(%rsi), %rax\n" + " mov %rax, 8(%r14)\n" + " mov 16(%rsi), %rax\n" + " mov %rax, 16(%r14)\n" + " mov 24(%rsi), %rax\n" + " mov %rax, 24(%r14)\n" + ".Lfspy_sigaction_done:\n" + " xor %eax, %eax\n" + " jmp .Lfspy_store_result\n" + ".Lfspy_sigaction_passthrough:\n" + " mov fspy_recursive_slot_rsi(%rip), %rcx\n" + " mov (%r13,%rcx), %rsi\n" + " mov fspy_recursive_slot_rdx(%rip), %rcx\n" + " mov (%r13,%rcx), %rdx\n" + " mov fspy_recursive_slot_r10(%rip), %rcx\n" + " mov (%r13,%rcx), %r10\n" + " mov $13, %eax\n" + " mov fspy_recursive_slot_magic(%rip), %r9\n" + " syscall\n" + " jmp .Lfspy_store_result\n" + + ".Lfspy_sigprocmask:\n" + " mov fspy_recursive_slot_rdx(%rip), %rcx\n" + " mov (%r13,%rcx), %rdx\n" /* oldset */ + " test %rdx, %rdx\n" + " je .Lfspy_sigmask_no_old\n" + " mov fspy_recursive_slot_sigmask(%rip), %rcx\n" + " mov (%r13,%rcx), %rax\n" + " mov %rax, (%rdx)\n" + ".Lfspy_sigmask_no_old:\n" + " mov fspy_recursive_slot_rsi(%rip), %rcx\n" + " mov (%r13,%rcx), %rsi\n" /* set */ + " test %rsi, %rsi\n" + " je .Lfspy_sigmask_success\n" + " mov fspy_recursive_slot_r10(%rip), %rcx\n" + " cmpq $8, (%r13,%rcx)\n" + " jne .Lfspy_sigmask_einval\n" + " mov fspy_recursive_slot_sigmask(%rip), %rcx\n" + " mov (%r13,%rcx), %rax\n" /* pre-signal mask */ + " mov (%rsi), %rdx\n" + " mov fspy_recursive_slot_rdi(%rip), %rcx\n" + " mov (%r13,%rcx), %rdi\n" /* how */ + " test %edi, %edi\n" /* SIG_BLOCK */ + " je .Lfspy_sigmask_block\n" + " cmp $1, %edi\n" /* SIG_UNBLOCK */ + " je .Lfspy_sigmask_unblock\n" + " cmp $2, %edi\n" /* SIG_SETMASK */ + " jne .Lfspy_sigmask_einval\n" + " mov %rdx, %rax\n" + " jmp .Lfspy_sigmask_apply\n" + ".Lfspy_sigmask_block:\n" + " or %rdx, %rax\n" + " jmp .Lfspy_sigmask_apply\n" + ".Lfspy_sigmask_unblock:\n" + " not %rdx\n" + " and %rdx, %rax\n" + ".Lfspy_sigmask_apply:\n" + " btr $30, %rax\n" /* SIGSYS must remain unblocked. */ + " mov fspy_recursive_slot_sigmask(%rip), %rcx\n" + " mov %rax, (%r13,%rcx)\n" + ".Lfspy_sigmask_success:\n" + " xor %eax, %eax\n" + " jmp .Lfspy_store_result\n" + ".Lfspy_sigmask_einval:\n" + " mov $-22, %rax\n" + " jmp .Lfspy_store_result\n" + + ".Lfspy_exec:\n" + " movq $0, -48(%rbp)\n" /* supervisor-release flag */ + " lea -192(%rbp), %rdi\n" + " xor %eax, %eax\n" + " mov $16, %ecx\n" + " rep stosq\n" /* clear a 128-byte siginfo_t */ + " mov fspy_recursive_slot_signal(%rip), %eax\n" + " mov %eax, -192(%rbp)\n" /* si_signo */ + " movl $-1, -184(%rbp)\n" /* si_code = SI_QUEUE */ + " mov $186, %eax\n" /* __NR_gettid */ + " syscall\n" + " mov %eax, %ebx\n" + " mov %eax, -176(%rbp)\n" /* si_pid: exact requesting TID */ + " mov $102, %eax\n" /* __NR_getuid */ + " syscall\n" + " mov %eax, -172(%rbp)\n" /* si_uid */ + " lea -48(%rbp), %rax\n" + " mov %rax, -168(%rbp)\n" /* si_value.sival_ptr */ + " mov fspy_recursive_slot_supervisor(%rip), %rdi\n" + " mov fspy_recursive_slot_signal(%rip), %rsi\n" + " lea -192(%rbp), %rdx\n" + " mov $129, %eax\n" /* __NR_rt_sigqueueinfo */ + " syscall\n" + " test %rax, %rax\n" + " js .Lfspy_bridge_failed\n" + ".Lfspy_wait_for_supervisor:\n" + " cmpq $0, -48(%rbp)\n" + " jne .Lfspy_exec_ready\n" + " lea -48(%rbp), %rdi\n" + " mov $128, %esi\n" /* FUTEX_WAIT_PRIVATE */ + " xor %edx, %edx\n" + " xor %r10d, %r10d\n" + " xor %r8d, %r8d\n" + " xor %r9d, %r9d\n" + " mov $202, %eax\n" /* __NR_futex */ + " syscall\n" + " jmp .Lfspy_wait_for_supervisor\n" + ".Lfspy_exec_ready:\n" + /* A successful exec never reaches rt_sigreturn, so undo the kernel's + * automatic SIGSYS block before replacing the image. */ + " movq $0x40000000, -56(%rbp)\n" + " mov $1, %edi\n" /* SIG_UNBLOCK */ + " lea -56(%rbp), %rsi\n" + " xor %edx, %edx\n" + " mov $8, %r10d\n" + " mov fspy_recursive_slot_magic(%rip), %r9\n" + " mov $14, %eax\n" /* __NR_rt_sigprocmask */ + " syscall\n" + " test %rax, %rax\n" + " js .Lfspy_bridge_failed\n" + " mov 24(%r12), %ebx\n" + " mov fspy_recursive_slot_rdi(%rip), %rcx\n" + " mov (%r13,%rcx), %rdi\n" + " mov fspy_recursive_slot_rsi(%rip), %rcx\n" + " mov (%r13,%rcx), %rsi\n" + " mov fspy_recursive_slot_rdx(%rip), %rcx\n" + " mov (%r13,%rcx), %rdx\n" + " mov fspy_recursive_slot_r10(%rip), %rcx\n" + " mov (%r13,%rcx), %r10\n" + " mov fspy_recursive_slot_r8(%rip), %rcx\n" + " mov (%r13,%rcx), %r8\n" + " mov fspy_recursive_slot_magic(%rip), %r9\n" + " mov %ebx, %eax\n" + " syscall\n" + " jmp .Lfspy_store_result\n" /* only a failed exec returns */ + ".Lfspy_bridge_failed:\n" + " mov $121, %edi\n" + " mov $231, %eax\n" /* __NR_exit_group */ + " syscall\n" + " ud2\n" + + ".Lfspy_store_result:\n" + " mov fspy_recursive_slot_rax(%rip), %rcx\n" + " mov %rax, (%r13,%rcx)\n" + ".Lfspy_return:\n" + " add $168, %rsp\n" + " pop %r15\n" + " pop %r14\n" + " pop %r13\n" + " pop %r12\n" + " pop %rbx\n" + " pop %rbp\n" + " ret\n" + ".balign 8\n" + "fspy_recursive_restorer:\n" + " mov $15, %eax\n" /* __NR_rt_sigreturn */ + " syscall\n" + " ud2\n" + ".Lfspy_recvmsg_positive_message: .ascii \"fspy: recvmsg returned positive\\n\"\n" + ".Lfspy_recvmsg_eof_message: .ascii \"fspy: recvmsg returned EOF\\n\"\n" + ".Lfspy_recvmsg_error_message: .ascii \"fspy: recvmsg returned error\\n\"\n" + ".Lfspy_recvmsg_eintr_message: .ascii \"fspy: recvmsg returned EINTR\\n\"\n" + ".Lfspy_recvmsg_enoent_message: .ascii \"fspy: recvmsg returned ENOENT\\n\"\n" + ".balign 8\n" + "fspy_recursive_slot_rax: .quad 0\n" + "fspy_recursive_slot_rdi: .quad 0\n" + "fspy_recursive_slot_rsi: .quad 0\n" + "fspy_recursive_slot_rdx: .quad 0\n" + "fspy_recursive_slot_r10: .quad 0\n" + "fspy_recursive_slot_r8: .quad 0\n" + "fspy_recursive_slot_r9: .quad 0\n" + "fspy_recursive_slot_sigmask: .quad 0\n" + "fspy_recursive_slot_supervisor: .quad 0\n" + "fspy_recursive_slot_signal: .quad 0\n" + "fspy_recursive_slot_state: .quad 0\n" + "fspy_recursive_slot_magic: .quad 0\n" + "fspy_recursive_blob_end:\n" + ".popsection\n"); + +extern const unsigned char fspy_recursive_blob_start[]; +extern const unsigned char fspy_recursive_handler[]; +extern const unsigned char fspy_recursive_restorer[]; +extern const unsigned char fspy_recursive_slot_rax[]; +extern const unsigned char fspy_recursive_slot_rdi[]; +extern const unsigned char fspy_recursive_slot_rsi[]; +extern const unsigned char fspy_recursive_slot_rdx[]; +extern const unsigned char fspy_recursive_slot_r10[]; +extern const unsigned char fspy_recursive_slot_r8[]; +extern const unsigned char fspy_recursive_slot_r9[]; +extern const unsigned char fspy_recursive_slot_sigmask[]; +extern const unsigned char fspy_recursive_slot_supervisor[]; +extern const unsigned char fspy_recursive_slot_signal[]; +extern const unsigned char fspy_recursive_slot_state[]; +extern const unsigned char fspy_recursive_slot_magic[]; +extern const unsigned char fspy_recursive_blob_end[]; + +struct kernel_sigaction_wire { + uint64_t handler; + uint64_t flags; + uint64_t restorer; + uint64_t mask; +}; + +struct bridge_state { + pid_t root_pid; + atomic_bool stopping; + atomic_bool root_reaped; + atomic_int root_status; + atomic_uint exec_count; + atomic_uint failed_exec_count; +}; + +static pid_t cleanup_process_group = -1; + +static void cleanup_children(void) +{ + if (cleanup_process_group > 0) + (void)kill(-cleanup_process_group, SIGKILL); +} + +static void fatal(const char *message) +{ + perror(message); + exit(EXIT_FAILURE); +} + +static void fatal_message(const char *message) +{ + fputs(message, stderr); + fputc('\n', stderr); + exit(EXIT_FAILURE); +} + +static size_t blob_offset(const unsigned char *symbol) +{ + return (size_t)(symbol - fspy_recursive_blob_start); +} + +static void patch_u64(unsigned char *blob, const unsigned char *slot, + uint64_t value) +{ + memcpy(blob + blob_offset(slot), &value, sizeof(value)); +} + +static unsigned char *prepare_blob(uintptr_t state_address, + pid_t supervisor_pid) +{ + const size_t blob_size = + (size_t)(fspy_recursive_blob_end - fspy_recursive_blob_start); + unsigned char *blob = malloc(blob_size); + + if (blob == NULL) + fatal("malloc injected blob"); + memcpy(blob, fspy_recursive_blob_start, blob_size); + patch_u64(blob, fspy_recursive_slot_rax, + offsetof(ucontext_t, uc_mcontext) + + offsetof(mcontext_t, gregs[REG_RAX])); + patch_u64(blob, fspy_recursive_slot_rdi, + offsetof(ucontext_t, uc_mcontext) + + offsetof(mcontext_t, gregs[REG_RDI])); + patch_u64(blob, fspy_recursive_slot_rsi, + offsetof(ucontext_t, uc_mcontext) + + offsetof(mcontext_t, gregs[REG_RSI])); + patch_u64(blob, fspy_recursive_slot_rdx, + offsetof(ucontext_t, uc_mcontext) + + offsetof(mcontext_t, gregs[REG_RDX])); + patch_u64(blob, fspy_recursive_slot_r10, + offsetof(ucontext_t, uc_mcontext) + + offsetof(mcontext_t, gregs[REG_R10])); + patch_u64(blob, fspy_recursive_slot_r8, + offsetof(ucontext_t, uc_mcontext) + + offsetof(mcontext_t, gregs[REG_R8])); + patch_u64(blob, fspy_recursive_slot_r9, + offsetof(ucontext_t, uc_mcontext) + + offsetof(mcontext_t, gregs[REG_R9])); + patch_u64(blob, fspy_recursive_slot_sigmask, + offsetof(ucontext_t, uc_sigmask)); + patch_u64(blob, fspy_recursive_slot_supervisor, + (uint64_t)supervisor_pid); + patch_u64(blob, fspy_recursive_slot_signal, BRIDGE_SIGNAL); + patch_u64(blob, fspy_recursive_slot_state, state_address); + patch_u64(blob, fspy_recursive_slot_magic, GATEWAY_MAGIC); + return blob; +} + +static struct kernel_sigaction_wire make_install_action(uintptr_t code_address) +{ + struct kernel_sigaction_wire action = {0}; + + action.handler = code_address + blob_offset(fspy_recursive_handler); + action.flags = SA_SIGINFO | SA_NODEFER | 0x04000000UL; /* SA_RESTORER */ + action.restorer = code_address + blob_offset(fspy_recursive_restorer); + return action; +} + +static void install_initial_handler(void) +{ + const size_t page_size = (size_t)sysconf(_SC_PAGESIZE); + const size_t blob_size = + (size_t)(fspy_recursive_blob_end - fspy_recursive_blob_start); + unsigned char *mapping; + unsigned char *blob; + struct kernel_sigaction_wire action; + + if (page_size == 0 || blob_size > page_size) + fatal_message("recursive injected blob does not fit in one page"); + mapping = mmap(NULL, page_size * 2, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED) + fatal("mmap initial handler"); + blob = prepare_blob((uintptr_t)mapping + page_size, getppid()); + memcpy(mapping, blob, blob_size); + free(blob); + action = make_install_action((uintptr_t)mapping); + if (syscall(SYS_rt_sigaction, SIGSYS, &action, NULL, 8) != 0) + fatal("rt_sigaction initial handler"); + if (mprotect(mapping, page_size, PROT_READ | PROT_EXEC) != 0) + fatal("mprotect initial handler"); +} + +#define FILTER_GATEWAY_BLOCK(syscall_number, tag) \ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, (syscall_number), 0, 4), \ + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, \ + offsetof(struct seccomp_data, args[5])), \ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, GATEWAY_MAGIC_LOW, 1, 0), \ + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_TRAP | (tag)), \ + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW) + +static void install_filter(void) +{ + struct sock_filter full_instructions[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, arch)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, AUDIT_ARCH_X86_64, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, nr)), + FILTER_GATEWAY_BLOCK(SYS_execve, FILTER_TAG), + FILTER_GATEWAY_BLOCK(SYS_execveat, FILTER_TAG), + FILTER_GATEWAY_BLOCK(SYS_getpid, FILTER_TAG), + FILTER_GATEWAY_BLOCK(SYS_getdents64, FILTER_TAG), + FILTER_GATEWAY_BLOCK(SYS_openat, FILTER_TAG), + FILTER_GATEWAY_BLOCK(SYS_openat2, FILTER_TAG), + FILTER_GATEWAY_BLOCK(SYS_newfstatat, FILTER_TAG), + FILTER_GATEWAY_BLOCK(SYS_statx, FILTER_TAG), + FILTER_GATEWAY_BLOCK(SYS_faccessat, FILTER_TAG), + FILTER_GATEWAY_BLOCK(SYS_faccessat2, FILTER_TAG), + FILTER_GATEWAY_BLOCK(SYS_rt_sigaction, FILTER_TAG), + FILTER_GATEWAY_BLOCK(SYS_rt_sigprocmask, FILTER_TAG), + FILTER_GATEWAY_BLOCK(SYS_recvmsg, FILTER_TAG), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + }; + struct sock_filter minimal_instructions[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, arch)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, AUDIT_ARCH_X86_64, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, nr)), + FILTER_GATEWAY_BLOCK(SYS_execve, FILTER_TAG), + FILTER_GATEWAY_BLOCK(SYS_execveat, FILTER_TAG), + FILTER_GATEWAY_BLOCK(SYS_rt_sigaction, FILTER_TAG), + FILTER_GATEWAY_BLOCK(SYS_rt_sigprocmask, FILTER_TAG), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + }; + const bool minimal = getenv("FSPY_MINIMAL_FILTER") != NULL; + struct sock_fprog program = { + .len = (unsigned short)(minimal ? ARRAY_LEN(minimal_instructions) + : ARRAY_LEN(full_instructions)), + .filter = minimal ? minimal_instructions : full_instructions, + }; + + fprintf(stderr, "bridge: installing %s syscall filter\n", + minimal ? "minimal exec/signal" : "full research"); + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) + fatal("PR_SET_NO_NEW_PRIVS"); + if (syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, 0, &program) != 0) + fatal("SECCOMP_SET_MODE_FILTER"); +} + +static int ptrace_get_regs(pid_t pid, struct user_regs_struct *regs) +{ + struct iovec iov = {.iov_base = regs, .iov_len = sizeof(*regs)}; + return (int)ptrace(PTRACE_GETREGSET, pid, (void *)(uintptr_t)NT_PRSTATUS, + &iov); +} + +static int ptrace_set_regs(pid_t pid, const struct user_regs_struct *regs) +{ + struct iovec iov = {.iov_base = (void *)regs, .iov_len = sizeof(*regs)}; + return (int)ptrace(PTRACE_SETREGSET, pid, (void *)(uintptr_t)NT_PRSTATUS, + &iov); +} + +static int wait_for_injected_breakpoint(pid_t pid) +{ + int status; + + if (waitpid(pid, &status, __WALL) < 0) + return -1; + if (!WIFSTOPPED(status) || WSTOPSIG(status) != SIGTRAP) { + fprintf(stderr, "unexpected remote-syscall status: %#x\n", status); + errno = EPROTO; + return -1; + } + return 0; +} + +static long remote_syscall(pid_t pid, long number, uint64_t a0, uint64_t a1, + uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) +{ + struct user_regs_struct saved; + struct user_regs_struct call_regs; + struct user_regs_struct stopped; + unsigned long original_word; + unsigned long patched_word; + uintptr_t pc; + long result; + + if (ptrace_get_regs(pid, &saved) < 0) + fatal("PTRACE_GETREGSET remote syscall"); + call_regs = saved; + pc = (uintptr_t)saved.rip; + errno = 0; + original_word = + (unsigned long)ptrace(PTRACE_PEEKTEXT, pid, (void *)pc, NULL); + if (original_word == (unsigned long)-1 && errno != 0) + fatal("PTRACE_PEEKTEXT remote syscall"); + patched_word = original_word; + ((unsigned char *)&patched_word)[0] = 0x0f; /* syscall */ + ((unsigned char *)&patched_word)[1] = 0x05; + ((unsigned char *)&patched_word)[2] = 0xcc; /* int3 */ + if (ptrace(PTRACE_POKETEXT, pid, (void *)pc, (void *)patched_word) < 0) + fatal("PTRACE_POKETEXT remote syscall"); + call_regs.rax = (uint64_t)number; + call_regs.orig_rax = UINT64_MAX; + call_regs.rdi = a0; + call_regs.rsi = a1; + call_regs.rdx = a2; + call_regs.r10 = a3; + call_regs.r8 = a4; + call_regs.r9 = a5; + if (ptrace_set_regs(pid, &call_regs) < 0) + fatal("PTRACE_SETREGSET remote syscall"); + if (ptrace(PTRACE_CONT, pid, NULL, NULL) < 0) + fatal("PTRACE_CONT remote syscall"); + if (wait_for_injected_breakpoint(pid) < 0) + fatal("waitpid remote syscall"); + if (ptrace_get_regs(pid, &stopped) < 0) + fatal("PTRACE_GETREGSET remote result"); + result = (long)stopped.rax; + if (ptrace(PTRACE_POKETEXT, pid, (void *)pc, (void *)original_word) < 0) + fatal("PTRACE_POKETEXT restore"); + if (ptrace_set_regs(pid, &saved) < 0) + fatal("PTRACE_SETREGSET restore"); + return result; +} + +static void remote_write(pid_t pid, uintptr_t destination, const void *source, + size_t length) +{ + const unsigned char *bytes = source; + size_t offset = 0; + + while (offset < length) { + unsigned long word = 0; + size_t chunk = length - offset; + if (chunk > sizeof(word)) + chunk = sizeof(word); + memcpy(&word, bytes + offset, chunk); + if (ptrace(PTRACE_POKEDATA, pid, (void *)(destination + offset), + (void *)word) < 0) + fatal("PTRACE_POKEDATA injected blob"); + offset += chunk; + } +} + +static void inject_handler(pid_t pid) +{ + const size_t page_size = (size_t)sysconf(_SC_PAGESIZE); + const size_t blob_size = + (size_t)(fspy_recursive_blob_end - fspy_recursive_blob_start); + unsigned char *blob; + struct kernel_sigaction_wire action; + uintptr_t remote_code; + uintptr_t remote_state; + long result; + + if (page_size == 0 || blob_size > page_size || + INSTALL_ACTION_OFFSET + sizeof(action) > page_size) + fatal_message("recursive injected mapping layout is invalid"); + result = remote_syscall(pid, SYS_mmap, 0, page_size * 2, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, UINT64_MAX, 0); + if (result < 0 && result >= -4095) { + errno = (int)-result; + fatal("remote mmap recursive handler"); + } + remote_code = (uintptr_t)result; + remote_state = remote_code + page_size; + blob = prepare_blob(remote_state, getpid()); + remote_write(pid, remote_code, blob, blob_size); + free(blob); + action = make_install_action(remote_code); + remote_write(pid, remote_state + INSTALL_ACTION_OFFSET, &action, + sizeof(action)); + result = remote_syscall(pid, SYS_rt_sigaction, SIGSYS, + remote_state + INSTALL_ACTION_OFFSET, 0, 8, 0, + GATEWAY_MAGIC); + if (result != 0) { + if (result < 0 && result >= -4095) + errno = (int)-result; + fatal("remote rt_sigaction recursive handler"); + } + result = remote_syscall(pid, SYS_mprotect, remote_code, page_size, + PROT_READ | PROT_EXEC, 0, 0, 0); + if (result != 0) { + if (result < 0 && result >= -4095) + errno = (int)-result; + fatal("remote mprotect recursive handler"); + } +} + +static void record_root_exit(struct bridge_state *bridge, pid_t pid, int status) +{ + if (pid == bridge->root_pid) { + atomic_store(&bridge->root_status, status); + atomic_store(&bridge->root_reaped, true); + } +} + +static pid_t wait_for_tracee(struct bridge_state *bridge, int *status) +{ + for (;;) { + pid_t stopped = waitpid(-1, status, __WALL); + if (stopped < 0) + fatal("waitpid ptrace bridge"); + if (WIFEXITED(*status) || WIFSIGNALED(*status)) { + record_root_exit(bridge, stopped, *status); + continue; + } + if (WIFSTOPPED(*status)) + return stopped; + } +} + +static void release_exec_handler(pid_t tid, uintptr_t flag_address) +{ + if (ptrace(PTRACE_POKEDATA, tid, (void *)flag_address, (void *)1UL) < 0) + fatal("PTRACE_POKEDATA exec release flag"); +} + +static void finish_exec_syscall(struct bridge_state *bridge, pid_t pid) +{ + int status; + pid_t stopped; + + if (ptrace(PTRACE_SYSCALL, pid, NULL, NULL) < 0) + fatal("PTRACE_SYSCALL after recursive exec event"); + stopped = wait_for_tracee(bridge, &status); + if (stopped != pid || !WIFSTOPPED(status) || + WSTOPSIG(status) != (SIGTRAP | 0x80) || + (unsigned int)status >> 16 != 0) + fatal_message("recursive exec did not reach syscall-exit stop"); +} + +static void print_exec_path(pid_t pid, unsigned int count, pid_t former_tid) +{ + char proc_path[64]; + char executable[4096]; + char command_line[4096]; + char descriptor_target[4096]; + int command_line_fd; + ssize_t command_line_length; + ssize_t length; + + snprintf(proc_path, sizeof(proc_path), "/proc/%d/exe", pid); + length = readlink(proc_path, executable, sizeof(executable) - 1); + if (length < 0) { + snprintf(executable, sizeof(executable), "", + strerror(errno)); + } else { + executable[length] = '\0'; + } + printf("bridge: injected exec #%u pid=%d former_tid=%d exe=%s\n", count, + pid, former_tid, executable); + snprintf(proc_path, sizeof(proc_path), "/proc/%d/cmdline", pid); + command_line_fd = open(proc_path, O_RDONLY | O_CLOEXEC); + if (command_line_fd >= 0) { + command_line_length = + read(command_line_fd, command_line, sizeof(command_line) - 1); + close(command_line_fd); + if (command_line_length > 0) { + for (ssize_t index = 0; index < command_line_length; index++) { + if (command_line[index] == '\0') + command_line[index] = ' '; + } + command_line[command_line_length] = '\0'; + printf("bridge: exec argv #%u %s\n", count, command_line); + if (strstr(command_line, "--type=zygote") != NULL) { + snprintf(proc_path, sizeof(proc_path), "/proc/%d/fd/3", pid); + length = readlink(proc_path, descriptor_target, + sizeof(descriptor_target) - 1); + if (length >= 0) { + descriptor_target[length] = '\0'; + printf("bridge: zygote fd 3=%s\n", descriptor_target); + } else { + printf("bridge: zygote fd 3=\n", + strerror(errno)); + } + } + } + } + fflush(stdout); +} + +static void handle_exec_request(struct bridge_state *bridge, + const siginfo_t *request) +{ + const unsigned long options = + PTRACE_O_TRACEEXEC | PTRACE_O_EXITKILL | PTRACE_O_TRACESYSGOOD; + const pid_t requesting_tid = request->si_pid; + const uintptr_t flag_address = + (uintptr_t)request->si_value.sival_ptr; + pid_t tracee; + bool saw_exec_entry = false; + int status; + + if (requesting_tid <= 0 || flag_address == 0) + fatal_message("bridge received malformed exec request"); + if (ptrace(PTRACE_SEIZE, requesting_tid, NULL, (void *)options) < 0) + fatal("PTRACE_SEIZE exec requester"); + if (ptrace(PTRACE_INTERRUPT, requesting_tid, NULL, NULL) < 0) + fatal("PTRACE_INTERRUPT exec requester"); + tracee = wait_for_tracee(bridge, &status); + if (tracee != requesting_tid || !WIFSTOPPED(status)) + fatal_message("unexpected initial ptrace bridge stop"); + release_exec_handler(tracee, flag_address); + if (ptrace(PTRACE_SYSCALL, tracee, NULL, NULL) < 0) + fatal("PTRACE_SYSCALL release exec requester"); + + for (;;) { + unsigned int event; + pid_t stopped = wait_for_tracee(bridge, &status); + struct user_regs_struct regs; + + event = (unsigned int)status >> 16; + if (event == PTRACE_EVENT_EXEC) { + unsigned long former_tid = 0; + unsigned int count; + + if (ptrace(PTRACE_GETEVENTMSG, stopped, NULL, &former_tid) < 0) + fatal("PTRACE_GETEVENTMSG recursive exec"); + finish_exec_syscall(bridge, stopped); + inject_handler(stopped); + count = atomic_fetch_add(&bridge->exec_count, 1) + 1; + print_exec_path(stopped, count, (pid_t)former_tid); + if (ptrace(PTRACE_DETACH, stopped, NULL, NULL) < 0) + fatal("PTRACE_DETACH recursive exec"); + return; + } + if (WSTOPSIG(status) == (SIGTRAP | 0x80)) { + long result; + long number; + + if (ptrace_get_regs(stopped, ®s) < 0) + fatal("PTRACE_GETREGSET recursive syscall stop"); + number = (long)regs.orig_rax; + result = (long)regs.rax; + if (number == SYS_execve || number == SYS_execveat) { + if (result == -ENOSYS) { + saw_exec_entry = true; + } else if (saw_exec_entry) { + atomic_fetch_add(&bridge->failed_exec_count, 1); + if (ptrace(PTRACE_DETACH, stopped, NULL, NULL) < 0) + fatal("PTRACE_DETACH failed exec"); + return; + } + } + if (ptrace(PTRACE_SYSCALL, stopped, NULL, NULL) < 0) + fatal("PTRACE_SYSCALL recursive bridge loop"); + continue; + } + if (ptrace(PTRACE_SYSCALL, stopped, NULL, + (void *)(uintptr_t)WSTOPSIG(status)) < 0) + fatal("PTRACE_SYSCALL forward bridge signal"); + } +} + +static void *bridge_thread_main(void *opaque) +{ + struct bridge_state *bridge = opaque; + sigset_t signal_set; + + sigemptyset(&signal_set); + sigaddset(&signal_set, BRIDGE_SIGNAL); + for (;;) { + siginfo_t request; + int signal_number = sigwaitinfo(&signal_set, &request); + + if (signal_number < 0) { + if (errno == EINTR) + continue; + fatal("sigwaitinfo ptrace bridge"); + } + if (atomic_load(&bridge->stopping) && request.si_code != SI_QUEUE) + return NULL; + if (request.si_code != SI_QUEUE) + continue; + handle_exec_request(bridge, &request); + } +} + +static void child_main(char **command) +{ + if (setpgid(0, 0) != 0) + fatal("setpgid child"); + if (prctl(PR_SET_PDEATHSIG, SIGKILL) != 0) + fatal("PR_SET_PDEATHSIG"); + install_initial_handler(); + install_filter(); + execvpe(command[0], command, environ); + fatal("execvpe target"); +} + +int main(int argc, char **argv) +{ + struct bridge_state bridge; + pthread_t bridge_thread; + sigset_t signal_set; + pid_t child; + int pidfd; + int status; + int exit_code; + + if (argc < 2) { + fprintf(stderr, "usage: %s command [arg ...]\n", argv[0]); + return EXIT_FAILURE; + } + sigemptyset(&signal_set); + sigaddset(&signal_set, BRIDGE_SIGNAL); + if (pthread_sigmask(SIG_BLOCK, &signal_set, NULL) != 0) + fatal("pthread_sigmask bridge signal"); + child = fork(); + if (child < 0) + fatal("fork recursive target"); + if (child == 0) + child_main(&argv[1]); + + cleanup_process_group = child; + if (atexit(cleanup_children) != 0) + fatal_message("atexit cleanup registration failed"); + memset(&bridge, 0, sizeof(bridge)); + bridge.root_pid = child; + atomic_init(&bridge.stopping, false); + atomic_init(&bridge.root_reaped, false); + atomic_init(&bridge.root_status, 0); + atomic_init(&bridge.exec_count, 0); + atomic_init(&bridge.failed_exec_count, 0); + if (pthread_create(&bridge_thread, NULL, bridge_thread_main, &bridge) != 0) + fatal("pthread_create ptrace bridge"); + pidfd = (int)syscall(SYS_pidfd_open, child, 0); + if (pidfd < 0) + fatal("pidfd_open root target"); + for (;;) { + struct pollfd descriptor = {.fd = pidfd, .events = POLLIN}; + int poll_result = poll(&descriptor, 1, -1); + if (poll_result < 0 && errno == EINTR) + continue; + if (poll_result < 0) + fatal("poll root pidfd"); + break; + } + close(pidfd); + atomic_store(&bridge.stopping, true); + if (pthread_kill(bridge_thread, BRIDGE_SIGNAL) != 0) + fatal("pthread_kill bridge shutdown"); + if (pthread_join(bridge_thread, NULL) != 0) + fatal("pthread_join ptrace bridge"); + if (atomic_load(&bridge.root_reaped)) { + status = atomic_load(&bridge.root_status); + } else if (waitpid(child, &status, 0) < 0) { + fatal("waitpid root target"); + } + cleanup_process_group = -1; + printf("bridge: summary injected_execs=%u failed_execs=%u\n", + atomic_load(&bridge.exec_count), + atomic_load(&bridge.failed_exec_count)); + if (WIFEXITED(status)) { + exit_code = WEXITSTATUS(status); + printf("bridge: target exit status %d\n", exit_code); + return exit_code; + } + if (WIFSIGNALED(status)) + fprintf(stderr, "bridge: target died from signal %d\n", + WTERMSIG(status)); + else + fprintf(stderr, "bridge: unexpected target status %#x\n", status); + return EXIT_FAILURE; +} diff --git a/research/ptrace-exec-prototype/run_esbuild_experiment.sh b/research/ptrace-exec-prototype/run_esbuild_experiment.sh new file mode 100755 index 000000000..2fb30cc84 --- /dev/null +++ b/research/ptrace-exec-prototype/run_esbuild_experiment.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -eu + +case "$(uname -m)" in + aarch64 | arm64) ;; + *) + echo "this bounded experiment requires native AArch64 Linux" >&2 + exit 1 + ;; +esac + +script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +experiment_dir=$(mktemp -d) +trap 'rm -rf -- "$experiment_dir"' EXIT INT TERM + +esbuild_version=${ESBUILD_VERSION:-0.28.1} +package_url="https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-${esbuild_version}.tgz" + +cc -O2 -g -Wall -Wextra -Werror -std=gnu11 \ + -o "$experiment_dir/esbuild-injector" "$script_dir/esbuild_injector.c" +curl -L --fail --silent "$package_url" -o "$experiment_dir/esbuild.tgz" +mkdir "$experiment_dir/package" +tar -xzf "$experiment_dir/esbuild.tgz" -C "$experiment_dir/package" \ + --strip-components=1 +cp "$script_dir/esbuild_input.js" "$experiment_dir/input.js" + +cd "$experiment_dir" +./esbuild-injector "$(realpath package/bin/esbuild)" --version +./esbuild-injector "$(realpath package/bin/esbuild)" input.js --bundle \ + --platform=node --outfile=out.js +test -s out.js +grep -q 'answer' out.js + +echo "PASS: esbuild ${esbuild_version} version and bundle operations completed" diff --git a/research/ptrace-exec-prototype/target.c b/research/ptrace-exec-prototype/target.c new file mode 100644 index 000000000..a5482aa8e --- /dev/null +++ b/research/ptrace-exec-prototype/target.c @@ -0,0 +1,49 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include + +#define TRAPPED_RESULT 0x51515151L + +static int tracer_pid(void) +{ + char line[256]; + FILE *status = fopen("/proc/self/status", "r"); + if (status == NULL) + return -1; + + while (fgets(line, sizeof(line), status) != NULL) { + int tracer; + if (sscanf(line, "TracerPid:\t%d", &tracer) == 1) { + fclose(status); + return tracer; + } + } + fclose(status); + return -1; +} + +int main(void) +{ + int tracer = tracer_pid(); + long result = syscall(SYS_getpid); + + printf("target: TracerPid=%d before trapped getpid\n", tracer); + printf("target: trapped getpid returned %#lx (expected %#lx)\n", result, + TRAPPED_RESULT); + + if (tracer != 0) { + fputs("FAIL: target was still ptraced\n", stderr); + return EXIT_FAILURE; + } + if (result != TRAPPED_RESULT) { + fputs("FAIL: injected SIGSYS handler did not emulate getpid\n", stderr); + return EXIT_FAILURE; + } + + puts("PASS: post-exec handler ran entirely in-process after detach"); + return EXIT_SUCCESS; +} diff --git a/research/rust-injected-runtime/.gitignore b/research/rust-injected-runtime/.gitignore new file mode 100644 index 000000000..ea8c4bf7f --- /dev/null +++ b/research/rust-injected-runtime/.gitignore @@ -0,0 +1 @@ +/target diff --git a/research/rust-injected-runtime/Cargo.lock b/research/rust-injected-runtime/Cargo.lock new file mode 100644 index 000000000..26e4c756b --- /dev/null +++ b/research/rust-injected-runtime/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "fspy-rust-injected-runtime" +version = "0.0.0" diff --git a/research/rust-injected-runtime/Cargo.toml b/research/rust-injected-runtime/Cargo.toml new file mode 100644 index 000000000..6640a95ba --- /dev/null +++ b/research/rust-injected-runtime/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "fspy-rust-injected-runtime" +version = "0.0.0" +edition = "2024" +publish = false + +[lib] +crate-type = ["staticlib"] + +[profile.release] +codegen-units = 1 +lto = "fat" +opt-level = "z" +panic = "abort" + +# Keep this research artifact independent from the repository workspace. Its +# linker contract is deliberately much narrower than an ordinary Rust crate. +[workspace] diff --git a/research/rust-injected-runtime/Makefile b/research/rust-injected-runtime/Makefile new file mode 100644 index 000000000..7a934a742 --- /dev/null +++ b/research/rust-injected-runtime/Makefile @@ -0,0 +1,57 @@ +SHELL := /bin/bash + +HOST := $(shell rustc -vV | sed -n 's/^host: //p') +SYSROOT := $(shell rustc --print sysroot) +LLVM_BIN := $(SYSROOT)/lib/rustlib/$(HOST)/bin +RUST_LLD := $(LLVM_BIN)/rust-lld +COMMON_RUSTFLAGS := -C relocation-model=pic -C no-redzone=yes -C force-unwind-tables=no -C overflow-checks=no + +.PHONY: all check smoke-x86_64 clean + +all: target/blob-x86_64.bin target/blob-aarch64.bin + +check: all + @bash ./verify.sh target/blob-x86_64.elf target/blob-x86_64.bin $(LLVM_BIN) + @bash ./verify.sh target/blob-aarch64.elf target/blob-aarch64.bin $(LLVM_BIN) + @if [[ "$$(uname -s)-$$(uname -m)" == Linux-x86_64 ]]; then \ + $(MAKE) --no-print-directory smoke-x86_64; \ + fi + +smoke-x86_64: target/smoke-x86_64 target/blob-x86_64.bin + @symbols="$$($(LLVM_BIN)/llvm-nm --numeric-sort target/blob-x86_64.elf)"; \ + state_ptr=$$(awk '$$3 == "FSPY_STATE_PTR" { print "0x" $$1 }' <<< "$$symbols"); \ + handler=$$(awk '$$3 == "fspy_sigsys_handler" { print "0x" $$1 }' <<< "$$symbols"); \ + restorer=$$(awk '$$3 == "fspy_rt_sigreturn" { print "0x" $$1 }' <<< "$$symbols"); \ + allocator=$$(awk '$$3 == "fspy_alloc" { print "0x" $$1 }' <<< "$$symbols"); \ + raw_syscall=$$(awk '$$3 == "fspy_raw_syscall6" { print "0x" $$1 }' <<< "$$symbols"); \ + target/smoke-x86_64 target/blob-x86_64.bin \ + "$$state_ptr" "$$handler" "$$restorer" "$$allocator" "$$raw_syscall" + +target/smoke-x86_64: smoke_x86_64.c + @mkdir -p target + $(CC) -O2 -Wall -Wextra -Werror -std=gnu11 -o $@ $< + +target/x86_64/x86_64-unknown-linux-musl/release/libfspy_rust_injected_runtime.a: Cargo.toml src/lib.rs + RUSTFLAGS='$(COMMON_RUSTFLAGS)' cargo build --release \ + --target x86_64-unknown-linux-musl --target-dir target/x86_64 + +target/aarch64/aarch64-unknown-linux-musl/release/libfspy_rust_injected_runtime.a: Cargo.toml src/lib.rs + RUSTFLAGS='$(COMMON_RUSTFLAGS) -C target-feature=-outline-atomics' cargo build --release \ + --target aarch64-unknown-linux-musl --target-dir target/aarch64 + +target/blob-x86_64.elf: target/x86_64/x86_64-unknown-linux-musl/release/libfspy_rust_injected_runtime.a blob.ld + $(RUST_LLD) -flavor gnu -m elf_x86_64 -nostdlib --gc-sections \ + --no-undefined --build-id=none -T blob.ld -o $@ $< + +target/blob-aarch64.elf: target/aarch64/aarch64-unknown-linux-musl/release/libfspy_rust_injected_runtime.a blob.ld + $(RUST_LLD) -flavor gnu -m aarch64linux -nostdlib --gc-sections \ + --no-undefined --build-id=none -T blob.ld -o $@ $< + +target/blob-x86_64.bin: target/blob-x86_64.elf + $(LLVM_BIN)/llvm-objcopy --only-section=.fspy_blob -O binary $< $@ + +target/blob-aarch64.bin: target/blob-aarch64.elf + $(LLVM_BIN)/llvm-objcopy --only-section=.fspy_blob -O binary $< $@ + +clean: + rm -rf target diff --git a/research/rust-injected-runtime/README.md b/research/rust-injected-runtime/README.md new file mode 100644 index 000000000..83bc328ff --- /dev/null +++ b/research/rust-injected-runtime/README.md @@ -0,0 +1,20 @@ +# Freestanding Rust handler blob + +This research artifact compiles a `#![no_std]` Rust `SIGSYS` handler, raw syscall gateway, `rt_sigreturn` restorer, and fixed-capacity lock-free allocator into relocation-free x86-64 and AArch64 blobs. + +Run: + +```sh +make check +``` + +The check rejects runtime relocations, undefined symbols, writable data, dynamic linking, GOT, PLT, TLS, and initialization sections. On native Linux x86-64 it also maps the blob RX, installs the Rust handler and restorer with `rt_sigaction`, triggers a seccomp `SIGSYS`, and exercises the allocator and raw syscall gateway. + +Generated files stay under `target/`. The source artifacts are: + +- [`src/lib.rs`](src/lib.rs): state ABI, handler probe, allocator, syscall wrapper, and restorer +- [`blob.ld`](blob.ld): extraction layout and forbidden-section assertions +- [`verify.sh`](verify.sh): post-link artifact audit +- [`smoke_x86_64.c`](smoke_x86_64.c): native execution harness + +See the [full injected-runtime design](../../docs/fspy-rust-injected-runtime.md) for the production architecture and remaining work. diff --git a/research/rust-injected-runtime/blob.ld b/research/rust-injected-runtime/blob.ld new file mode 100644 index 000000000..4ea290dd8 --- /dev/null +++ b/research/rust-injected-runtime/blob.ld @@ -0,0 +1,47 @@ +ENTRY(fspy_sigsys_handler) + +SECTIONS +{ + . = 0; + .fspy_blob : ALIGN(16) + { + HIDDEN(FSPY_BLOB_START = .); + KEEP(*(.text.fspy_entry)) + KEEP(*(.text.fspy_restorer)) + KEEP(*(.text.fspy_alloc)) + KEEP(*(.text.fspy_raw_syscall6)) + *(.text .text.*) + *(.rodata .rodata.*) + . = ALIGN(8); + HIDDEN(FSPY_STATE_PTR = .); + QUAD(0); + HIDDEN(FSPY_BLOB_END = .); + } + + .forbidden_writable : + { + *(.data .data.*) + *(.bss .bss.* COMMON) + *(.data.rel.ro .data.rel.ro.*) + *(.got .got.*) + *(.plt .plt.*) + *(.tdata .tdata.*) + *(.tbss .tbss.*) + *(.init_array .init_array.*) + *(.fini_array .fini_array.*) + } + + /DISCARD/ : + { + *(.comment) + *(.debug_frame .debug_frame.*) + *(.eh_frame .eh_frame.*) + *(.note .note.*) + *(.gnu.attributes) + } +} + +ASSERT(SIZEOF(.forbidden_writable) == 0, + "injected runtime contains writable, GOT, PLT, TLS, or init data") +ASSERT((FSPY_BLOB_END - FSPY_BLOB_START) < 0x100000, + "AArch64 ADR cannot reach FSPY_STATE_PTR") diff --git a/research/rust-injected-runtime/smoke_x86_64.c b/research/rust-injected-runtime/smoke_x86_64.c new file mode 100644 index 000000000..dea3c5c09 --- /dev/null +++ b/research/rust-injected-runtime/smoke_x86_64.c @@ -0,0 +1,186 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(__x86_64__) +#error "this native smoke harness currently validates x86-64 only" +#endif + +#ifndef SA_RESTORER +#define SA_RESTORER 0x04000000 +#endif + +enum { + STATE_MAPPING_LEN = 17 * 4096, + STATE_TRAP_COUNT_OFFSET = 32, + STATE_LAST_SYSCALL_OFFSET = 40, + STATE_ARENA_NEXT_OFFSET = 48, + STATE_ARENA_OFFSET = 4096, + PROBE_RESULT = 0x51515151, +}; + +struct kernel_sigaction { + void (*handler)(int, siginfo_t *, void *); + unsigned long flags; + void (*restorer)(void); + unsigned long mask; +}; + +typedef void *(*alloc_fn)(size_t, size_t); +typedef long (*syscall6_fn)(long, long, long, long, long, long, long); + +static void fatal(const char *message) { + perror(message); + exit(1); +} + +static uintptr_t parse_offset(const char *text) { + errno = 0; + char *end = NULL; + unsigned long long value = strtoull(text, &end, 0); + if (errno != 0 || end == text || *end != '\0') { + fprintf(stderr, "invalid symbol offset: %s\n", text); + exit(1); + } + return (uintptr_t)value; +} + +static void *read_blob(const char *path, size_t *size_out) { + FILE *file = fopen(path, "rb"); + if (file == NULL) + fatal("fopen blob"); + if (fseek(file, 0, SEEK_END) != 0) + fatal("fseek blob"); + long length = ftell(file); + if (length <= 0) + fatal("ftell blob"); + rewind(file); + + void *bytes = malloc((size_t)length); + if (bytes == NULL) + fatal("malloc blob"); + if (fread(bytes, 1, (size_t)length, file) != (size_t)length) + fatal("fread blob"); + if (fclose(file) != 0) + fatal("fclose blob"); + *size_out = (size_t)length; + return bytes; +} + +static void install_filter(void) { + struct sock_filter instructions[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_getpid, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_TRAP | 0x4653), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + }; + struct sock_fprog program = { + .len = (unsigned short)(sizeof(instructions) / sizeof(instructions[0])), + .filter = instructions, + }; + + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) + fatal("PR_SET_NO_NEW_PRIVS"); + if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program) != 0) + fatal("PR_SET_SECCOMP"); +} + +int main(int argc, char **argv) { + if (argc != 7) { + fprintf(stderr, + "usage: %s BLOB STATE_PTR HANDLER RESTORER ALLOC RAW_SYSCALL\n", + argv[0]); + return 2; + } + + const uintptr_t state_slot_offset = parse_offset(argv[2]); + const uintptr_t handler_offset = parse_offset(argv[3]); + const uintptr_t restorer_offset = parse_offset(argv[4]); + const uintptr_t alloc_offset = parse_offset(argv[5]); + const uintptr_t raw_syscall_offset = parse_offset(argv[6]); + + size_t blob_size = 0; + void *blob = read_blob(argv[1], &blob_size); + if (blob_size < sizeof(uintptr_t) || + state_slot_offset > blob_size - sizeof(uintptr_t) || + handler_offset >= blob_size || restorer_offset >= blob_size || + alloc_offset >= blob_size || raw_syscall_offset >= blob_size) { + fprintf(stderr, "symbol offset lies outside the raw blob\n"); + return 1; + } + long page_size = sysconf(_SC_PAGESIZE); + if (page_size <= 0) + fatal("sysconf page size"); + size_t code_mapping_len = + (blob_size + (size_t)page_size - 1) & ~((size_t)page_size - 1); + + unsigned char *state = mmap(NULL, STATE_MAPPING_LEN, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (state == MAP_FAILED) + fatal("mmap state"); + unsigned char *code = mmap(NULL, code_mapping_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (code == MAP_FAILED) + fatal("mmap code"); + + memcpy(code, blob, blob_size); + free(blob); + memcpy(code + state_slot_offset, &state, sizeof(state)); + if (mprotect(code, code_mapping_len, PROT_READ | PROT_EXEC) != 0) + fatal("mprotect code RX"); + + alloc_fn allocate = (alloc_fn)(code + alloc_offset); + void *first = allocate(32, 16); + void *second = allocate(17, 64); + if (first != state + STATE_ARENA_OFFSET || + second != state + STATE_ARENA_OFFSET + 64 || + *(uintptr_t *)(state + STATE_ARENA_NEXT_OFFSET) != 81) { + fprintf(stderr, "fixed allocator returned unexpected ranges\n"); + return 1; + } + + syscall6_fn raw_syscall = (syscall6_fn)(code + raw_syscall_offset); + if (raw_syscall(SYS_getppid, 0, 0, 0, 0, 0, 0) <= 0) { + fprintf(stderr, "raw Rust syscall gateway failed\n"); + return 1; + } + + struct kernel_sigaction action = { + .handler = (void (*)(int, siginfo_t *, void *))(code + handler_offset), + .flags = SA_SIGINFO | SA_NODEFER | SA_RESTORER, + .restorer = (void (*)(void))(code + restorer_offset), + .mask = 0, + }; + if (syscall(SYS_rt_sigaction, SIGSYS, &action, NULL, + sizeof(action.mask)) != 0) + fatal("rt_sigaction"); + + install_filter(); + long result = syscall(SYS_getpid); + if (result != PROBE_RESULT) { + fprintf(stderr, "trapped getpid returned %#lx, expected %#x\n", result, + PROBE_RESULT); + return 1; + } + if (*(uintptr_t *)(state + STATE_TRAP_COUNT_OFFSET) != 1 || + *(uintptr_t *)(state + STATE_LAST_SYSCALL_OFFSET) != SYS_getpid) { + fprintf(stderr, "Rust handler did not update its state ABI\n"); + return 1; + } + + puts("PASS: Rust SIGSYS handler, restorer, syscall gateway, and allocator"); + return 0; +} diff --git a/research/rust-injected-runtime/src/lib.rs b/research/rust-injected-runtime/src/lib.rs new file mode 100644 index 000000000..6064cfe5f --- /dev/null +++ b/research/rust-injected-runtime/src/lib.rs @@ -0,0 +1,278 @@ +#![no_std] + +use core::{ + alloc::{GlobalAlloc, Layout}, + arch::{asm, global_asm}, + cell::UnsafeCell, + ptr::{null_mut, read_unaligned, write_unaligned}, + sync::atomic::{AtomicUsize, Ordering::Relaxed}, +}; + +const ABI_MAGIC: u64 = 0x4653_5059_5254_3031; // "FSPYRT01" +const ABI_VERSION: u32 = 1; +const ARENA_LEN: usize = 64 * 1024; +const MAX_SUPPORTED_ALIGN: usize = 4096; +const SYS_SECCOMP: i32 = 1; +const PROBE_RESULT: usize = 0x5151_5151; + +/// The supervisor owns this fixed-size RW mapping. The injected RX blob only +/// contains a patched pointer to it, so the blob has no writable sections. +#[repr(C, align(4096))] +pub struct FixedArena { + bytes: UnsafeCell<[u8; ARENA_LEN]>, +} + +#[repr(C, align(64))] +pub struct RuntimeState { + pub abi_magic: u64, + pub abi_version: u32, + pub state_size: u32, + pub supervisor_pid: u32, + pub bridge_signal: u32, + pub gateway_magic: u64, + pub trap_count: AtomicUsize, + pub last_syscall: AtomicUsize, + arena_next: AtomicUsize, + arena: FixedArena, +} + +impl RuntimeState { + /// Creates the bytes that the supervisor copies into the separate RW + /// mapping before it patches `FSPY_STATE_PTR` in the blob image. + pub const fn new(supervisor_pid: u32, bridge_signal: u32, gateway_magic: u64) -> Self { + Self { + abi_magic: ABI_MAGIC, + abi_version: ABI_VERSION, + state_size: size_of::() as u32, + supervisor_pid, + bridge_signal, + gateway_magic, + trap_count: AtomicUsize::new(0), + last_syscall: AtomicUsize::new(0), + arena_next: AtomicUsize::new(0), + arena: FixedArena { bytes: UnsafeCell::new([0; ARENA_LEN]) }, + } + } +} + +struct FixedBump; + +#[global_allocator] +static ALLOCATOR: FixedBump = FixedBump; + +/// A monotonic, fixed-capacity allocator for non-fast-path runtime setup. +/// +/// Allocation uses only pointer arithmetic and a native `AtomicUsize` CAS. +/// Deallocation is intentionally a no-op. This makes it lock-free, +/// syscall-free, and immune to allocator lock reentrancy, but memory is not +/// reclaimed until the logical exec replaces the runtime. +unsafe impl GlobalAlloc for FixedBump { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let align = layout.align(); + if align > MAX_SUPPORTED_ALIGN { + return null_mut(); + } + + let state = runtime_state(); + let mut current = state.arena_next.load(Relaxed); + loop { + let Some(aligned) = current.checked_add(align - 1).map(|value| value & !(align - 1)) + else { + return null_mut(); + }; + let Some(end) = aligned.checked_add(layout.size()) else { + return null_mut(); + }; + if end > ARENA_LEN { + return null_mut(); + } + + match state.arena_next.compare_exchange_weak(current, end, Relaxed, Relaxed) { + Ok(_) => { + // SAFETY: the successful CAS reserves the disjoint range + // [aligned, end), and FixedArena has 4096-byte alignment. + return unsafe { state.arena.bytes.get().cast::().add(aligned) }; + } + Err(observed) => current = observed, + } + } + } + + unsafe fn dealloc(&self, _ptr: *mut u8, _layout: Layout) {} +} + +/// Exported only so the research build retains and audits the allocator. +/// Production handler code must use preallocated stack/ring/scratch records +/// instead of allocating in the SIGSYS fast path. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fspy_alloc(size: usize, align: usize) -> *mut u8 { + let Ok(layout) = Layout::from_size_align(size, align) else { + return null_mut(); + }; + // SAFETY: `layout` was validated above. + unsafe { ALLOCATOR.alloc(layout) } +} + +#[inline(always)] +fn runtime_state() -> &'static RuntimeState { + let slot: *const *mut RuntimeState; + + #[cfg(target_arch = "x86_64")] + // SAFETY: the linker script defines this local PC-relative symbol inside + // the copied blob. The supervisor patches its pointer-sized contents. + unsafe { + asm!( + "lea {slot}, [rip + FSPY_STATE_PTR]", + slot = out(reg) slot, + options(nostack, preserves_flags), + ); + } + + #[cfg(target_arch = "aarch64")] + // SAFETY: the blob is asserted to remain inside ADR's +/-1 MiB range. + unsafe { + asm!( + "adr {slot}, FSPY_STATE_PTR", + slot = out(reg) slot, + options(nostack, preserves_flags), + ); + } + + // SAFETY: injection is not allowed to install the handler until the slot + // points at a fully initialized, suitably aligned RuntimeState mapping. + unsafe { &**slot } +} + +/// Minimal ABI probe for the final artifact. A production version dispatches +/// the trapped syscall and writes its raw result at the same ucontext offset. +#[unsafe(no_mangle)] +#[unsafe(link_section = ".text.fspy_entry")] +pub unsafe extern "C" fn fspy_sigsys_handler(_signal: i32, siginfo: *const u8, ucontext: *mut u8) { + if siginfo.is_null() || ucontext.is_null() { + return; + } + + // Linux's SIGSYS siginfo layout has si_code at byte 8 and si_syscall at + // byte 24 on both supported 64-bit architectures. Native C offsetof tests + // must remain an acceptance gate for these constants. + let code = unsafe { read_unaligned(siginfo.add(8).cast::()) }; + if code != SYS_SECCOMP { + return; + } + let syscall = unsafe { read_unaligned(siginfo.add(24).cast::()) }; + + let state = runtime_state(); + state.last_syscall.store(syscall as usize, Relaxed); + state.trap_count.fetch_add(1, Relaxed); + + #[cfg(target_arch = "x86_64")] + const RETURN_REGISTER_OFFSET: usize = 144; // ucontext.uc_mcontext.rax + #[cfg(target_arch = "aarch64")] + const RETURN_REGISTER_OFFSET: usize = 184; // ucontext.uc_mcontext.regs[0] + + // SAFETY: the kernel supplied this ucontext to an SA_SIGINFO handler, and + // the architecture-specific offset is validated against Linux headers. + unsafe { + write_unaligned(ucontext.add(RETURN_REGISTER_OFFSET).cast::(), PROBE_RESULT); + } +} + +/// Raw six-argument Linux syscall gateway. Do not add `nomem`: the kernel may +/// read or write memory named by the arguments. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn fspy_raw_syscall6( + number: usize, + arg0: usize, + arg1: usize, + arg2: usize, + arg3: usize, + arg4: usize, + arg5: usize, +) -> isize { + #[cfg(target_arch = "x86_64")] + { + let result: isize; + // SAFETY: the caller owns the raw Linux syscall contract. + unsafe { + asm!( + "syscall", + inlateout("rax") number as isize => result, + in("rdi") arg0, + in("rsi") arg1, + in("rdx") arg2, + in("r10") arg3, + in("r8") arg4, + in("r9") arg5, + lateout("rcx") _, + lateout("r11") _, + options(nostack), + ); + } + result + } + + #[cfg(target_arch = "aarch64")] + { + let result: isize; + // SAFETY: the caller owns the raw Linux syscall contract. + unsafe { + asm!( + "svc #0", + in("x8") number, + inlateout("x0") arg0 as isize => result, + in("x1") arg1, + in("x2") arg2, + in("x3") arg3, + in("x4") arg4, + in("x5") arg5, + options(nostack), + ); + } + result + } +} + +#[cfg(target_arch = "x86_64")] +global_asm!( + ".pushsection .text.fspy_restorer,\"ax\",@progbits", + ".global fspy_rt_sigreturn", + ".type fspy_rt_sigreturn,@function", + "fspy_rt_sigreturn:", + "mov rax, 15", + "syscall", + "ud2", + ".size fspy_rt_sigreturn, .-fspy_rt_sigreturn", + ".popsection", +); + +#[cfg(target_arch = "aarch64")] +global_asm!( + ".pushsection .text.fspy_restorer,\"ax\",@progbits", + ".global fspy_rt_sigreturn", + ".type fspy_rt_sigreturn,%function", + "fspy_rt_sigreturn:", + "mov x8, #139", + "svc #0", + "brk #0", + ".size fspy_rt_sigreturn, .-fspy_rt_sigreturn", + ".popsection", +); + +#[panic_handler] +fn panic(_info: &core::panic::PanicInfo<'_>) -> ! { + #[cfg(target_arch = "x86_64")] + const EXIT_GROUP: usize = 231; + #[cfg(target_arch = "aarch64")] + const EXIT_GROUP: usize = 94; + + // SAFETY: exit_group does not return and all unused arguments are zero. + unsafe { + let _ = fspy_raw_syscall6(EXIT_GROUP, 127, 0, 0, 0, 0, 0); + } + loop { + core::hint::spin_loop(); + } +} + +#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] +compile_error!("the injected runtime only supports x86-64 and AArch64 Linux"); diff --git a/research/rust-injected-runtime/verify.sh b/research/rust-injected-runtime/verify.sh new file mode 100644 index 000000000..454e11f56 --- /dev/null +++ b/research/rust-injected-runtime/verify.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail + +elf=$1 +raw=$2 +llvm_bin=$3 + +if "$llvm_bin/llvm-objdump" --reloc "$elf" | grep -q 'RELOCATION RECORDS FOR'; then + echo "unexpected relocation in $elf" >&2 + exit 1 +fi + +if [[ -n $("$llvm_bin/llvm-nm" --undefined-only "$elf") ]]; then + echo "unexpected undefined symbol in $elf" >&2 + "$llvm_bin/llvm-nm" --undefined-only "$elf" >&2 + exit 1 +fi + +if "$llvm_bin/llvm-readobj" --sections "$elf" \ + | grep -Eq 'Name: \.(data|bss|got|plt|tdata|tbss|dynamic|dynsym|init_array|fini_array)'; then + echo "unexpected runtime section in $elf" >&2 + exit 1 +fi + +state_offset=$("$llvm_bin/llvm-nm" --numeric-sort "$elf" \ + | awk '$3 == "FSPY_STATE_PTR" { print "0x" $1 }') +if [[ -z $state_offset ]]; then + echo "missing FSPY_STATE_PTR in $elf" >&2 + exit 1 +fi + +size=$(wc -c < "$raw" | tr -d ' ') +printf '%s: %s bytes, state pointer patch offset %s\n' "$raw" "$size" "$state_offset" diff --git a/research/sigsys-prototype/README.md b/research/sigsys-prototype/README.md new file mode 100644 index 000000000..9f7c40d52 --- /dev/null +++ b/research/sigsys-prototype/README.md @@ -0,0 +1,311 @@ +# Linux `SECCOMP_RET_TRAP` prototype results + +Research date: 2026-08-02 + +This is an isolated feasibility harness. It does not modify the existing fspy +implementation and is not production-ready. + +## Result in one paragraph + +The same-process `SECCOMP_RET_TRAP` mechanism is viable on the tested Linux +kernel, including direct assembly syscalls, filesystem argument rewriting, +trusted syscall reissue, nested traps, and concurrent calls from four threads. +A static-musl build passed. The median minimal trap cost was about 0.566 us for +register emulation and 0.714 us for a trusted syscall reissue. A comparable +cross-process seccomp user-notification round trip was about 13.9 us, roughly +20-25 times slower than the in-process paths. Exec and signal semantics carry +most of the implementation risk. In particular, fspy must keep +kernel `SIGSYS` unblocked and non-ignored while virtualizing the target's view, +must decide how to coexist with per-thread alternate stacks, and must replace +kernel exec with a substantially hardened ELF loader. A corrected `AT_RANDOM` +implementation was enough to run and bundle with the current static Go esbuild +0.28.1 under the prototype filter. + +## Environment + +- Host: macOS 27.0 ARM64, Lima 2.2.0 with the Virtualization.framework driver. +- Guest: Ubuntu 24.04.4 LTS ARM64, 4 vCPUs, 6 GiB memory. +- Kernel: `6.8.0-134-generic #134-Ubuntu SMP PREEMPT_DYNAMIC`. +- Toolchain: GCC 13.3.0, glibc 2.39, musl 1.2.4, Python 3.12.3. +- Seccomp actions: `kill_process kill_thread trap errno user_notif trace log allow`. +- Benchmark affinity: single-thread trap and preload samples used CPU 0; + user notification used CPU 0 for the tracee and CPU 1 for the supervisor. +- This is a VM microbenchmark. Absolute results should be rerun on native + x86-64 and ARM64 CI machines; the relative process-boundary cost is clear. + +## Artifacts + +- `trap_bench.c`: direct syscall, filesystem rewrite, signal virtualization, + nested trap, multithreading, alternate-stack, and timing probe. +- `reexec_bootstrap.c`: real kernel exec into a static-musl second stage, then + trusted raw handler reinstallation under the inherited filter. +- `unotify_bench.c`: forked seccomp user-notification emulation and `CONTINUE` + timing probe. +- `preload_open_bench.c` and `preload_open_interposer.c`: minimal LD_PRELOAD + dispatch lower bound. +- `trap_preload.c`: retained native handler DSO used to stress a pure userland + handoff into esbuild. It traps the filesystem syscall family used by current + fspy plus signal-mask/action changes and reports counts at `exit_group`. +- `libreflect_runner.c`: general argv-preserving runner for libreflect's pure + `reflect_execve` path. +- `ulexecve_at_random_fix.py`: narrow experimental correction for the reference + Anvil loader's invalid `AT_RANDOM` pointer. +- `esbuild_input.js` and `esbuild_value.js`: two-file bundle fixture. + +## Reproduction commands + +Run these inside the Linux guest from this directory (or copy the files to a +guest-local temporary directory first): + +```bash +gcc -O2 -Wall -Wextra -Werror -pthread trap_bench.c -o trap_bench +taskset -c 0 ./trap_bench + +gcc -O2 -Wall -Wextra -Werror -pthread -static trap_bench.c \ + -o trap_bench_glibc_static +taskset -c 0 ./trap_bench_glibc_static + +musl-gcc -O2 -Wall -Wextra -Werror -pthread -static \ + -idirafter /usr/include -idirafter /usr/include/aarch64-linux-gnu \ + trap_bench.c -o trap_bench_musl_static +taskset -c 0 ./trap_bench_musl_static + +musl-gcc -O2 -Wall -Wextra -Werror -static \ + -idirafter /usr/include -idirafter /usr/include/aarch64-linux-gnu \ + reexec_bootstrap.c -o reexec_bootstrap_musl +./reexec_bootstrap_musl + +gcc -O2 -Wall -Wextra -Werror unotify_bench.c -o unotify_bench +./unotify_bench + +gcc -O2 -Wall -Wextra -Werror -fno-builtin \ + preload_open_bench.c -o preload_open_bench +gcc -O2 -Wall -Wextra -Werror -shared -fPIC \ + preload_open_interposer.c -ldl -o libopen_interposer.so +taskset -c 0 ./preload_open_bench +taskset -c 0 env LD_PRELOAD="$PWD/libopen_interposer.so" \ + ./preload_open_bench + +gcc -O2 -Wall -Wextra -Werror -shared -fPIC \ + trap_preload.c -o libtrap_preload.so + +gcc -O2 -Wall -Wextra -Werror libreflect_runner.c \ + -I/path/to/libreflect/include -L/path/to/libreflect/lib \ + -Wl,-rpath,/path/to/libreflect/lib -lreflect -o libreflect_runner +``` + +The esbuild handoff used the reference Anvil `ulexecve.py` named by the prior +research, and the current `@esbuild/linux-arm64` package (0.28.1): + +```bash +export ULEXECVE_PATH=/path/to/reference/ulexecve.py +export ESBUILD=/path/to/@esbuild/linux-arm64/bin/esbuild + +python3 ./ulexecve_at_random_fix.py "$ESBUILD" --version +LD_PRELOAD="$PWD/libtrap_preload.so" \ + python3 ./ulexecve_at_random_fix.py "$ESBUILD" --version +LD_PRELOAD="$PWD/libtrap_preload.so" \ + python3 ./ulexecve_at_random_fix.py "$ESBUILD" \ + esbuild_input.js --bundle --minify +``` + +## Measured results + +Five pinned runs of the minimal trap probe produced these median values: + +| Path | Median ns/op | Relative to its baseline | +| -------------------------------------------------------- | -----------: | --------------------------: | +| Direct `getpid` syscall, no filter | 115.6 | 1.00x | +| `RET_TRAP`, set return register only | 565.8 | 4.93x | +| `RET_TRAP`, trusted in-process syscall reissue | 713.7 | 6.17x | +| `openat("/dev/null") + close`, no filter | 531.7 | 1.00x | +| Trap + safe path copy + trusted `openat` reissue + close | 1451.7 | 2.72x | +| User notification, supervisor emulates result | 13922.7 | about 122x syscall baseline | +| User notification with `CONTINUE` | 13931.4 | about 123x syscall baseline | + +The in-process emulation and reissue paths intentionally disabled the +alternate-stack diagnostic, which performs an extra syscall. The filesystem +path includes a self `process_vm_readv` and the real `openat`, so it is a more +representative lower bound for fspy than the register-only number. It still +does not include path normalization or recording. + +The four-thread probe completed 200,000 trapped direct syscalls with zero bad +returns. Its first trap on each new thread ran on the target stack; the handler +installed a private alternate stack and edited `ucontext.uc_stack`, after which +all subsequent traps used it. One sample reported an aggregate wall time of +403 ns per call across four vCPUs. This proves concurrency and the lazy-stack +mechanic, not that fspy should claim the application's alternate stack. + +The minimal LD_PRELOAD wrapper only called the next `openat`. Five samples had +median `openat+close` times of 529.4 ns without preload and 527.6 ns with it, +which is indistinguishable from noise. This is a dispatch lower bound, not the +cost of current fspy logging. Its key limitation remains that inline/direct +syscalls bypass it entirely. + +## Mechanics validated + +### Trusted syscall reissue + +The filter checks a random-looking 64-bit magic value in +`seccomp_data.args[5]`. Every syscall currently intercepted by fspy has at most +five real arguments, so the native handler can put the magic in the unused +sixth argument and issue the original syscall without recursively trapping. +This avoids depending on the handler's instruction address, which is useful +when a userland loader preserves a small survivor mapping. + +This is an accidental-bypass guard. A malicious target can supply +the magic and bypass tracing. A per-process random value reduces accidental +collision and casual spoofing, but not an adversary that can inspect the +process. An instruction-pointer allowlist is the harder alternative. + +### Safe argument access + +The handler uses raw self `process_vm_readv`/`process_vm_writev` calls rather +than directly dereferencing target pointers. The invalid-action-pointer test +returned `EFAULT` instead of recursively faulting inside `SIGSYS`. Production +code needs bounded string-copy loops and architecture-specific ABI tests. + +### Signal-state virtualization + +The host action uses `SA_SIGINFO | SA_NODEFER`. A deliberate direct syscall +from inside the handler nested successfully; without `SA_NODEFER`, automatic +blocking of `SIGSYS` makes a foreign nested trap fatal. + +The filter traps target `rt_sigaction(SIGSYS, ...)` calls. The target can +install/query a virtual handler or `SIG_IGN`, while the kernel retains the host +handler. It also traps `rt_sigprocmask`, strips `SIGSYS` from the real mask, +and maintains a virtual target-visible bit. A target that observed `SIGSYS` as +blocked still completed the next trapped syscall. This is required because a +seccomp-generated `SIGSYS` whose real disposition is blocked or ignored is +forced to the default disposition. + +The prototype mask state is global for simplicity. Production state must be +per-thread and async-signal-safe. It must also consider masks restored by +`rt_sigreturn`, target-installed seccomp filters, and preexisting uses of +`SIGSYS`. + +### Handler bootstrap after the host exec + +`reexec_bootstrap.c` installed the filter, issued a trusted real `execve` of +its static-musl image, and entered a fresh second stage. The kernel reset the +caught handler and preserved the filter. The second stage used a raw +magic-bypassed `rt_sigaction` before its first trapped syscall; a direct +`getpid` then trapped and returned the emulated value: + +```text +reexec-bootstrap: handler reinstalled before trapped syscall PASS +status=0 +``` + +This validates the central target-exec-to-host-exec cycle on the tested ARM64 +musl startup. It does not prove every CRT/toolchain is quiet before `main`; +production should use a small audited custom entry routine rather than depend +on that property. + +### Alternate signal stacks + +Alternate stacks are per-thread and new threads do not inherit one. The lazy +experiment works, but silently replacing the application's one alternate stack +breaks its `SA_ONSTACK` semantics and leaks mappings when threads churn. The +least intrusive production default is likely to let the native handler run on +the current target stack. Full isolation requires virtualizing `sigaltstack` +and multiplexing host/application state per thread; that is substantially more +work and the first trap on a new thread still arrives on its current stack. + +## Userland handoff and esbuild + +The earlier libreflect survival probe was rerun on this VM: + +```text +pure loader: handler=1 altstack=1 trapped_getpid=424242 (PASS) +real execve: Bad system call, status 159 +``` + +The exec control demonstrates the expected reset of the caught signal +disposition and alternate stack while the seccomp filter survives. + +Libreflect also loaded current esbuild 0.28.1 directly. With the full probe DSO +retained across the handoff, both version output and the two-file bundle passed: + +```text +0.28.1 +fspy-sigsys-probe: fs=5 getpid=0 sigaction=2 sigprocmask=14 +fspy-sigsys-probe: fs=72 getpid=0 sigaction=2 sigprocmask=14 +(()=>{console.log(12**2);})(); +``` + +These are cleaner target counts than the Anvil experiment below because +libreflect's native runner needs very little filesystem setup. Esbuild did not +call `getpid`, but its Go runtime performed two `SIGSYS` action operations and +14 mask operations, so the compatibility test directly exercised signal +virtualization as well as static Go's filesystem syscalls and worker threads. + +An unmodified Anvil loader ran esbuild 0.17.19, 0.19.12, and 0.21.5, but +segfaulted for 0.24.2, 0.25.12, 0.27.3, and current 0.28.1. The dividing line +correlated with Go 1.20 versus Go 1.23+. A GDB hardware watchpoint found the +cause: the loader's `AT_RANDOM` value points to +`stack_base + auxv_word_index`, treating a word index as bytes and landing in +`argv`. Go 1.23+ `runtime.randinit` overwrites the consumed 16-byte seed, which +corrupts `argv[1]` and its null terminator. + +The wrapper in this directory allocates 16 dedicated random bytes and patches +the auxiliary vector before handoff. With that correction, current esbuild +0.28.1 (static ARM64, Go 1.26.4) passed both `--version` and a two-file bundle +under the filter: + +```text +0.28.1 +fspy-sigsys-probe: fs=422 getpid=2 sigaction=6 sigprocmask=23 +fspy-sigsys-probe: fs=489 getpid=2 sigaction=6 sigprocmask=23 +(()=>{console.log(12**2);})(); +``` + +The counts include Python/loader setup before the handoff, but the successful +bundle necessarily exercised static Go's direct syscalls, worker threads, and +signal initialization after the handoff. Go attempted six `SIGSYS` action +operations and 23 mask operations, demonstrating why signal virtualization is +not optional. + +## Feasibility boundary + +A purpose-built fspy loader can fix the defects seen in reference loaders: + +- validate ELF headers and segment bounds; +- map `PT_LOAD` ranges with correct final W^X permissions and collision checks; +- copy argv, environment, platform, exec filename, and 16 random bytes into an + owned initial-stack mapping; +- construct correct native auxv entries and interpreter state; +- retain a small position-independent survivor island containing the handler, + restorer, raw syscall gate, and state; +- fail cleanly when fixed-address target segments collide with that island. + +The proposed exec rewrite avoids the largest multithreading gap. On every +logical target exec, the handler can issue a real kernel exec of the static +`fspy_host`, carrying the intended target and argv as host arguments. That +kernel transition kills sibling threads, releases a vfork parent, closes +CLOEXEC descriptors, and performs the normal kernel exec resets. The new host +is single-threaded when it performs the pure userland target handoff. Residual +threads only affect direct in-process-loader experiments +or if `fspy_host` itself starts threads before handoff. + +The remaining differences come from target-specific exec behavior. +`/proc/self/exe` and kernel process identity name the +host; target setuid/file-capability and LSM transitions are not applied; +shebang and `binfmt_misc` dispatch, noexec-mount policy, executable-file +accounting, dumpability, and other target-specific decisions need explicit +handling or remain observably different. There is also an atomicity problem: +after kernel exec successfully commits to `fspy_host`, a later parse, mapping, +interpreter, or startup failure cannot return the target's original exec errno +to the old image. Preflight checks reduce but cannot eliminate this +failure-after-host-commit window. + +Because the inherited seccomp filter survives the kernel exec while caught +signal actions do not, the static host must install its native handler before +performing any intercepted syscall. Its earliest entry code must use the +trusted bypass for `rt_sigaction`; a dynamic loader or ordinary CRT startup is +too early to trust unless audited syscall by syscall. + +The prototype supports continuing this design. The next milestone should be a +hardened, custom-entry static host that validates and loads the target without +starting threads, plus explicit policy for unsupported target-specific exec +semantics and post-commit failures. diff --git a/research/sigsys-prototype/RESULTS.txt b/research/sigsys-prototype/RESULTS.txt new file mode 100644 index 000000000..050d0b057 --- /dev/null +++ b/research/sigsys-prototype/RESULTS.txt @@ -0,0 +1,93 @@ +Environment +=========== +Linux lima-fspy-sigsys 6.8.0-134-generic #134-Ubuntu SMP PREEMPT_DYNAMIC Fri Jun 26 18:28:11 UTC 2026 aarch64 +Ubuntu 24.04.4 LTS; 4 vCPU; 6 GiB; GCC 13.3.0; glibc 2.39; musl 1.2.4 + +trap_bench, five taskset -c 0 runs +================================== +baseline_ns=114.8 trap_emulated_ns=565.9 trap_reissued_ns=708.1 open_baseline_ns=545.2 open_trapped_ns=1457.1 +baseline_ns=115.8 trap_emulated_ns=574.0 trap_reissued_ns=713.7 open_baseline_ns=526.9 open_trapped_ns=1449.6 +baseline_ns=115.6 trap_emulated_ns=564.3 trap_reissued_ns=718.8 open_baseline_ns=531.7 open_trapped_ns=1466.6 +baseline_ns=116.8 trap_emulated_ns=564.3 trap_reissued_ns=760.7 open_baseline_ns=530.3 open_trapped_ns=1439.5 +baseline_ns=114.4 trap_emulated_ns=565.8 trap_reissued_ns=703.7 open_baseline_ns=533.4 open_trapped_ns=1451.7 + +Representative complete validation output +========================================= +environment: pid=5873 arch=aarch64 dynamic_probe=yes +filesystem: rewrite=/virtual/fspy-hostname->/etc/hostname bytes=17 errno_passthrough=ENOENT +multithreading: threads=4 calls=200000 failures=0 +benchmark: iterations=200000 baseline_ns=114.6 trap_emulated_ns=556.6 trap_reissued_ns=700.6 emulated_overhead_x=4.86 reissued_overhead_x=6.12 +openat_benchmark: iterations=100000 baseline_open_close_ns=530.6 trap_reissued_open_close_ns=1442.2 overhead_x=2.72 +traps: total=700032 getpid=600005 openat=100002 sigaction=4 sigprocmask=21 +altstack: on_alt=200013 on_normal=4 lazy_installed=4 +result: PASS + +Real exec into static-musl host bootstrap +======================================== +reexec_bootstrap_musl: ELF 64-bit LSB executable, ARM aarch64, statically linked +reexec-bootstrap: handler reinstalled before trapped syscall PASS +status=0 + +Static musl representative +========================== +trap_bench_musl_static: ELF 64-bit LSB executable, ARM aarch64, statically linked +baseline_ns=115.1 trap_emulated_ns=554.3 trap_reissued_ns=708.5 +open_baseline_ns=547.8 open_trapped_ns=1470.5 +altstack: on_alt=200013 on_normal=4 lazy_installed=4 +result: PASS + +unotify_bench, five runs +======================== +emulated_ns=13991.5 continue_ns=13846.4 +emulated_ns=13922.7 continue_ns=14009.8 +emulated_ns=13885.2 continue_ns=13931.4 +emulated_ns=13896.1 continue_ns=13824.1 +emulated_ns=14100.5 continue_ns=14226.6 +result: PASS (all runs) + +Minimal LD_PRELOAD openat+close, alternating plain/preloaded +============================================================ +plain_ns=546.6 preloaded_ns=527.6 +plain_ns=521.3 preloaded_ns=524.6 +plain_ns=529.4 preloaded_ns=526.0 +plain_ns=523.6 preloaded_ns=533.7 +plain_ns=547.4 preloaded_ns=528.4 + +Signal persistence controls +=========================== +PURE_LOADER +sigsys-survived handler=1 altstack=1 trapped_getpid=424242 exe=/tmp/fspy-loader-probe.o8BDFe/sigsys-test/libreflect_sigsys +REAL_EXEC +Bad system call (core dumped) +real_exec_status=159 + +Current esbuild 0.28.1 through libreflect + trap_preload +======================================================= +PLAIN_VERSION +0.28.1 +plain_status=0 +FILTERED_VERSION +fspy-sigsys-probe: fs=5 getpid=0 sigaction=2 sigprocmask=14 +0.28.1 +filtered_status=0 +FILTERED_BUNDLE +fspy-sigsys-probe: fs=72 getpid=0 sigaction=2 sigprocmask=14 +(()=>{console.log(12**2);})(); +bundle_status=0 + +Anvil esbuild version matrix before AT_RANDOM fix +================================================ +0.17.19 Go 1.20.4 status=0 +0.19.12 Go 1.20.12 status=0 +0.21.5 Go 1.20.12 status=0 +0.24.2 Go 1.23.1 status=139 +0.25.12 Go 1.23.12 status=139 +0.27.3 Go 1.25.7 status=139 +0.28.1 Go 1.26.4 status=139 + +After dedicated 16-byte AT_RANDOM fix +===================================== +0.28.1 +fspy-sigsys-probe: fs=422 getpid=2 sigaction=6 sigprocmask=23 +fspy-sigsys-probe: fs=489 getpid=2 sigaction=6 sigprocmask=23 +(()=>{console.log(12**2);})(); diff --git a/research/sigsys-prototype/esbuild_input.js b/research/sigsys-prototype/esbuild_input.js new file mode 100644 index 000000000..3cfccf518 --- /dev/null +++ b/research/sigsys-prototype/esbuild_input.js @@ -0,0 +1,3 @@ +import { value } from './esbuild_value.js'; + +console.log(value ** 2); diff --git a/research/sigsys-prototype/esbuild_value.js b/research/sigsys-prototype/esbuild_value.js new file mode 100644 index 000000000..cb179abf6 --- /dev/null +++ b/research/sigsys-prototype/esbuild_value.js @@ -0,0 +1 @@ +export const value = 12; diff --git a/research/sigsys-prototype/libreflect_runner.c b/research/sigsys-prototype/libreflect_runner.c new file mode 100644 index 000000000..b7bda1553 --- /dev/null +++ b/research/sigsys-prototype/libreflect_runner.c @@ -0,0 +1,32 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include + +#include + +extern char **environ; + +int main(int argc, char **argv) +{ + struct stat status; + unsigned char *elf; + int descriptor; + + if (argc < 2) { + fprintf(stderr, "usage: %s TARGET [ARG ...]\n", argv[0]); + return 64; + } + descriptor = open(argv[1], O_RDONLY); + if (descriptor < 0 || fstat(descriptor, &status) != 0) + return 65; + elf = mmap(NULL, (size_t)status.st_size, PROT_READ, MAP_PRIVATE, + descriptor, 0); + close(descriptor); + if (elf == MAP_FAILED) + return 66; + reflect_execve(elf, &argv[1], environ); +} diff --git a/research/sigsys-prototype/preload_open_bench.c b/research/sigsys-prototype/preload_open_bench.c new file mode 100644 index 000000000..de3c7ac43 --- /dev/null +++ b/research/sigsys-prototype/preload_open_bench.c @@ -0,0 +1,32 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include + +#define ITERATIONS 500000 + +static uint64_t monotonic_nanoseconds(void) +{ + struct timespec value; + if (clock_gettime(CLOCK_MONOTONIC, &value) != 0) + abort(); + return (uint64_t)value.tv_sec * UINT64_C(1000000000) + value.tv_nsec; +} + +int main(void) +{ + uint64_t start = monotonic_nanoseconds(); + for (int index = 0; index < ITERATIONS; ++index) { + int descriptor = openat(AT_FDCWD, "/dev/null", O_RDONLY, 0); + if (descriptor < 0 || close(descriptor) != 0) + abort(); + } + uint64_t elapsed = monotonic_nanoseconds() - start; + printf("openat_close: iterations=%d ns_per_call=%.1f\n", ITERATIONS, + (double)elapsed / ITERATIONS); + return 0; +} diff --git a/research/sigsys-prototype/preload_open_interposer.c b/research/sigsys-prototype/preload_open_interposer.c new file mode 100644 index 000000000..926a9dd8f --- /dev/null +++ b/research/sigsys-prototype/preload_open_interposer.c @@ -0,0 +1,29 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include + +typedef int (*openat_function)(int, const char *, int, ...); +static openat_function next_openat; + +__attribute__((constructor)) +static void initialize_interposer(void) +{ + *(void **)(&next_openat) = dlsym(RTLD_NEXT, "openat"); + if (next_openat == NULL) + abort(); +} + +int openat(int directory, const char *path, int flags, ...) +{ + mode_t mode = 0; + if (flags & (O_CREAT | O_TMPFILE)) { + va_list arguments; + va_start(arguments, flags); + mode = va_arg(arguments, mode_t); + va_end(arguments); + } + return next_openat(directory, path, flags, mode); +} diff --git a/research/sigsys-prototype/reexec_bootstrap.c b/research/sigsys-prototype/reexec_bootstrap.c new file mode 100644 index 000000000..0ec0825f5 --- /dev/null +++ b/research/sigsys-prototype/reexec_bootstrap.c @@ -0,0 +1,200 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TRUST_MAGIC UINT64_C(0xf5f05ec0dec0de55) + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define U64_LO_OFFSET(field) offsetof(struct seccomp_data, field) +#define U64_HI_OFFSET(field) (offsetof(struct seccomp_data, field) + 4) +#else +#define U64_LO_OFFSET(field) (offsetof(struct seccomp_data, field) + 4) +#define U64_HI_OFFSET(field) offsetof(struct seccomp_data, field) +#endif + +#if defined(__aarch64__) +#define EXPECTED_ARCH AUDIT_ARCH_AARCH64 +#elif defined(__x86_64__) +#define EXPECTED_ARCH AUDIT_ARCH_X86_64 +#else +#error Unsupported architecture +#endif + +extern long trusted_syscall6(long number, long arg0, long arg1, long arg2, + long arg3, long arg4, long arg5); + +#if defined(__aarch64__) +__asm__( + ".text\n" + ".global trusted_syscall6\n" + ".type trusted_syscall6, %function\n" + "trusted_syscall6:\n" + " mov x8, x0\n" + " mov x0, x1\n" + " mov x1, x2\n" + " mov x2, x3\n" + " mov x3, x4\n" + " mov x4, x5\n" + " mov x5, x6\n" + " svc #0\n" + " ret\n"); +#else +__asm__( + ".text\n" + ".global trusted_syscall6\n" + ".type trusted_syscall6, @function\n" + "trusted_syscall6:\n" + " mov %rdi, %rax\n" + " mov %rsi, %rdi\n" + " mov %rdx, %rsi\n" + " mov %rcx, %rdx\n" + " mov %r8, %r10\n" + " mov %r9, %r8\n" + " mov 8(%rsp), %r9\n" + " syscall\n" + " ret\n" + ".global sigreturn_restorer\n" + "sigreturn_restorer:\n" + " mov $15, %rax\n" + " syscall\n"); +extern void sigreturn_restorer(void); +#endif + +struct kernel_sigaction { + uintptr_t handler; + unsigned long flags; + uintptr_t restorer; + unsigned long mask; +}; + +static volatile sig_atomic_t trap_count; + +static void handler(int signal_number, siginfo_t *info, void *opaque) +{ + ucontext_t *context = opaque; + if (signal_number != SIGSYS || info->si_syscall != SYS_getpid) + trusted_syscall6(SYS_exit_group, 90, 0, 0, 0, 0, 0); + ++trap_count; +#if defined(__aarch64__) + context->uc_mcontext.regs[0] = 424242; +#else + context->uc_mcontext.gregs[REG_RAX] = 424242; +#endif +} + +static long direct_getpid(void) +{ + long result; +#if defined(__aarch64__) + register long number __asm__("x8") = SYS_getpid; + register long sixth __asm__("x5") = 0; + register long returned __asm__("x0"); + __asm__ volatile("svc #0" : "=r"(returned) + : "r"(number), "r"(sixth) : "memory", "cc"); + result = returned; +#else + register long sixth __asm__("r9") = 0; + __asm__ volatile("syscall" : "=a"(result) + : "a"(SYS_getpid), "r"(sixth) + : "rcx", "r11", "memory"); +#endif + return result; +} + +static int install_filter(void) +{ + struct sock_filter instructions[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, arch)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, EXPECTED_ARCH, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, nr)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_getpid, 3, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_rt_sigaction, 2, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_execve, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, U64_HI_OFFSET(args[5])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, + (uint32_t)(TRUST_MAGIC >> 32), 0, 3), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, U64_LO_OFFSET(args[5])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, + (uint32_t)TRUST_MAGIC, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_TRAP), + }; + struct sock_fprog program = { + .len = (unsigned short)(sizeof(instructions) / sizeof(instructions[0])), + .filter = instructions, + }; + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) + return -1; + return (int)syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, 0, &program); +} + +static void install_initial_handler(void) +{ + struct sigaction action; + memset(&action, 0, sizeof(action)); + action.sa_sigaction = handler; + sigemptyset(&action.sa_mask); + action.sa_flags = SA_SIGINFO | SA_NODEFER; + if (sigaction(SIGSYS, &action, NULL) != 0) + exit(2); +} + +static void bootstrap_handler_with_raw_syscall(void) +{ + struct kernel_sigaction action = { + .handler = (uintptr_t)handler, + .flags = SA_SIGINFO | SA_NODEFER, + .restorer = 0, + .mask = 0, + }; +#if defined(__x86_64__) + action.flags |= 0x04000000UL; /* SA_RESTORER */ + action.restorer = (uintptr_t)sigreturn_restorer; +#endif + long result = trusted_syscall6(SYS_rt_sigaction, SIGSYS, (long)&action, + 0, sizeof(unsigned long), 0, + (long)TRUST_MAGIC); + if (result != 0) + trusted_syscall6(SYS_exit_group, 3, 0, 0, 0, 0, 0); +} + +int main(int argc, char **argv) +{ + if (argc == 2 && strcmp(argv[1], "stage2") == 0) { + bootstrap_handler_with_raw_syscall(); + if (direct_getpid() != 424242 || trap_count != 1) + return 4; + static const char passed[] = + "reexec-bootstrap: handler reinstalled before trapped syscall PASS\n"; + trusted_syscall6(SYS_write, STDERR_FILENO, (long)passed, + sizeof(passed) - 1, 0, 0, 0); + return 0; + } + + install_initial_handler(); + if (install_filter() != 0) + return 5; + char *next_argv[] = {argv[0], "stage2", NULL}; + char *next_env[] = {"PATH=/usr/bin:/bin", NULL}; + long result = trusted_syscall6(SYS_execve, (long)argv[0], + (long)next_argv, (long)next_env, + 0, 0, (long)TRUST_MAGIC); + (void)result; + return 6; +} diff --git a/research/sigsys-prototype/trap_bench.c b/research/sigsys-prototype/trap_bench.c new file mode 100644 index 000000000..9cc173fce --- /dev/null +++ b/research/sigsys-prototype/trap_bench.c @@ -0,0 +1,719 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* fspy's intercepted syscalls currently have at most five real arguments. */ +#define TRUST_MAGIC UINT64_C(0xf5f05ec0dec0de55) +#define ALTSTACK_SIZE (128U * 1024U) +#define BENCH_ITERS 200000 +#define OPEN_BENCH_ITERS 100000 +#define THREADS 4 +#define THREAD_ITERS 50000 + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define U64_LO_OFFSET(field) offsetof(struct seccomp_data, field) +#define U64_HI_OFFSET(field) (offsetof(struct seccomp_data, field) + 4) +#else +#define U64_LO_OFFSET(field) (offsetof(struct seccomp_data, field) + 4) +#define U64_HI_OFFSET(field) offsetof(struct seccomp_data, field) +#endif + +#if defined(__aarch64__) +#define EXPECTED_ARCH AUDIT_ARCH_AARCH64 +#elif defined(__x86_64__) +#define EXPECTED_ARCH AUDIT_ARCH_X86_64 +#else +#error This probe supports AArch64 and x86-64 only +#endif + +extern long trusted_syscall6(long number, long arg0, long arg1, long arg2, + long arg3, long arg4, long arg5); + +#if defined(__aarch64__) +__asm__( + ".text\n" + ".balign 16\n" + ".global trusted_syscall6\n" + ".type trusted_syscall6, %function\n" + "trusted_syscall6:\n" + " mov x8, x0\n" + " mov x0, x1\n" + " mov x1, x2\n" + " mov x2, x3\n" + " mov x3, x4\n" + " mov x4, x5\n" + " mov x5, x6\n" + " svc #0\n" + " ret\n" + ".size trusted_syscall6, .-trusted_syscall6\n"); +#elif defined(__x86_64__) +__asm__( + ".text\n" + ".balign 16\n" + ".global trusted_syscall6\n" + ".type trusted_syscall6, @function\n" + "trusted_syscall6:\n" + " mov %rdi, %rax\n" + " mov %rsi, %rdi\n" + " mov %rdx, %rsi\n" + " mov %rcx, %rdx\n" + " mov %r8, %r10\n" + " mov %r9, %r8\n" + " mov 8(%rsp), %r9\n" + " syscall\n" + " ret\n" + ".size trusted_syscall6, .-trusted_syscall6\n"); +#endif + +struct kernel_sigaction { + uintptr_t handler; + unsigned long flags; + uintptr_t restorer; + unsigned long mask; +}; + +static _Atomic uint64_t total_traps; +static _Atomic uint64_t getpid_traps; +static _Atomic uint64_t openat_traps; +static _Atomic uint64_t sigaction_traps; +static _Atomic uint64_t sigprocmask_traps; +static _Atomic uint64_t handlers_on_altstack; +static _Atomic uint64_t handlers_on_normal_stack; +static _Atomic uint64_t lazy_altstacks_installed; +static volatile sig_atomic_t getpid_passthrough; +static volatile sig_atomic_t altstack_probe_enabled; +static volatile sig_atomic_t nested_test_pending; +static volatile sig_atomic_t nested_test_result; +static pid_t real_pid; +static struct kernel_sigaction virtual_sigsys_action; +static unsigned long virtual_sigsys_mask; + +static long payload_direct_getpid(void); + +static inline int raw_failed(long result) +{ + return result < 0 && result >= -4095; +} + +static long safe_copy_from_target(void *local, const void *remote, size_t size) +{ + struct iovec local_iov = {.iov_base = local, .iov_len = size}; + struct iovec remote_iov = {.iov_base = (void *)remote, .iov_len = size}; + + return trusted_syscall6(SYS_process_vm_readv, real_pid, + (long)&local_iov, 1, (long)&remote_iov, 1, 0); +} + +static long safe_copy_to_target(void *remote, const void *local, size_t size) +{ + struct iovec local_iov = {.iov_base = (void *)local, .iov_len = size}; + struct iovec remote_iov = {.iov_base = remote, .iov_len = size}; + + return trusted_syscall6(SYS_process_vm_writev, real_pid, + (long)&local_iov, 1, (long)&remote_iov, 1, 0); +} + +static int bytes_equal(const char *left, const char *right) +{ + size_t index = 0; + + for (;;) { + if (left[index] != right[index]) + return 0; + if (left[index] == '\0') + return 1; + ++index; + } +} + +static void set_result(ucontext_t *context, long result) +{ +#if defined(__aarch64__) + context->uc_mcontext.regs[0] = (unsigned long)result; +#elif defined(__x86_64__) + context->uc_mcontext.gregs[REG_RAX] = (greg_t)result; +#endif +} + +static void get_arguments(ucontext_t *context, long arguments[6]) +{ +#if defined(__aarch64__) + for (size_t index = 0; index < 6; ++index) + arguments[index] = (long)context->uc_mcontext.regs[index]; +#elif defined(__x86_64__) + arguments[0] = context->uc_mcontext.gregs[REG_RDI]; + arguments[1] = context->uc_mcontext.gregs[REG_RSI]; + arguments[2] = context->uc_mcontext.gregs[REG_RDX]; + arguments[3] = context->uc_mcontext.gregs[REG_R10]; + arguments[4] = context->uc_mcontext.gregs[REG_R8]; + arguments[5] = context->uc_mcontext.gregs[REG_R9]; +#endif +} + +/* + * New threads start with their alternate signal stack disabled. The first + * trapped syscall safely arrives on the target stack, installs a private + * stack with raw syscalls, and edits uc_stack so rt_sigreturn preserves it. + */ +static void ensure_thread_altstack(ucontext_t *context, const void *frame) +{ + stack_t current; + long result = trusted_syscall6(SYS_sigaltstack, 0, (long)¤t, + 0, 0, 0, 0); + uintptr_t frame_address = (uintptr_t)frame; + int on_altstack = 0; + + if (result == 0 && !(current.ss_flags & SS_DISABLE)) { + uintptr_t low = (uintptr_t)current.ss_sp; + uintptr_t high = low + current.ss_size; + on_altstack = frame_address >= low && frame_address < high; + } + + if (on_altstack) { + atomic_fetch_add_explicit(&handlers_on_altstack, 1, + memory_order_relaxed); + return; + } + + atomic_fetch_add_explicit(&handlers_on_normal_stack, 1, + memory_order_relaxed); + if (result != 0 || !(current.ss_flags & SS_DISABLE)) + return; + + long mapping = trusted_syscall6( + SYS_mmap, 0, ALTSTACK_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (raw_failed(mapping)) + return; + + stack_t replacement = { + .ss_sp = (void *)mapping, + .ss_size = ALTSTACK_SIZE, + .ss_flags = 0, + }; + result = trusted_syscall6(SYS_sigaltstack, (long)&replacement, 0, + 0, 0, 0, 0); + if (result == 0) { + context->uc_stack = replacement; + atomic_fetch_add_explicit(&lazy_altstacks_installed, 1, + memory_order_relaxed); + } +} + +static void handle_openat(ucontext_t *context, const long arguments[6]) +{ + static const char virtual_path[] = "/virtual/fspy-hostname"; + static const char real_path[] = "/etc/hostname"; + char path[128]; + const char *effective_path = (const char *)arguments[1]; + long copied = safe_copy_from_target(path, effective_path, sizeof(path)); + + if (copied > 0) { + path[sizeof(path) - 1] = '\0'; + if (bytes_equal(path, virtual_path)) + effective_path = real_path; + } + + long result = trusted_syscall6(SYS_openat, arguments[0], + (long)effective_path, arguments[2], + arguments[3], arguments[4], + (long)TRUST_MAGIC); + set_result(context, result); +} + +static void handle_rt_sigaction(ucontext_t *context, const long arguments[6]) +{ + struct kernel_sigaction next; + long result; + + if (arguments[2] != 0) { + result = safe_copy_to_target((void *)arguments[2], + &virtual_sigsys_action, + sizeof(virtual_sigsys_action)); + if (result != (long)sizeof(virtual_sigsys_action)) { + set_result(context, -EFAULT); + return; + } + } + + if (arguments[1] != 0) { + result = safe_copy_from_target(&next, (void *)arguments[1], + sizeof(next)); + if (result != (long)sizeof(next)) { + set_result(context, -EFAULT); + return; + } + virtual_sigsys_action = next; + } + set_result(context, 0); +} + +static void handle_rt_sigprocmask(ucontext_t *context, const long arguments[6]) +{ + const unsigned long sigsys_bit = 1UL << (SIGSYS - 1); + unsigned long requested = 0; + unsigned long sanitized = 0; + unsigned long actual_old = 0; + unsigned long visible_old; + long result; + + if (arguments[3] < (long)sizeof(unsigned long)) { + set_result(context, -EINVAL); + return; + } + if (arguments[1] != 0) { + result = safe_copy_from_target(&requested, (void *)arguments[1], + sizeof(requested)); + if (result != (long)sizeof(requested)) { + set_result(context, -EFAULT); + return; + } + sanitized = requested & ~sigsys_bit; + } + + result = trusted_syscall6(SYS_rt_sigprocmask, arguments[0], + arguments[1] ? (long)&sanitized : 0, + (long)&actual_old, arguments[3], 0, + (long)TRUST_MAGIC); + if (raw_failed(result)) { + set_result(context, result); + return; + } + + visible_old = actual_old | virtual_sigsys_mask; + if (arguments[2] != 0) { + result = safe_copy_to_target((void *)arguments[2], &visible_old, + sizeof(visible_old)); + if (result != (long)sizeof(visible_old)) { + set_result(context, -EFAULT); + return; + } + } + + if (arguments[1] != 0) { + switch (arguments[0]) { + case SIG_BLOCK: + virtual_sigsys_mask |= requested & sigsys_bit; + break; + case SIG_UNBLOCK: + virtual_sigsys_mask &= ~(requested & sigsys_bit); + break; + case SIG_SETMASK: + virtual_sigsys_mask = requested & sigsys_bit; + break; + default: + set_result(context, -EINVAL); + return; + } + } + set_result(context, 0); +} + +static void sigsys_handler(int signal_number, siginfo_t *info, + void *context_pointer) +{ + char frame_byte; + long arguments[6]; + ucontext_t *context = context_pointer; + + if (signal_number != SIGSYS) + trusted_syscall6(SYS_exit_group, 90, 0, 0, 0, 0, 0); + + if (altstack_probe_enabled) + ensure_thread_altstack(context, &frame_byte); + atomic_fetch_add_explicit(&total_traps, 1, memory_order_relaxed); + get_arguments(context, arguments); + + switch (info->si_syscall) { + case SYS_getpid: + atomic_fetch_add_explicit(&getpid_traps, 1, memory_order_relaxed); + if (getpid_passthrough) { + set_result(context, trusted_syscall6( + SYS_getpid, 0, 0, 0, 0, 0, (long)TRUST_MAGIC)); + } else { + set_result(context, 424242); + } + if (nested_test_pending) { + nested_test_pending = 0; + nested_test_result = (sig_atomic_t)payload_direct_getpid(); + } + return; + case SYS_openat: + atomic_fetch_add_explicit(&openat_traps, 1, memory_order_relaxed); + handle_openat(context, arguments); + return; + case SYS_rt_sigaction: + atomic_fetch_add_explicit(&sigaction_traps, 1, + memory_order_relaxed); + handle_rt_sigaction(context, arguments); + return; + case SYS_rt_sigprocmask: + atomic_fetch_add_explicit(&sigprocmask_traps, 1, + memory_order_relaxed); + handle_rt_sigprocmask(context, arguments); + return; + default: + trusted_syscall6(SYS_exit_group, 91, 0, 0, 0, 0, 0); + } +} + +static long payload_direct_getpid(void) +{ + long result; +#if defined(__aarch64__) + register long syscall_number __asm__("x8") = SYS_getpid; + register long sixth_argument __asm__("x5") = 0; + register long return_value __asm__("x0"); + __asm__ volatile("svc #0" + : "=r"(return_value) + : "r"(syscall_number), "r"(sixth_argument) + : "memory", "cc"); + result = return_value; +#elif defined(__x86_64__) + register long sixth_argument __asm__("r9") = 0; + __asm__ volatile("syscall" + : "=a"(result) + : "a"(SYS_getpid), "r"(sixth_argument) + : "rcx", "r11", "memory"); +#endif + return result; +} + +static uint64_t monotonic_nanoseconds(void) +{ + struct timespec time; + if (clock_gettime(CLOCK_MONOTONIC, &time) != 0) + abort(); + return (uint64_t)time.tv_sec * UINT64_C(1000000000) + time.tv_nsec; +} + +static double benchmark_getpid(int iterations, long expected) +{ + volatile uint64_t accumulator = 0; + uint64_t start = monotonic_nanoseconds(); + + for (int index = 0; index < iterations; ++index) + accumulator += (uint64_t)payload_direct_getpid(); + + uint64_t elapsed = monotonic_nanoseconds() - start; + if (accumulator != (uint64_t)expected * (uint64_t)iterations) { + fprintf(stderr, "unexpected accumulator: %llu\n", + (unsigned long long)accumulator); + exit(2); + } + return (double)elapsed / iterations; +} + +static double benchmark_open_close(int iterations) +{ + uint64_t start = monotonic_nanoseconds(); + for (int index = 0; index < iterations; ++index) { + int descriptor = (int)syscall(SYS_openat, AT_FDCWD, "/dev/null", + O_RDONLY, 0); + if (descriptor < 0 || close(descriptor) != 0) { + perror("benchmark openat/close"); + exit(2); + } + } + uint64_t elapsed = monotonic_nanoseconds() - start; + return (double)elapsed / iterations; +} + +static int install_filter(void) +{ + const uint32_t magic_low = (uint32_t)TRUST_MAGIC; + const uint32_t magic_high = (uint32_t)(TRUST_MAGIC >> 32); + struct sock_filter instructions[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, arch)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, EXPECTED_ARCH, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, nr)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_getpid, 6, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_openat, 5, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_rt_sigprocmask, 4, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_rt_sigaction, 0, 2), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + U64_LO_OFFSET(args[0])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SIGSYS, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + U64_HI_OFFSET(args[5])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, magic_high, 0, 3), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + U64_LO_OFFSET(args[5])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, magic_low, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_TRAP), + }; + struct sock_fprog program = { + .len = (unsigned short)(sizeof(instructions) / sizeof(instructions[0])), + .filter = instructions, + }; + + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) + return -1; + return (int)syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, 0, &program); +} + +static void install_handler_and_main_altstack(void) +{ + void *mapping = mmap(NULL, ALTSTACK_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (mapping == MAP_FAILED) { + perror("mmap altstack"); + exit(2); + } + stack_t stack = {.ss_sp = mapping, .ss_size = ALTSTACK_SIZE, .ss_flags = 0}; + if (sigaltstack(&stack, NULL) != 0) { + perror("sigaltstack"); + exit(2); + } + + struct sigaction action; + memset(&action, 0, sizeof(action)); + action.sa_sigaction = sigsys_handler; + sigemptyset(&action.sa_mask); + action.sa_flags = SA_SIGINFO | SA_ONSTACK | SA_NODEFER; + if (sigaction(SIGSYS, &action, NULL) != 0) { + perror("sigaction"); + exit(2); + } +} + +static void dummy_target_sigsys_handler(int signal_number) +{ + (void)signal_number; +} + +static void test_sigaction_virtualization(void) +{ + struct sigaction action; + struct sigaction old_action; + struct sigaction queried_action; + + memset(&action, 0, sizeof(action)); + action.sa_handler = dummy_target_sigsys_handler; + sigemptyset(&action.sa_mask); + if (sigaction(SIGSYS, &action, &old_action) != 0) { + perror("virtual sigaction install"); + exit(3); + } + if (sigaction(SIGSYS, NULL, &queried_action) != 0) { + perror("virtual sigaction query"); + exit(3); + } + if (old_action.sa_handler != SIG_DFL || + queried_action.sa_handler != dummy_target_sigsys_handler) { + fprintf(stderr, "virtual sigaction state mismatch\n"); + exit(3); + } + + errno = 0; + long invalid = syscall(SYS_rt_sigaction, SIGSYS, (void *)1, NULL, + sizeof(unsigned long)); + if (invalid != -1 || errno != EFAULT) { + fprintf(stderr, "invalid sigaction pointer was not EFAULT: %ld/%d\n", + invalid, errno); + exit(3); + } + + if (payload_direct_getpid() != 424242) { + fprintf(stderr, "host SIGSYS handler was replaced\n"); + exit(3); + } + + memset(&action, 0, sizeof(action)); + action.sa_handler = SIG_IGN; + sigemptyset(&action.sa_mask); + if (sigaction(SIGSYS, &action, NULL) != 0 || + payload_direct_getpid() != 424242) { + fprintf(stderr, "virtual SIG_IGN disabled host handler\n"); + exit(3); + } +} + +static void test_sigprocmask_virtualization(void) +{ + sigset_t requested; + sigset_t visible; + + sigemptyset(&requested); + sigaddset(&requested, SIGSYS); + if (sigprocmask(SIG_BLOCK, &requested, NULL) != 0 || + sigprocmask(SIG_BLOCK, NULL, &visible) != 0 || + sigismember(&visible, SIGSYS) != 1) { + fprintf(stderr, "SIGSYS block was not virtualized\n"); + exit(3); + } + if (payload_direct_getpid() != 424242) { + fprintf(stderr, "virtual SIGSYS block reached the kernel\n"); + exit(3); + } + if (sigprocmask(SIG_UNBLOCK, &requested, NULL) != 0 || + sigprocmask(SIG_BLOCK, NULL, &visible) != 0 || + sigismember(&visible, SIGSYS) != 0) { + fprintf(stderr, "SIGSYS unblock was not virtualized\n"); + exit(3); + } +} + +static void test_nested_trap(void) +{ + nested_test_result = -1; + nested_test_pending = 1; + if (payload_direct_getpid() != 424242 || nested_test_result != 424242) { + fprintf(stderr, "nested SIGSYS trap failed\n"); + exit(3); + } +} + +static void test_filesystem_interception(void) +{ + char buffer[256]; + int descriptor = (int)syscall(SYS_openat, AT_FDCWD, + "/virtual/fspy-hostname", O_RDONLY, 0); + if (descriptor < 0) { + perror("rewritten openat"); + exit(4); + } + ssize_t length = read(descriptor, buffer, sizeof(buffer) - 1); + close(descriptor); + if (length <= 0) { + perror("read rewritten openat"); + exit(4); + } + buffer[length] = '\0'; + + errno = 0; + descriptor = (int)syscall(SYS_openat, AT_FDCWD, + "/definitely/not/present/fspy", O_RDONLY, 0); + if (descriptor != -1 || errno != ENOENT) { + fprintf(stderr, "openat errno propagation failed: %d/%d\n", + descriptor, errno); + exit(4); + } + printf("filesystem: rewrite=/virtual/fspy-hostname->/etc/hostname " + "bytes=%zd errno_passthrough=ENOENT\n", length); +} + +struct worker_result { + int failures; +}; + +static void *worker_main(void *opaque) +{ + struct worker_result *result = opaque; + for (int index = 0; index < THREAD_ITERS; ++index) { + if (payload_direct_getpid() != 424242) + ++result->failures; + } + return NULL; +} + +static void test_multithreading(void) +{ + pthread_t threads[THREADS]; + struct worker_result results[THREADS] = {{0}}; + + getpid_passthrough = 0; + altstack_probe_enabled = 1; + uint64_t start = monotonic_nanoseconds(); + for (int index = 0; index < THREADS; ++index) { + if (pthread_create(&threads[index], NULL, worker_main, + &results[index]) != 0) { + perror("pthread_create"); + exit(5); + } + } + for (int index = 0; index < THREADS; ++index) { + pthread_join(threads[index], NULL); + if (results[index].failures != 0) { + fprintf(stderr, "thread %d had %d failures\n", + index, results[index].failures); + exit(5); + } + } + uint64_t elapsed = monotonic_nanoseconds() - start; + printf("multithreading: threads=%d calls=%d failures=0 " + "wall_ns_per_call=%.1f\n", + THREADS, THREADS * THREAD_ITERS, + (double)elapsed / (THREADS * THREAD_ITERS)); + altstack_probe_enabled = 0; +} + +int main(void) +{ + real_pid = getpid(); + printf("environment: pid=%d arch=%s dynamic_probe=yes\n", real_pid, +#if defined(__aarch64__) + "aarch64" +#else + "x86_64" +#endif + ); + + double baseline = benchmark_getpid(BENCH_ITERS, real_pid); + double open_baseline = benchmark_open_close(OPEN_BENCH_ITERS); + install_handler_and_main_altstack(); + if (install_filter() != 0) { + perror("seccomp(SECCOMP_RET_TRAP)"); + return 1; + } + + getpid_passthrough = 0; + double trapped_emulated = benchmark_getpid(BENCH_ITERS, 424242); + getpid_passthrough = 1; + double trapped_reissued = benchmark_getpid(BENCH_ITERS, real_pid); + getpid_passthrough = 0; + double open_trapped = benchmark_open_close(OPEN_BENCH_ITERS); + + test_filesystem_interception(); + test_sigaction_virtualization(); + test_sigprocmask_virtualization(); + test_nested_trap(); + test_multithreading(); + + printf("benchmark: iterations=%d baseline_ns=%.1f " + "trap_emulated_ns=%.1f trap_reissued_ns=%.1f " + "emulated_overhead_x=%.2f reissued_overhead_x=%.2f\n", + BENCH_ITERS, baseline, trapped_emulated, trapped_reissued, + trapped_emulated / baseline, trapped_reissued / baseline); + printf("openat_benchmark: iterations=%d baseline_open_close_ns=%.1f " + "trap_reissued_open_close_ns=%.1f overhead_x=%.2f\n", + OPEN_BENCH_ITERS, open_baseline, open_trapped, + open_trapped / open_baseline); + printf("traps: total=%llu getpid=%llu openat=%llu sigaction=%llu " + "sigprocmask=%llu\n", + (unsigned long long)atomic_load(&total_traps), + (unsigned long long)atomic_load(&getpid_traps), + (unsigned long long)atomic_load(&openat_traps), + (unsigned long long)atomic_load(&sigaction_traps), + (unsigned long long)atomic_load(&sigprocmask_traps)); + printf("altstack: on_alt=%llu on_normal=%llu lazy_installed=%llu\n", + (unsigned long long)atomic_load(&handlers_on_altstack), + (unsigned long long)atomic_load(&handlers_on_normal_stack), + (unsigned long long)atomic_load(&lazy_altstacks_installed)); + printf("result: PASS\n"); + return 0; +} diff --git a/research/sigsys-prototype/trap_preload.c b/research/sigsys-prototype/trap_preload.c new file mode 100644 index 000000000..3cd9e8703 --- /dev/null +++ b/research/sigsys-prototype/trap_preload.c @@ -0,0 +1,413 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TRUST_MAGIC UINT64_C(0xf5f05ec0dec0de55) +#define ALTSTACK_SIZE (256U * 1024U) + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define U64_LO_OFFSET(field) offsetof(struct seccomp_data, field) +#define U64_HI_OFFSET(field) (offsetof(struct seccomp_data, field) + 4) +#else +#define U64_LO_OFFSET(field) (offsetof(struct seccomp_data, field) + 4) +#define U64_HI_OFFSET(field) offsetof(struct seccomp_data, field) +#endif + +#if defined(__aarch64__) +#define EXPECTED_ARCH AUDIT_ARCH_AARCH64 +#elif defined(__x86_64__) +#define EXPECTED_ARCH AUDIT_ARCH_X86_64 +#else +#error This probe supports AArch64 and x86-64 only +#endif + +extern long trusted_syscall6(long number, long arg0, long arg1, long arg2, + long arg3, long arg4, long arg5); + +#if defined(__aarch64__) +__asm__( + ".text\n" + ".balign 16\n" + ".global trusted_syscall6\n" + ".type trusted_syscall6, %function\n" + "trusted_syscall6:\n" + " mov x8, x0\n" + " mov x0, x1\n" + " mov x1, x2\n" + " mov x2, x3\n" + " mov x3, x4\n" + " mov x4, x5\n" + " mov x5, x6\n" + " svc #0\n" + " ret\n" + ".size trusted_syscall6, .-trusted_syscall6\n"); +#elif defined(__x86_64__) +__asm__( + ".text\n" + ".balign 16\n" + ".global trusted_syscall6\n" + ".type trusted_syscall6, @function\n" + "trusted_syscall6:\n" + " mov %rdi, %rax\n" + " mov %rsi, %rdi\n" + " mov %rdx, %rsi\n" + " mov %rcx, %rdx\n" + " mov %r8, %r10\n" + " mov %r9, %r8\n" + " mov 8(%rsp), %r9\n" + " syscall\n" + " ret\n" + ".size trusted_syscall6, .-trusted_syscall6\n"); +#endif + +struct kernel_sigaction { + uintptr_t handler; + unsigned long flags; + uintptr_t restorer; + unsigned long mask; +}; + +static uint64_t filesystem_traps; +static uint64_t getpid_traps; +static uint64_t sigaction_traps; +static uint64_t sigprocmask_traps; +static pid_t real_pid; +static struct kernel_sigaction virtual_sigsys_action; +static unsigned long virtual_sigsys_mask; + +static inline int raw_failed(long result) +{ + return result < 0 && result >= -4095; +} + +static long safe_copy_from_target(void *local, const void *remote, size_t size) +{ + struct iovec local_iov = {.iov_base = local, .iov_len = size}; + struct iovec remote_iov = {.iov_base = (void *)remote, .iov_len = size}; + return trusted_syscall6(SYS_process_vm_readv, real_pid, + (long)&local_iov, 1, (long)&remote_iov, 1, 0); +} + +static long safe_copy_to_target(void *remote, const void *local, size_t size) +{ + struct iovec local_iov = {.iov_base = (void *)local, .iov_len = size}; + struct iovec remote_iov = {.iov_base = remote, .iov_len = size}; + return trusted_syscall6(SYS_process_vm_writev, real_pid, + (long)&local_iov, 1, (long)&remote_iov, 1, 0); +} + +static void set_result(ucontext_t *context, long result) +{ +#if defined(__aarch64__) + context->uc_mcontext.regs[0] = (unsigned long)result; +#else + context->uc_mcontext.gregs[REG_RAX] = (greg_t)result; +#endif +} + +static void get_arguments(ucontext_t *context, long arguments[6]) +{ +#if defined(__aarch64__) + for (size_t index = 0; index < 6; ++index) + arguments[index] = (long)context->uc_mcontext.regs[index]; +#else + arguments[0] = context->uc_mcontext.gregs[REG_RDI]; + arguments[1] = context->uc_mcontext.gregs[REG_RSI]; + arguments[2] = context->uc_mcontext.gregs[REG_RDX]; + arguments[3] = context->uc_mcontext.gregs[REG_R10]; + arguments[4] = context->uc_mcontext.gregs[REG_R8]; + arguments[5] = context->uc_mcontext.gregs[REG_R9]; +#endif +} + +static void handle_rt_sigaction(ucontext_t *context, const long arguments[6]) +{ + struct kernel_sigaction next; + long result; + + if (arguments[2]) { + result = safe_copy_to_target((void *)arguments[2], + &virtual_sigsys_action, + sizeof(virtual_sigsys_action)); + if (result != (long)sizeof(virtual_sigsys_action)) { + set_result(context, -EFAULT); + return; + } + } + if (arguments[1]) { + result = safe_copy_from_target(&next, (void *)arguments[1], + sizeof(next)); + if (result != (long)sizeof(next)) { + set_result(context, -EFAULT); + return; + } + virtual_sigsys_action = next; + } + set_result(context, 0); +} + +static void handle_rt_sigprocmask(ucontext_t *context, const long arguments[6]) +{ + const unsigned long sigsys_bit = 1UL << (SIGSYS - 1); + unsigned long requested = 0; + unsigned long sanitized = 0; + unsigned long actual_old = 0; + unsigned long visible_old; + long result; + + if (arguments[3] < (long)sizeof(unsigned long)) { + set_result(context, -EINVAL); + return; + } + if (arguments[1]) { + result = safe_copy_from_target(&requested, (void *)arguments[1], + sizeof(requested)); + if (result != (long)sizeof(requested)) { + set_result(context, -EFAULT); + return; + } + sanitized = requested & ~sigsys_bit; + } + result = trusted_syscall6(SYS_rt_sigprocmask, arguments[0], + arguments[1] ? (long)&sanitized : 0, + (long)&actual_old, arguments[3], 0, + (long)TRUST_MAGIC); + if (raw_failed(result)) { + set_result(context, result); + return; + } + visible_old = actual_old | virtual_sigsys_mask; + if (arguments[2]) { + result = safe_copy_to_target((void *)arguments[2], &visible_old, + sizeof(visible_old)); + if (result != (long)sizeof(visible_old)) { + set_result(context, -EFAULT); + return; + } + } + if (arguments[1]) { + if (arguments[0] == SIG_BLOCK) + virtual_sigsys_mask |= requested & sigsys_bit; + else if (arguments[0] == SIG_UNBLOCK) + virtual_sigsys_mask &= ~(requested & sigsys_bit); + else if (arguments[0] == SIG_SETMASK) + virtual_sigsys_mask = requested & sigsys_bit; + else { + set_result(context, -EINVAL); + return; + } + } + set_result(context, 0); +} + +static char *append_text(char *cursor, const char *text) +{ + while (*text) + *cursor++ = *text++; + return cursor; +} + +static char *append_decimal(char *cursor, uint64_t value) +{ + char reversed[32]; + size_t count = 0; + do { + reversed[count++] = (char)('0' + value % 10); + value /= 10; + } while (value); + while (count) + *cursor++ = reversed[--count]; + return cursor; +} + +static void write_summary(void) +{ + char buffer[256]; + char *cursor = append_text(buffer, "fspy-sigsys-probe: fs="); + cursor = append_decimal(cursor, __atomic_load_n(&filesystem_traps, + __ATOMIC_RELAXED)); + cursor = append_text(cursor, " getpid="); + cursor = append_decimal(cursor, __atomic_load_n(&getpid_traps, + __ATOMIC_RELAXED)); + cursor = append_text(cursor, " sigaction="); + cursor = append_decimal(cursor, __atomic_load_n(&sigaction_traps, + __ATOMIC_RELAXED)); + cursor = append_text(cursor, " sigprocmask="); + cursor = append_decimal(cursor, __atomic_load_n(&sigprocmask_traps, + __ATOMIC_RELAXED)); + *cursor++ = '\n'; + trusted_syscall6(SYS_write, STDERR_FILENO, (long)buffer, + cursor - buffer, 0, 0, 0); +} + +static void sigsys_handler(int signal_number, siginfo_t *info, + void *context_pointer) +{ + ucontext_t *context = context_pointer; + long arguments[6]; + if (signal_number != SIGSYS) + trusted_syscall6(SYS_exit_group, 90, 0, 0, 0, 0, + (long)TRUST_MAGIC); + get_arguments(context, arguments); + + if (info->si_syscall == SYS_getpid) { + __atomic_fetch_add(&getpid_traps, 1, __ATOMIC_RELAXED); + set_result(context, trusted_syscall6( + info->si_syscall, arguments[0], arguments[1], arguments[2], + arguments[3], arguments[4], (long)TRUST_MAGIC)); + return; + } + if (info->si_syscall == SYS_rt_sigaction) { + __atomic_fetch_add(&sigaction_traps, 1, __ATOMIC_RELAXED); + handle_rt_sigaction(context, arguments); + return; + } + if (info->si_syscall == SYS_rt_sigprocmask) { + __atomic_fetch_add(&sigprocmask_traps, 1, __ATOMIC_RELAXED); + handle_rt_sigprocmask(context, arguments); + return; + } + if (info->si_syscall == SYS_exit_group) { + write_summary(); + trusted_syscall6(SYS_exit_group, arguments[0], 0, 0, 0, 0, + (long)TRUST_MAGIC); + return; + } + + __atomic_fetch_add(&filesystem_traps, 1, __ATOMIC_RELAXED); + set_result(context, trusted_syscall6( + info->si_syscall, arguments[0], arguments[1], arguments[2], + arguments[3], arguments[4], (long)TRUST_MAGIC)); +} + +static size_t append_instruction(struct sock_filter *instructions, + size_t *length, unsigned short code, + unsigned char jump_true, + unsigned char jump_false, uint32_t value) +{ + size_t index = (*length)++; + instructions[index] = (struct sock_filter){ + .code = code, + .jt = jump_true, + .jf = jump_false, + .k = value, + }; + return index; +} + +static int install_filter(void) +{ + struct sock_filter instructions[64]; + size_t length = 0; + size_t magic_jumps[32]; + size_t magic_jump_count = 0; + + append_instruction(instructions, &length, BPF_LD | BPF_W | BPF_ABS, + 0, 0, offsetof(struct seccomp_data, arch)); + append_instruction(instructions, &length, BPF_JMP | BPF_JEQ | BPF_K, + 1, 0, EXPECTED_ARCH); + append_instruction(instructions, &length, BPF_RET | BPF_K, 0, 0, + SECCOMP_RET_KILL_PROCESS); + append_instruction(instructions, &length, BPF_LD | BPF_W | BPF_ABS, + 0, 0, offsetof(struct seccomp_data, nr)); + +#define INTERCEPT(syscall_name) \ + do { \ + magic_jumps[magic_jump_count++] = append_instruction( \ + instructions, &length, BPF_JMP | BPF_JEQ | BPF_K, 0, 0, \ + SYS_##syscall_name); \ + } while (0) + INTERCEPT(getpid); + INTERCEPT(openat); +#ifdef SYS_openat2 + INTERCEPT(openat2); +#endif +#ifdef SYS_newfstatat + INTERCEPT(newfstatat); +#elif defined(SYS_fstatat) + INTERCEPT(fstatat); +#endif + INTERCEPT(statx); + INTERCEPT(getdents64); + INTERCEPT(faccessat); +#ifdef SYS_faccessat2 + INTERCEPT(faccessat2); +#endif + INTERCEPT(rt_sigprocmask); + INTERCEPT(exit_group); +#undef INTERCEPT + + size_t sigaction_jump = append_instruction( + instructions, &length, BPF_JMP | BPF_JEQ | BPF_K, 0, 2, + SYS_rt_sigaction); + (void)sigaction_jump; + append_instruction(instructions, &length, BPF_LD | BPF_W | BPF_ABS, + 0, 0, U64_LO_OFFSET(args[0])); + size_t sigsys_jump = append_instruction( + instructions, &length, BPF_JMP | BPF_JEQ | BPF_K, 0, 0, SIGSYS); + append_instruction(instructions, &length, BPF_RET | BPF_K, 0, 0, + SECCOMP_RET_ALLOW); + + size_t magic_check = length; + for (size_t index = 0; index < magic_jump_count; ++index) + instructions[magic_jumps[index]].jt = + (unsigned char)(magic_check - magic_jumps[index] - 1); + instructions[sigsys_jump].jt = + (unsigned char)(magic_check - sigsys_jump - 1); + + append_instruction(instructions, &length, BPF_LD | BPF_W | BPF_ABS, + 0, 0, U64_HI_OFFSET(args[5])); + append_instruction(instructions, &length, BPF_JMP | BPF_JEQ | BPF_K, + 0, 3, (uint32_t)(TRUST_MAGIC >> 32)); + append_instruction(instructions, &length, BPF_LD | BPF_W | BPF_ABS, + 0, 0, U64_LO_OFFSET(args[5])); + append_instruction(instructions, &length, BPF_JMP | BPF_JEQ | BPF_K, + 0, 1, (uint32_t)TRUST_MAGIC); + append_instruction(instructions, &length, BPF_RET | BPF_K, 0, 0, + SECCOMP_RET_ALLOW); + append_instruction(instructions, &length, BPF_RET | BPF_K, 0, 0, + SECCOMP_RET_TRAP); + + struct sock_fprog program = { + .len = (unsigned short)length, + .filter = instructions, + }; + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) + return -1; + return (int)syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, 0, &program); +} + +__attribute__((constructor)) +static void initialize_probe(void) +{ + unsetenv("LD_PRELOAD"); + real_pid = getpid(); + void *mapping = mmap(NULL, ALTSTACK_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (mapping == MAP_FAILED) + _exit(80); + stack_t stack = {.ss_sp = mapping, .ss_size = ALTSTACK_SIZE, .ss_flags = 0}; + if (sigaltstack(&stack, NULL) != 0) + _exit(81); + struct sigaction action = {0}; + action.sa_sigaction = sigsys_handler; + sigemptyset(&action.sa_mask); + action.sa_flags = SA_SIGINFO | SA_ONSTACK | SA_NODEFER; + if (sigaction(SIGSYS, &action, NULL) != 0) + _exit(82); + if (install_filter() != 0) + _exit(83); +} diff --git a/research/sigsys-prototype/ulexecve_at_random_fix.py b/research/sigsys-prototype/ulexecve_at_random_fix.py new file mode 100644 index 000000000..875e0be77 --- /dev/null +++ b/research/sigsys-prototype/ulexecve_at_random_fix.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Run the reference ulexecve loader with a kernel-compatible AT_RANDOM. + +The reference loader points AT_RANDOM into the synthetic initial stack and +uses a word index as a byte offset. Go 1.23+ overwrites the startup random +seed after consuming it, so that bug corrupts argv. This wrapper is only a +small experimental fix; it is not an endorsement of the loader's other +execve emulation choices. +""" + +import ctypes +import importlib.util +import os +import sys + + +loader_path = os.environ.get("ULEXECVE_PATH") +if not loader_path: + raise SystemExit("ULEXECVE_PATH must name the reference ulexecve.py") + +spec = importlib.util.spec_from_file_location("ulexecve_reference", loader_path) +module = importlib.util.module_from_spec(spec) +assert spec.loader is not None +spec.loader.exec_module(module) + +original_setup_auxv = module.Stack.setup_auxv + + +def setup_auxv_with_real_random(self, offset, executable): + end = original_setup_auxv(self, offset, executable) + random_bytes = ctypes.create_string_buffer(os.urandom(16)) + self.add_ref(random_bytes) + + cursor = offset + while self.stack[cursor] != module.Stack.AT_NULL: + if self.stack[cursor] == module.Stack.AT_RANDOM: + self.stack[cursor + 1] = ctypes.addressof(random_bytes) + break + cursor += 2 + else: + raise RuntimeError("synthetic auxiliary vector has no AT_RANDOM") + return end + + +module.Stack.setup_auxv = setup_auxv_with_real_random +module.main() diff --git a/research/sigsys-prototype/unotify_bench.c b/research/sigsys-prototype/unotify_bench.c new file mode 100644 index 000000000..349fc4bf6 --- /dev/null +++ b/research/sigsys-prototype/unotify_bench.c @@ -0,0 +1,286 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BENCH_ITERS 100000 + +#if defined(__aarch64__) +#define EXPECTED_ARCH AUDIT_ARCH_AARCH64 +#elif defined(__x86_64__) +#define EXPECTED_ARCH AUDIT_ARCH_X86_64 +#else +#error This probe supports AArch64 and x86-64 only +#endif + +enum response_mode { + RESPONSE_EMULATE, + RESPONSE_CONTINUE, +}; + +struct child_result { + double baseline_ns; + double notified_ns; + uint64_t accumulator; +}; + +static long direct_getpid(void) +{ + long result; +#if defined(__aarch64__) + register long syscall_number __asm__("x8") = SYS_getpid; + register long return_value __asm__("x0"); + __asm__ volatile("svc #0" + : "=r"(return_value) + : "r"(syscall_number) + : "memory", "cc"); + result = return_value; +#elif defined(__x86_64__) + __asm__ volatile("syscall" + : "=a"(result) + : "a"(SYS_getpid) + : "rcx", "r11", "memory"); +#endif + return result; +} + +static uint64_t monotonic_nanoseconds(void) +{ + struct timespec value; + if (clock_gettime(CLOCK_MONOTONIC, &value) != 0) { + perror("clock_gettime"); + exit(2); + } + return (uint64_t)value.tv_sec * UINT64_C(1000000000) + value.tv_nsec; +} + +static double benchmark_getpid(uint64_t *accumulator) +{ + uint64_t sum = 0; + uint64_t start = monotonic_nanoseconds(); + for (int index = 0; index < BENCH_ITERS; ++index) + sum += (uint64_t)direct_getpid(); + uint64_t elapsed = monotonic_nanoseconds() - start; + *accumulator = sum; + return (double)elapsed / BENCH_ITERS; +} + +static void pin_to_cpu(int cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + if (sched_setaffinity(0, sizeof(set), &set) != 0) { + perror("sched_setaffinity"); + exit(2); + } +} + +static int install_listener(void) +{ + struct sock_filter instructions[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, arch)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, EXPECTED_ARCH, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, + offsetof(struct seccomp_data, nr)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_getpid, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_USER_NOTIF), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + }; + struct sock_fprog program = { + .len = (unsigned short)(sizeof(instructions) / sizeof(instructions[0])), + .filter = instructions, + }; + + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) + return -1; + return (int)syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, + SECCOMP_FILTER_FLAG_NEW_LISTENER, &program); +} + +static void send_listener(int socket_fd, int listener_fd) +{ + char byte = 'L'; + struct iovec iov = {.iov_base = &byte, .iov_len = 1}; + char control[CMSG_SPACE(sizeof(int))]; + memset(control, 0, sizeof(control)); + struct msghdr message = { + .msg_iov = &iov, + .msg_iovlen = 1, + .msg_control = control, + .msg_controllen = sizeof(control), + }; + struct cmsghdr *header = CMSG_FIRSTHDR(&message); + header->cmsg_level = SOL_SOCKET; + header->cmsg_type = SCM_RIGHTS; + header->cmsg_len = CMSG_LEN(sizeof(int)); + memcpy(CMSG_DATA(header), &listener_fd, sizeof(listener_fd)); + if (sendmsg(socket_fd, &message, 0) != 1) { + perror("sendmsg listener"); + exit(3); + } +} + +static int receive_listener(int socket_fd) +{ + char byte; + struct iovec iov = {.iov_base = &byte, .iov_len = 1}; + char control[CMSG_SPACE(sizeof(int))]; + memset(control, 0, sizeof(control)); + struct msghdr message = { + .msg_iov = &iov, + .msg_iovlen = 1, + .msg_control = control, + .msg_controllen = sizeof(control), + }; + if (recvmsg(socket_fd, &message, 0) != 1) { + perror("recvmsg listener"); + exit(3); + } + struct cmsghdr *header = CMSG_FIRSTHDR(&message); + if (header == NULL || header->cmsg_level != SOL_SOCKET || + header->cmsg_type != SCM_RIGHTS) { + fprintf(stderr, "listener fd missing from control message\n"); + exit(3); + } + int listener_fd; + memcpy(&listener_fd, CMSG_DATA(header), sizeof(listener_fd)); + return listener_fd; +} + +static struct child_result run_benchmark(enum response_mode mode) +{ + int sockets[2]; + if (socketpair(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0, sockets) != 0) { + perror("socketpair"); + exit(2); + } + pid_t child = fork(); + if (child < 0) { + perror("fork"); + exit(2); + } + + if (child == 0) { + close(sockets[0]); + pin_to_cpu(0); + struct child_result result; + result.baseline_ns = benchmark_getpid(&result.accumulator); + int listener = install_listener(); + if (listener < 0) { + perror("seccomp user notification listener"); + _exit(3); + } + send_listener(sockets[1], listener); + close(listener); + char ready; + if (read(sockets[1], &ready, 1) != 1) + _exit(3); + result.notified_ns = benchmark_getpid(&result.accumulator); + if (write(sockets[1], &result, sizeof(result)) != sizeof(result)) + _exit(3); + _exit(0); + } + + close(sockets[1]); + pin_to_cpu(1); + int listener = receive_listener(sockets[0]); + char ready = 'R'; + if (write(sockets[0], &ready, 1) != 1) { + perror("write ready"); + exit(3); + } + + struct seccomp_notif request; + struct seccomp_notif_resp response; + for (int index = 0; index < BENCH_ITERS; ++index) { + memset(&request, 0, sizeof(request)); + if (ioctl(listener, SECCOMP_IOCTL_NOTIF_RECV, &request) != 0) { + perror("SECCOMP_IOCTL_NOTIF_RECV"); + exit(3); + } + if (request.data.nr != SYS_getpid || request.pid != (uint32_t)child) { + fprintf(stderr, "unexpected notification nr=%d pid=%u\n", + request.data.nr, request.pid); + exit(3); + } + memset(&response, 0, sizeof(response)); + response.id = request.id; + if (mode == RESPONSE_CONTINUE) { + response.flags = SECCOMP_USER_NOTIF_FLAG_CONTINUE; + } else { + response.val = 424242; + } + if (ioctl(listener, SECCOMP_IOCTL_NOTIF_SEND, &response) != 0) { + perror("SECCOMP_IOCTL_NOTIF_SEND"); + exit(3); + } + } + + struct child_result result; + if (read(sockets[0], &result, sizeof(result)) != sizeof(result)) { + perror("read result"); + exit(3); + } + close(listener); + close(sockets[0]); + int status; + if (waitpid(child, &status, 0) != child || !WIFEXITED(status) || + WEXITSTATUS(status) != 0) { + fprintf(stderr, "child failed: status=%#x\n", status); + exit(3); + } + + uint64_t expected = (uint64_t)(mode == RESPONSE_CONTINUE ? child : 424242) + * BENCH_ITERS; + if (result.accumulator != expected) { + fprintf(stderr, "bad accumulator: %llu expected %llu\n", + (unsigned long long)result.accumulator, + (unsigned long long)expected); + exit(3); + } + return result; +} + +int main(void) +{ + struct child_result emulated = run_benchmark(RESPONSE_EMULATE); + struct child_result continued = run_benchmark(RESPONSE_CONTINUE); + + printf("environment: arch=%s iterations=%d cpus=child:0,supervisor:1\n", +#if defined(__aarch64__) + "aarch64", +#else + "x86_64", +#endif + BENCH_ITERS); + printf("user_notify_emulated: baseline_ns=%.1f notified_ns=%.1f " + "overhead_x=%.2f\n", + emulated.baseline_ns, emulated.notified_ns, + emulated.notified_ns / emulated.baseline_ns); + printf("user_notify_continue: baseline_ns=%.1f notified_ns=%.1f " + "overhead_x=%.2f\n", + continued.baseline_ns, continued.notified_ns, + continued.notified_ns / continued.baseline_ns); + printf("result: PASS\n"); + return 0; +} diff --git a/research/userland-exec-compat/RESULTS.md b/research/userland-exec-compat/RESULTS.md new file mode 100644 index 000000000..92b4e29b5 --- /dev/null +++ b/research/userland-exec-compat/RESULTS.md @@ -0,0 +1,270 @@ +# Userland exec compatibility study + +Research date: 2026-08-02 + +## Verdict + +The proposed architecture is feasible for fspy's build-tool workload, with an +important qualification: every logical exec must first perform a real kernel +exec of a fresh, single-threaded `fspy_host`, and only that host may enter the +target through the userland ELF loader. + +That kernel exec kills sibling threads, +closes `CLOEXEC` descriptors, clears the alternate signal stack, resets caught +signal dispositions, and discards the old address space. The seccomp filter +survives. The new host must reinstall its physical `SIGSYS` handler through a +trusted raw-syscall bootstrap before it makes any syscall that the inherited +filter traps. It can then map and enter the logical target without another +kernel exec, preserving the new handler. + +This does not fully implement Linux `execve`. It can cover +ordinary unprivileged build tools, including current esbuild, Node, shells, +dynamic glibc, static musl, and static Go. It cannot faithfully reproduce +target-file credential/LSM transitions or the target's kernel-owned process +identity. Those limitations need an explicit support contract. + +The loader should be implemented in-house. Libreflect is the most useful small +reference and passed the broadest relevant matrix in this run, but its parser, +mapping decisions, initial stack, and auxiliary vector are not production +quality. Anvil remains a useful behavior reference, not an implementation +base. + +## Environment and method + +- Ubuntu 24.04.4 AArch64 in the `fspy-sigsys` Lima VM +- Linux 6.8.0-134-generic, 4 vCPUs, 6 GiB +- GCC 13.3, glibc 2.39, musl 1.2.4, Go 1.22.2, Node 18.19.1 +- esbuild 0.28.1 from `@esbuild/linux-arm64`, a static Go executable +- Direct AArch64 build of libreflect's pure mapper; its configure script + incorrectly selected the `memfd_create`/`execveat` fallback in this layout +- Anvil `ulexecve.py` at the revision in the supplied prior research +- The full trap case uses `../sigsys-prototype/trap_preload.c`, which traps the + filesystem syscall family, `getpid`, `exit_group`, `rt_sigaction(SIGSYS)`, + and `rt_sigprocmask`, then reissues trusted syscalls in-process + +`run-study.sh` recreates the build and matrix. It accepts an output directory +as its second argument and uses a temporary-directory default. The last compact +result set is in `aarch64-lima-summary.tsv`. + +The duration column is diagnostic, one sample per case, and not a benchmark. + +## Compatibility results + +| Workload | Libreflect pure handoff | Anvil pure handoff | Interpretation | +| -------------------------------------------- | ----------------------- | ------------------ | ------------------------------------------------------- | +| glibc dynamic PIE C, pthreads, `posix_spawn` | Pass | Pass | Ordinary dynamic ELF works | +| glibc dynamic non-PIE | Pass | Pass | Both happened to obtain the fixed mapping | +| static musl, pthreads, `posix_spawn` | Pass | Pass | Static musl works despite imperfect auxv | +| static Go 1.22 | Pass | SIGSEGV | Anvil reference bug, not a kernel limit | +| esbuild 0.28.1 CLI bundle | Pass | SIGSEGV | Libreflect handles a current static Go frontend | +| Node, filesystem, worker thread, shell child | Pass | Pass | Main runtime functionality works | +| Node self-reexec | Fail | Fail | Both expose the physical host as executable identity | +| `/bin/sh` and `/bin/echo` | Pass | Pass | Common dynamic tools work | +| Direct shebang input | Abort | Rejected | Neither reference parses scripts | +| Shebang expanded to `/bin/sh script ...` | Pass | Not tested | Implementable host feature | +| SIGSYS handler survival, dynamic C | Pass | Pass | Pure handoff preserves the physical handler | +| Full trapped-filesystem esbuild bundle | Pass | Not tested here | Signal virtualization is sufficient for current esbuild | + +The earlier supplied evaluation reported libreflect failures for static ELF. +That is not true in this Ubuntu 24.04/Linux 6.8 run: static musl, static Go, and +current esbuild all completed. This does not make libreflect deterministic. +For an `ET_EXEC` image it uses the requested address only as an `mmap` hint and +then assumes the hint was honored. A collision will still break it. + +The raw Anvil failures on Go and esbuild are also narrower than they look. The +companion SIGSYS study traced current esbuild's crash to Anvil's invalid +`AT_RANDOM` pointer; its corrected wrapper runs esbuild. That is a fixable +loader defect. Anvil still maps an entire image RWX and is unsuitable for the +production host. + +## esbuild and frontend-tool result + +Three esbuild paths passed: + +1. Libreflect directly loaded the static esbuild 0.28.1 executable and bundled + the two-file TypeScript fixture with a source map. +2. Userland-loaded Node invoked the esbuild JavaScript API. The API kernel- + execed the test host wrapper, which userland-loaded the static esbuild + service. Build and transform output matched the native control exactly: + esbuild 0.28.1, a 1,395-byte bundled output, and `const answer=42;`. +3. Libreflect loaded esbuild under the full seccomp trap prototype. Bundle and + minification succeeded with this final counter line: + + ```text + fspy-sigsys-probe: fs=86 getpid=0 sigaction=2 sigprocmask=14 + ``` + +The third case is the meaningful SIGSYS result. The simpler inherited-handler +probe trapped only `getpid`; esbuild happened not to call it after Go startup, +so that apparent pass did not exercise the collision. + +The full trap also ran Node's main thread, filesystem operations, and worker +thread. Its child execs did not work because the research DSO does not yet +transform `execve` into a fresh host. A real exec clears the handler while +leaving the filter installed, so a child cannot run until the new host +reinstalls the handler. + +The companion `reexec_bootstrap.c` closes the basic bootstrap question: a +static-musl program installed the filter, performed a trusted real exec of its +own image, reinstalled the handler with a magic-bypassed raw `rt_sigaction`, +then successfully trapped `getpid` in the fresh image. What remains unproven +end to end is decoding an arbitrary trapped `execve`/`execveat`, carrying its +target fd, argv, and environment into the host, and completing Node's real +child-process chain. + +## SIGSYS requirements + +A naive preserved handler is not compatible with Go or multithreaded Node: + +- Static Go installs its own `SIGSYS` action. With the simple filter, its next + trapped `getpid` entered Go's handler and terminated with `SIGSYS: bad system +call`. +- Go repeatedly changes signal masks while creating threads. +- Alternate signal stacks are per-thread. A handler that requires the host's + original alternate stack failed when Node trapped on a worker thread. + +The full prototype fixes the tested Go case by keeping the physical host action +installed while presenting a virtual action to the target, and by removing +`SIGSYS` from the real kernel mask while preserving a target-visible mask. Its +successful esbuild run proves this approach, including direct Go syscalls and +Go's signal initialization. + +Production work remains: + +- make virtual masks per-thread rather than global; +- mediate `rt_sigreturn` and syscalls with temporary signal masks such as + `pselect6`, `ppoll`, and `epoll_pwait`; +- define `sigaltstack`, `signalfd(SIGSYS)`, and explicit `kill(SIGSYS)` behavior; +- run the host handler on the target stack by default, or fully multiplex the + one per-thread alternate-stack slot; +- keep the handler, restorer, syscall gate, and state free of TLS, allocation, + libc locks, and callbacks into the abandoned host runtime; +- use `SA_NODEFER` or an equivalent trusted path for nested traps. + +This is deliberate signal virtualization, not native signal compatibility. + +## Exec transition that should be built + +The safe sequence is: + +1. A target thread calls `execve` or `execveat` and receives `SIGSYS`. +2. The handler resolves the logical target, preserves an executable fd when + needed, and reissues a trusted real `execve` of `fspy_host`. The original + logical argv should be passed as the host argv; target metadata belongs in + a reserved fd or scrubbed environment entry, not as an extra argv prefix. +3. Kernel exec performs normal thread, fd, signal, and address-space cleanup. +4. A minimal static host bootstrap uses a trusted syscall gate to reinstall + the physical handler under the inherited filter. +5. The host parses scripts or ELF, validates and maps the image, constructs the + initial stack, removes private metadata from the logical environment, and + enters the target. + +Steps 3 and the early handler reinstall in step 4 passed in the companion +static-musl bootstrap probe. The target transformation and loader integration +around them still need to be built. + +Passing the original argv to the host matters. A host-shaped Node experiment +made `/proc/self/cmdline` exactly logical and retained `process.argv0` as +`/usr/bin/node`. Node still derived `process.execPath` and `process.argv[0]` +from `/proc/self/exe`, so identity virtualization is separately required. + +The host must not create a background thread before handoff. The residual- +thread probe showed that a pure mapper leaves such a thread alive. This is +avoidable because a freshly kernel-execed host can remain single-threaded. + +## In-house ELF loader requirements + +Start from the behavior of libreflect, not its API or unchecked code: + +- Accept a bounded byte slice or fd and validate ELF magic, class, endian, + machine, ABI, header sizes, table bounds, segment bounds, and all arithmetic. +- Parse shebangs before ELF, including Linux optional-argument rules, + `/usr/bin/env -S`, recursion limits, and `execveat(AT_EMPTY_PATH)` inputs. +- Reserve the complete image span, choose aligned load biases for `ET_DYN`, and + use checked fixed placement for `ET_EXEC`. Reject collisions with retained + host mappings instead of overwriting or silently relocating. +- Map `PT_LOAD` from the target fd where useful, zero partial-page and full BSS + correctly, and apply exact final W^X permissions. +- Handle `PT_INTERP`, `PT_GNU_STACK`, `PT_GNU_RELRO`, large `p_align`, and the + supported static-PIE relocation set. +- Build a properly aligned owned initial stack. Copy argv, environment, + platform, exec filename, and 16 genuine random bytes into it. +- Construct correct `AT_PHDR`, `AT_PHENT`, `AT_PHNUM`, `AT_ENTRY`, `AT_BASE`, + `AT_RANDOM`, `AT_EXECFN`, HWCAP, UID/GID, vDSO, and architecture entries. +- Keep a small position-independent survivor region for the handler, raw + syscall gate, restorer, immutable logical-exec metadata, and mutable state. +- Trap `mmap(MAP_FIXED)`, `mremap`, `munmap`, and `mprotect` operations that + would replace or alter the survivor region. + +Libreflect's observed auxv remains wrong even where programs tolerate it. It +omits `AT_EXECFN`, uses auxiliary-vector bytes as `AT_RANDOM`, and reports a +nonzero `AT_BASE` for a static musl executable. These are required fixes. + +## Identity and kernel-semantic boundary + +The userland target still has the kernel identity of `fspy_host`: + +- `/proc/self/exe`, `/proc//exe`, kernel `PR_GET_AUXV`, audit and ptrace + exec events, and external observers identify the host; +- the address map retains the host/survivor mappings; +- kernel exec applies the host file's credentials and LSM transition, not the + target file's set-user-ID bits, file capabilities, or exec labels. + +For ordinary build tools, mediate self-inspection calls for `/proc/self/exe`, +`/proc/self/auxv`, and `PR_GET_AUXV`, and construct the correct user stack. +The exec handler already knows the logical executable, so it can also treat an +exec of `/proc/self/exe` as a logical self-reexec. This should fix the Node and +Go self-reexec failures seen here. It cannot make external observers or kernel +subsystems see the logical target. + +Declare these cases unsupported on the SIGSYS backend: + +- set-user-ID/set-group-ID and file-capability executables; +- programs that require a target-specific LSM exec transition; +- hostile programs that deliberately bypass the trusted gate or overwrite + retained mappings; +- an ELF whose mandatory fixed mappings collide with the survivor region; +- exact ptrace/audit/procfs executable identity. + +`PR_SET_MM_EXE_FILE` is an optional privileged improvement, not a general +solution; it is unavailable in normal rootless containers and CI jobs. + +## Performance implication + +The companion SIGSYS prototype on the same VM measured approximately: + +| Path | Median | +| -------------------------------------------------- | -------: | +| register-only in-process trap | 0.566 us | +| trap plus trusted syscall reissue | 0.714 us | +| trapped filesystem path with safe copy and reissue | 1.452 us | +| cross-process seccomp user notification | 13.9 us | + +The in-process path is about an order of magnitude faster than +user notification for a representative filesystem trap, while unlike +`LD_PRELOAD` it sees direct syscalls. A minimal preload interposer was within +measurement noise of native dispatch; the SIGSYS backend necessarily costs +more per intercepted syscall. + +Single, unpinned compatibility samples showed native versus libreflect times +of 5 versus 13 ms for the esbuild CLI and 55 versus 68 ms for the Node esbuild +API chain. These include process startup, mapping, and workload time and should +not be treated as stable benchmark numbers. A production benchmark must +compare actual fspy recording work, repeated exec-heavy graphs, and +filesystem-heavy builds on native x86-64 and AArch64 CI machines. + +## Recommendation + +Proceed with an in-house prototype behind a feature flag. The next milestone +should turn the proven AArch64 host bootstrap into one complete path: + +- custom trusted host bootstrap under an inherited filter; +- trapped `execve`/`execveat` transformed to a real exec of the host; +- correct dynamic PIE, static Go, and shebang loading; +- process-wide virtual SIGSYS action plus per-thread virtual signal masks; +- Node `child_process`, Node self-spawn, Go self-reexec, and esbuild API tests; +- explicit failure for credentials, LSM transitions, and mapping collisions. + +Then port the survivor/gate ABI to x86-64 and run the same matrix in Docker, +WSL2, GitHub Actions, and Kubernetes. Keep the current backend as a fallback +until that matrix and workload benchmarks pass. diff --git a/research/userland-exec-compat/aarch64-lima-summary.tsv b/research/userland-exec-compat/aarch64-lima-summary.tsv new file mode 100644 index 000000000..afcca74df --- /dev/null +++ b/research/userland-exec-compat/aarch64-lima-summary.tsv @@ -0,0 +1,42 @@ +case exit duration_ms +native-c-probe 0 1 +native-glibc-nopie 0 1 +native-musl-static 0 2 +native-node 0 116 +native-static-go 0 2 +native-shebang 0 1 +native-coreutils 0 1 +native-esbuild-cli 0 5 +native-esbuild-api 0 55 +libreflect-c-probe 0 2 +libreflect-glibc-nopie 0 2 +libreflect-musl-static 0 1 +libreflect-node 0 76 +libreflect-host-shaped-node 0 69 +libreflect-shell 0 1 +libreflect-coreutils 0 1 +libreflect-static-go 0 3 +libreflect-esbuild-cli 0 13 +libreflect-esbuild-api 0 68 +libreflect-direct-shebang 134 62 +libreflect-expanded-shebang 0 3 +libreflect-state 0 3 +libreflect-residual-thread 0 2 +anvil-c-probe 0 32 +anvil-glibc-nopie 0 26 +anvil-musl-static 0 23 +anvil-node 0 91 +anvil-shell 0 23 +anvil-coreutils 0 23 +anvil-static-go 139 69 +anvil-esbuild-cli 139 77 +anvil-direct-shebang 1 53 +sigsys-libreflect 0 2 +sigsys-anvil 0 77 +sigsys-libreflect-c 0 1 +sigsys-libreflect-node 91 49 +sigsys-libreflect-static-go 2 5 +sigsys-libreflect-esbuild 0 12 +sigsys-full-libreflect-esbuild 0 11 +sigsys-full-libreflect-node 0 66 +sigsys-anvil-static-go 139 121 diff --git a/research/userland-exec-compat/compat_probe.c b/research/userland-exec-compat/compat_probe.c new file mode 100644 index 000000000..e36c542f9 --- /dev/null +++ b/research/userland-exec-compat/compat_probe.c @@ -0,0 +1,179 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef PR_GET_AUXV +#define PR_GET_AUXV 0x41555856 +#endif + +extern char **environ; + +static __thread int tls_value; + +static void print_file(const char *label, const char *path, bool replace_nuls) +{ + char buffer[4096]; + ssize_t length; + int fd = open(path, O_RDONLY); + if (fd < 0) { + printf("%s=\n", label, strerror(errno)); + return; + } + length = read(fd, buffer, sizeof(buffer) - 1); + close(fd); + if (length < 0) { + printf("%s=\n", label, strerror(errno)); + return; + } + for (ssize_t i = 0; replace_nuls && i < length; i++) + if (buffer[i] == '\0') + buffer[i] = '|'; + while (length > 0 && (buffer[length - 1] == '\n' || buffer[length - 1] == '\0')) + length--; + buffer[length] = '\0'; + printf("%s=%s\n", label, buffer); +} + +static int task_count(void) +{ + DIR *directory = opendir("/proc/self/task"); + struct dirent *entry; + int count = 0; + if (!directory) + return -1; + while ((entry = readdir(directory))) + if (entry->d_name[0] != '.') + count++; + closedir(directory); + return count; +} + +static void *thread_worker(void *argument) +{ + long value = (long)argument; + int fd; + tls_value = (int)value; + fd = open("/etc/hostname", O_RDONLY); + if (fd >= 0) + close(fd); + return (void *)(long)tls_value; +} + +static void test_threads(void) +{ + pthread_t threads[4]; + long sum = 0; + for (long i = 0; i < 4; i++) + if (pthread_create(&threads[i], NULL, thread_worker, (void *)(i + 1)) != 0) { + printf("threads=creation-failed\n"); + return; + } + for (int i = 0; i < 4; i++) { + void *result = NULL; + pthread_join(threads[i], &result); + sum += (long)result; + } + printf("threads=ok tls_sum=%ld tasks_after=%d\n", sum, task_count()); +} + +static void test_subprocess(void) +{ + char *child_argv[] = {"echo", "probe-child", NULL}; + pid_t child; + int status; + int error = posix_spawn(&child, "/bin/echo", NULL, NULL, child_argv, environ); + if (error != 0) { + printf("subprocess=spawn-error:%s\n", strerror(error)); + return; + } + if (waitpid(child, &status, 0) < 0) { + printf("subprocess=wait-error:%s\n", strerror(errno)); + return; + } + printf("subprocess=exit:%d\n", WIFEXITED(status) ? WEXITSTATUS(status) : -1); +} + +static void print_auxv(void) +{ + unsigned long execfn = getauxval(AT_EXECFN); + unsigned long base = getauxval(AT_BASE); + Elf64_auxv_t kernel_auxv[64]; + long copied = prctl(PR_GET_AUXV, kernel_auxv, sizeof(kernel_auxv), 0, 0); + unsigned long kernel_execfn = 0; + if (copied >= 0) + for (size_t i = 0; i < sizeof(kernel_auxv) / sizeof(kernel_auxv[0]); i++) { + if (kernel_auxv[i].a_type == AT_EXECFN) + kernel_execfn = kernel_auxv[i].a_un.a_val; + if (kernel_auxv[i].a_type == AT_NULL) + break; + } + printf("auxv_execfn=%s\n", execfn ? (char *)execfn : ""); + printf("auxv_base=0x%lx\n", base); + printf("kernel_auxv=%s kernel_execfn=%s\n", copied < 0 ? strerror(errno) : "ok", + kernel_execfn ? (char *)kernel_execfn : ""); +} + +static void print_signal_state(void) +{ + struct sigaction usr1 = {0}; + struct sigaction sys = {0}; + stack_t stack = {0}; + sigaction(SIGUSR1, NULL, &usr1); + sigaction(SIGSYS, NULL, &sys); + sigaltstack(NULL, &stack); + printf("signals=usr1_%s sigsys_%s altstack_%s\n", + usr1.sa_handler == SIG_DFL ? "default" : usr1.sa_handler == SIG_IGN ? "ignored" : "caught", + sys.sa_handler == SIG_DFL ? "default" : sys.sa_handler == SIG_IGN ? "ignored" : "caught", + (stack.ss_flags & SS_DISABLE) ? "disabled" : "enabled"); +} + +int main(int argc, char **argv) +{ + char executable[PATH_MAX]; + ssize_t executable_length = readlink("/proc/self/exe", executable, sizeof(executable) - 1); + const char *cloexec_text = getenv("PROBE_CLOEXEC_FD"); + long raw_pid = syscall(SYS_getpid); + + if (executable_length >= 0) + executable[executable_length] = '\0'; + else + strcpy(executable, ""); + + printf("probe=compat-v1 pid=%ld libc_pid=%ld ppid=%ld tasks_before=%d\n", + raw_pid, (long)getpid(), (long)getppid(), task_count()); + printf("proc_exe=%s\n", executable); + print_file("proc_cmdline", "/proc/self/cmdline", true); + print_file("proc_comm", "/proc/self/comm", false); + printf("argc=%d", argc); + for (int i = 0; i < argc; i++) + printf(" argv%d=%s", i, argv[i]); + printf("\n"); + print_auxv(); + print_signal_state(); + if (cloexec_text) { + int fd = atoi(cloexec_text); + printf("cloexec_fd=%d state=%s\n", fd, + fcntl(fd, F_GETFD) < 0 && errno == EBADF ? "closed" : "open"); + } else { + printf("cloexec_fd=not-provided\n"); + } + test_threads(); + test_subprocess(); + return 0; +} diff --git a/research/userland-exec-compat/esbuild_api_probe.js b/research/userland-exec-compat/esbuild_api_probe.js new file mode 100644 index 000000000..b2ccfe1b2 --- /dev/null +++ b/research/userland-exec-compat/esbuild_api_probe.js @@ -0,0 +1,40 @@ +'use strict'; + +const fs = require('node:fs'); + +const [esbuildModule, physicalWrapper, logicalBinary, entryPoint] = process.argv.slice(2); +process.env.ESBUILD_BINARY_PATH = physicalWrapper; +process.env.LIBREFLECT_TARGET = logicalBinary; + +const esbuild = require(esbuildModule); + +async function main() { + const built = await esbuild.build({ + bundle: true, + entryPoints: [entryPoint], + format: 'esm', + minify: true, + sourcemap: 'inline', + write: false, + }); + const transformed = await esbuild.transform('const answer: number = 6 * 7', { + loader: 'ts', + minify: true, + }); + console.log( + JSON.stringify({ + esbuild: esbuild.version, + outputFiles: built.outputFiles.map((file) => ({ + path: file.path, + bytes: file.contents.length, + })), + transformed: transformed.code.trim(), + entryBytes: fs.statSync(entryPoint).size, + }), + ); +} + +main().catch((error) => { + console.error(error.stack || error); + process.exitCode = 1; +}); diff --git a/research/userland-exec-compat/esbuild_entry.ts b/research/userland-exec-compat/esbuild_entry.ts new file mode 100644 index 000000000..afac83cf2 --- /dev/null +++ b/research/userland-exec-compat/esbuild_entry.ts @@ -0,0 +1,4 @@ +import { label, square } from './esbuild_math'; + +const values: number[] = Array.from({ length: 128 }, (_, index) => square(index)); +console.log(label, values.at(-1), globalThis?.Object?.keys({ traced: true }).length); diff --git a/research/userland-exec-compat/esbuild_math.ts b/research/userland-exec-compat/esbuild_math.ts new file mode 100644 index 000000000..c3ae1c12e --- /dev/null +++ b/research/userland-exec-compat/esbuild_math.ts @@ -0,0 +1,2 @@ +export const label: string = 'esbuild-compat-ok'; +export const square = (value: number): number => value * value; diff --git a/research/userland-exec-compat/libreflect_runner.c b/research/userland-exec-compat/libreflect_runner.c new file mode 100644 index 000000000..6bc5eb4b7 --- /dev/null +++ b/research/userland-exec-compat/libreflect_runner.c @@ -0,0 +1,115 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +extern char **environ; + +static void inherited_handler(int signal_number) +{ + (void)signal_number; +} + +static void *background_thread(void *unused) +{ + (void)unused; + for (;;) + pause(); + return NULL; +} + +static void prepare_compatibility_state(void) +{ + if (getenv("RUNNER_CLOEXEC")) { + char fd_text[32]; + int fd = open("/dev/null", O_RDONLY | O_CLOEXEC); + if (fd < 0) { + perror("open(O_CLOEXEC)"); + exit(70); + } + snprintf(fd_text, sizeof(fd_text), "%d", fd); + setenv("PROBE_CLOEXEC_FD", fd_text, 1); + } + + if (getenv("RUNNER_SIGUSR1")) { + struct sigaction action = {0}; + stack_t stack = {0}; + void *memory = mmap(NULL, 1024 * 1024, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (memory == MAP_FAILED) { + perror("mmap(signal stack)"); + exit(71); + } + stack.ss_sp = memory; + stack.ss_size = 1024 * 1024; + if (sigaltstack(&stack, NULL) != 0) { + perror("sigaltstack"); + exit(72); + } + action.sa_handler = inherited_handler; + sigemptyset(&action.sa_mask); + if (sigaction(SIGUSR1, &action, NULL) != 0) { + perror("sigaction(SIGUSR1)"); + exit(73); + } + } + + if (getenv("RUNNER_BACKGROUND_THREAD")) { + pthread_t thread; + if (pthread_create(&thread, NULL, background_thread, NULL) != 0) { + perror("pthread_create"); + exit(74); + } + pthread_detach(thread); + } +} + +int main(int argc, char **argv) +{ + struct stat status; + unsigned char *elf; + char **target_argv; + const char *target; + int fd; + + target = getenv("LIBREFLECT_TARGET"); + if (target) { + /* Model a transformed child exec: the kernel starts this host, while + * metadata tells it which logical executable to map. */ + target_argv = argv; + target_argv[0] = (char *)target; + unsetenv("LIBREFLECT_TARGET"); + } else if (argc >= 2) { + target = argv[1]; + target_argv = argv + 1; + } else { + fprintf(stderr, "usage: %s TARGET [ARG ...]\n", argv[0]); + return 64; + } + + fd = open(target, O_RDONLY); + if (fd < 0 || fstat(fd, &status) != 0) { + fprintf(stderr, "open target %s: %s\n", target, strerror(errno)); + return 65; + } + elf = mmap(NULL, status.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + if (elf == MAP_FAILED) { + perror("mmap target"); + return 66; + } + + prepare_compatibility_state(); + reflect_execve(elf, target_argv, environ); + return 67; +} diff --git a/research/userland-exec-compat/node_probe.js b/research/userland-exec-compat/node_probe.js new file mode 100644 index 000000000..d34be8c9c --- /dev/null +++ b/research/userland-exec-compat/node_probe.js @@ -0,0 +1,67 @@ +'use strict'; + +const fs = require('node:fs'); +const childProcess = require('node:child_process'); +const { Worker } = require('node:worker_threads'); + +function visibleFile(path, nulReplacement = '') { + try { + return fs.readFileSync(path, 'utf8').replaceAll('\0', nulReplacement).trim(); + } catch (error) { + return ``; + } +} + +async function main() { + const worker = await new Promise((resolve) => { + const instance = new Worker( + `const { parentPort } = require('node:worker_threads'); parentPort.postMessage(6 * 7)`, + { eval: true }, + ); + instance.once('message', (value) => resolve({ value })); + instance.once('error', (error) => resolve({ error: error.message })); + }); + + const shell = childProcess.spawnSync('/bin/sh', ['-c', 'printf node-shell-child'], { + encoding: 'utf8', + }); + const self = childProcess.spawnSync( + process.execPath, + ['-e', 'process.stdout.write("node-self-child")'], + { encoding: 'utf8', timeout: 3000 }, + ); + + console.log( + JSON.stringify({ + node: 'compat-v1', + pid: process.pid, + ppid: process.ppid, + argv: process.argv, + argv0: process.argv0, + execPath: process.execPath, + procExe: fs.readlinkSync('/proc/self/exe'), + procCmdline: visibleFile('/proc/self/cmdline', '|'), + procComm: visibleFile('/proc/self/comm'), + hostnameBytes: fs.readFileSync('/etc/hostname').length, + worker, + shell: { + status: shell.status, + signal: shell.signal, + stdout: shell.stdout, + stderr: shell.stderr, + }, + self: { + status: self.status, + signal: self.signal, + error: self.error && self.error.message, + stdout: self.stdout, + stderr: self.stderr, + }, + }), + ); +} + +main().catch((error) => { + console.error(error.stack || error); + process.exitCode = 1; +}); diff --git a/research/userland-exec-compat/run-study.sh b/research/userland-exec-compat/run-study.sh new file mode 100755 index 000000000..3de1982e1 --- /dev/null +++ b/research/userland-exec-compat/run-study.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +set -euo pipefail + +CLONES=${1:?usage: run-study.sh CLONE_ROOT [OUTPUT_ROOT] [TRAP_PRELOAD_SOURCE]} +OUTPUT_ROOT=${2:-${TMPDIR:-/tmp}/fspy-userland-exec-compat-results} +SOURCE_ROOT=$(cd "$(dirname "$0")" && pwd) +TRAP_PRELOAD_SOURCE=${3:-"$SOURCE_ROOT/../sigsys-prototype/trap_preload.c"} +WORK_ROOT=${FSPY_USERLAND_WORK_ROOT:-${TMPDIR:-/tmp}/fspy-userland-exec-compat-work} +BUILD_ROOT="$WORK_ROOT/build" +SOURCE_COPY="$WORK_ROOT/source" +RESULTS="$OUTPUT_ROOT/raw" +ESBUILD_ROOT="$WORK_ROOT/esbuild" + +mkdir -p "$BUILD_ROOT" "$SOURCE_COPY" "$RESULTS" \ + "$ESBUILD_ROOT/platform" "$ESBUILD_ROOT/js" +rm -f "$OUTPUT_ROOT/summary.tsv" "$OUTPUT_ROOT/environment.txt" +cp -a "$SOURCE_ROOT/." "$SOURCE_COPY" + +{ + uname -a + sed -n '1,12p' /etc/os-release + gcc --version | head -1 + python3 --version + node --version + go version +} >"$OUTPUT_ROOT/environment.txt" 2>&1 + +cp -a "$CLONES/mettle/libreflect/." "$BUILD_ROOT/libreflect" +# Build the AArch64 assembly path directly. libreflect's generated configure +# tests an absolute-looking header name as a system include and incorrectly +# selects its memfd_create/execveat fallback here. The fallback would perform +# a real kernel exec and invalidate this userland-exec experiment. +sed 's/@HAVE_ASM@/1/' "$BUILD_ROOT/libreflect/include/reflect.h.in" \ + >"$BUILD_ROOT/libreflect/include/reflect.h" + +gcc -O2 -g -Wall -Wextra -pthread \ + -I"$BUILD_ROOT/libreflect/include" \ + -I"$BUILD_ROOT/libreflect/src" \ + -I"$BUILD_ROOT/libreflect/arch/linux/aarch64" \ + "$SOURCE_COPY/libreflect_runner.c" \ + "$BUILD_ROOT/libreflect/src/map_elf.c" \ + "$BUILD_ROOT/libreflect/src/stack_setup.c" \ + "$BUILD_ROOT/libreflect/src/jump.c" \ + "$BUILD_ROOT/libreflect/src/exec.c" \ + -o "$BUILD_ROOT/libreflect-runner" \ + >"$OUTPUT_ROOT/build-libreflect.log" 2>&1 +gcc -O2 -g -Wall -Wextra -pthread "$SOURCE_COPY/compat_probe.c" \ + -o "$BUILD_ROOT/compat-probe" +gcc -O2 -g -Wall -Wextra -pthread -no-pie "$SOURCE_COPY/compat_probe.c" \ + -o "$BUILD_ROOT/glibc-nopie-probe" +musl-gcc -O2 -g -Wall -Wextra -pthread -static \ + "$SOURCE_COPY/compat_probe.c" -o "$BUILD_ROOT/musl-static-probe" +CGO_ENABLED=0 go build -trimpath -o "$BUILD_ROOT/static-go-probe" \ + "$SOURCE_COPY/static_probe.go" +gcc -O2 -g -Wall -Wextra -fPIC -shared "$CLONES/sigsys-test/sigsys_preload.c" \ + -o "$BUILD_ROOT/libsigsys-preload.so" +gcc -O2 -g -Wall -Wextra "$CLONES/sigsys-test/sigsys_target.c" \ + -o "$BUILD_ROOT/sigsys-target" +gcc -O2 -Wall -Wextra -Werror -shared -fPIC "$TRAP_PRELOAD_SOURCE" \ + -o "$BUILD_ROOT/libtrap-preload.so" +chmod +x "$SOURCE_COPY/shebang_probe.sh" + +# Pin a real, statically linked frontend tool and its JavaScript API. The API +# case exercises Node -> transformed host -> userland-loaded esbuild service. +curl -fsSL \ + https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz \ + -o "$ESBUILD_ROOT/platform.tgz" +tar -xzf "$ESBUILD_ROOT/platform.tgz" -C "$ESBUILD_ROOT/platform" \ + --strip-components=1 +curl -fsSL https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz \ + -o "$ESBUILD_ROOT/js.tgz" +tar -xzf "$ESBUILD_ROOT/js.tgz" -C "$ESBUILD_ROOT/js" \ + --strip-components=1 +ESBUILD_BINARY="$ESBUILD_ROOT/platform/bin/esbuild" +ESBUILD_MODULE="$ESBUILD_ROOT/js" + +ANVIL=(python3 "$CLONES/ulexecve/ulexecve.py") +LIBREFLECT=("$BUILD_ROOT/libreflect-runner") + +printf 'case\texit\tduration_ms\n' >"$OUTPUT_ROOT/summary.tsv" + +run_case() { + local name=$1 + shift + local started ended status + started=$(date +%s%3N) + set +e + timeout --signal=KILL 25s "$@" >"$RESULTS/$name.stdout" 2>"$RESULTS/$name.stderr" + status=$? + set -e + ended=$(date +%s%3N) + printf '%s\t%s\t%s\n' "$name" "$status" "$((ended - started))" >>"$OUTPUT_ROOT/summary.tsv" +} + +# Native controls. +run_case native-c-probe "$BUILD_ROOT/compat-probe" alpha beta +run_case native-glibc-nopie "$BUILD_ROOT/glibc-nopie-probe" alpha +run_case native-musl-static "$BUILD_ROOT/musl-static-probe" alpha +run_case native-node node "$SOURCE_COPY/node_probe.js" alpha +run_case native-static-go "$BUILD_ROOT/static-go-probe" alpha +run_case native-shebang "$SOURCE_COPY/shebang_probe.sh" alpha +run_case native-coreutils /bin/echo coreutils-ok +run_case native-esbuild-cli "$ESBUILD_BINARY" "$SOURCE_COPY/esbuild_entry.ts" \ + --bundle --sourcemap --outfile="$BUILD_ROOT/native-esbuild.js" +run_case native-esbuild-api node "$SOURCE_COPY/esbuild_api_probe.js" \ + "$ESBUILD_MODULE" "$ESBUILD_BINARY" "$ESBUILD_BINARY" \ + "$SOURCE_COPY/esbuild_entry.ts" + +# libreflect: best embeddable dynamic-PIE reference. +run_case libreflect-c-probe "${LIBREFLECT[@]}" "$BUILD_ROOT/compat-probe" alpha beta +run_case libreflect-glibc-nopie "${LIBREFLECT[@]}" \ + "$BUILD_ROOT/glibc-nopie-probe" alpha +run_case libreflect-musl-static "${LIBREFLECT[@]}" \ + "$BUILD_ROOT/musl-static-probe" alpha +run_case libreflect-node "${LIBREFLECT[@]}" /usr/bin/node "$SOURCE_COPY/node_probe.js" alpha +# $1 and $2 belong to the nested bash command. +# shellcheck disable=SC2016 +run_case libreflect-host-shaped-node env LIBREFLECT_TARGET=/usr/bin/node \ + bash -c 'exec -a /usr/bin/node "$1" "$2" alpha' bash \ + "$BUILD_ROOT/libreflect-runner" "$SOURCE_COPY/node_probe.js" +run_case libreflect-shell "${LIBREFLECT[@]}" /bin/sh -c 'printf shell-ok; /bin/echo shell-child' +run_case libreflect-coreutils "${LIBREFLECT[@]}" /bin/echo coreutils-ok +run_case libreflect-static-go "${LIBREFLECT[@]}" "$BUILD_ROOT/static-go-probe" alpha +run_case libreflect-esbuild-cli "${LIBREFLECT[@]}" "$ESBUILD_BINARY" \ + "$SOURCE_COPY/esbuild_entry.ts" --bundle --sourcemap \ + --outfile="$BUILD_ROOT/libreflect-esbuild.js" +run_case libreflect-esbuild-api "${LIBREFLECT[@]}" /usr/bin/node \ + "$SOURCE_COPY/esbuild_api_probe.js" "$ESBUILD_MODULE" \ + "$BUILD_ROOT/libreflect-runner" "$ESBUILD_BINARY" \ + "$SOURCE_COPY/esbuild_entry.ts" +run_case libreflect-direct-shebang "${LIBREFLECT[@]}" "$SOURCE_COPY/shebang_probe.sh" alpha +run_case libreflect-expanded-shebang "${LIBREFLECT[@]}" /bin/sh \ + "$SOURCE_COPY/shebang_probe.sh" alpha +run_case libreflect-state env RUNNER_CLOEXEC=1 RUNNER_SIGUSR1=1 \ + "${LIBREFLECT[@]}" "$BUILD_ROOT/compat-probe" state +run_case libreflect-residual-thread env RUNNER_BACKGROUND_THREAD=1 \ + "${LIBREFLECT[@]}" "$BUILD_ROOT/compat-probe" threaded + +# Anvil: broadest pure-loader reference across ELF forms. +run_case anvil-c-probe "${ANVIL[@]}" "$BUILD_ROOT/compat-probe" alpha beta +run_case anvil-glibc-nopie "${ANVIL[@]}" \ + "$BUILD_ROOT/glibc-nopie-probe" alpha +run_case anvil-musl-static "${ANVIL[@]}" "$BUILD_ROOT/musl-static-probe" alpha +run_case anvil-node "${ANVIL[@]}" /usr/bin/node "$SOURCE_COPY/node_probe.js" alpha +run_case anvil-shell "${ANVIL[@]}" /bin/sh -c 'printf shell-ok; /bin/echo shell-child' +run_case anvil-coreutils "${ANVIL[@]}" /bin/echo coreutils-ok +run_case anvil-static-go "${ANVIL[@]}" "$BUILD_ROOT/static-go-probe" alpha +run_case anvil-esbuild-cli "${ANVIL[@]}" "$ESBUILD_BINARY" \ + "$SOURCE_COPY/esbuild_entry.ts" --bundle --sourcemap \ + --outfile="$BUILD_ROOT/anvil-esbuild.js" +run_case anvil-direct-shebang "${ANVIL[@]}" "$SOURCE_COPY/shebang_probe.sh" alpha + +# Preserved SIGSYS/alternate-stack checks. +run_case sigsys-libreflect env LD_PRELOAD="$BUILD_ROOT/libsigsys-preload.so" \ + "${LIBREFLECT[@]}" "$BUILD_ROOT/sigsys-target" +run_case sigsys-anvil env LD_PRELOAD="$BUILD_ROOT/libsigsys-preload.so" \ + "${ANVIL[@]}" "$BUILD_ROOT/sigsys-target" +run_case sigsys-libreflect-c env LD_PRELOAD="$BUILD_ROOT/libsigsys-preload.so" \ + "${LIBREFLECT[@]}" "$BUILD_ROOT/compat-probe" sigsys +run_case sigsys-libreflect-node env LD_PRELOAD="$BUILD_ROOT/libsigsys-preload.so" \ + "${LIBREFLECT[@]}" /usr/bin/node "$SOURCE_COPY/node_probe.js" sigsys +run_case sigsys-libreflect-static-go env LD_PRELOAD="$BUILD_ROOT/libsigsys-preload.so" \ + "${LIBREFLECT[@]}" "$BUILD_ROOT/static-go-probe" sigsys +run_case sigsys-libreflect-esbuild env LD_PRELOAD="$BUILD_ROOT/libsigsys-preload.so" \ + "${LIBREFLECT[@]}" "$ESBUILD_BINARY" "$SOURCE_COPY/esbuild_entry.ts" \ + --bundle --outfile="$BUILD_ROOT/sigsys-esbuild.js" +run_case sigsys-full-libreflect-esbuild env \ + LD_PRELOAD="$BUILD_ROOT/libtrap-preload.so" \ + "${LIBREFLECT[@]}" "$ESBUILD_BINARY" "$SOURCE_COPY/esbuild_entry.ts" \ + --bundle --minify --outfile="$BUILD_ROOT/sigsys-full-esbuild.js" +run_case sigsys-full-libreflect-node env \ + LD_PRELOAD="$BUILD_ROOT/libtrap-preload.so" \ + "${LIBREFLECT[@]}" /usr/bin/node "$SOURCE_COPY/node_probe.js" fulltrap +run_case sigsys-anvil-static-go env LD_PRELOAD="$BUILD_ROOT/libsigsys-preload.so" \ + "${ANVIL[@]}" "$BUILD_ROOT/static-go-probe" sigsys + +printf '%s\n' "$OUTPUT_ROOT" diff --git a/research/userland-exec-compat/shebang_probe.sh b/research/userland-exec-compat/shebang_probe.sh new file mode 100644 index 000000000..6ee85b406 --- /dev/null +++ b/research/userland-exec-compat/shebang_probe.sh @@ -0,0 +1,3 @@ +#!/bin/sh +printf 'shebang=compat-v1 argv0=%s arg1=%s exe=%s\n' "$0" "${1-}" "$(readlink /proc/self/exe)" +/bin/echo shebang-child diff --git a/research/userland-exec-compat/static_probe.go b/research/userland-exec-compat/static_probe.go new file mode 100644 index 000000000..c2bd2a5fb --- /dev/null +++ b/research/userland-exec-compat/static_probe.go @@ -0,0 +1,42 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "runtime" + "sync" + "time" +) + +func main() { + if len(os.Args) > 1 && os.Args[1] == "--self-child" { + fmt.Println("go-self-child") + return + } + + executable, executableErr := os.Executable() + data, readErr := os.ReadFile("/etc/hostname") + var workers sync.WaitGroup + workers.Add(8) + for range 8 { + go func() { + defer workers.Done() + for i := 0; i < 1000; i++ { + _ = i * i + } + }() + } + workers.Wait() + + childOutput, childErr := exec.Command("/bin/echo", "go-child").CombinedOutput() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + selfOutput, selfErr := exec.CommandContext(ctx, executable, "--self-child").CombinedOutput() + + fmt.Printf("go=compat-v1 pid=%d args=%q executable=%q executable_err=%v goroutines=%d hostname_bytes=%d read_err=%v\n", + os.Getpid(), os.Args, executable, executableErr, runtime.NumGoroutine(), len(data), readErr) + fmt.Printf("go_child=%q error=%v\n", childOutput, childErr) + fmt.Printf("go_self_reexec=%q error=%v\n", selfOutput, selfErr) +}