Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 27 additions & 2 deletions packages/bcode-browser/src/browser-execute.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import fs from "fs/promises"
import path from "path"
import { Effect, Schema } from "effect"
import type { Page } from "./cdp/generated"
import { SessionStore } from "./session-store"
import { Skills } from "./skills"

Expand Down Expand Up @@ -184,6 +185,30 @@ export const make = Effect.fn("BrowserExecute.make")(function* (dataDir: string)
debug: tee,
})

// BrowserCode extension to CDP's screenshot params. The wrapper strips
// this field before the command reaches Chrome; false keeps the returned
// base64 available to the snippet without attaching it to model context.
type ScreenshotParams = Page.CaptureScreenshotParams & {
readonly attachToContext?: boolean
}
const localOnlyScreenshotParams = new WeakSet<object>()
const page = Object.assign(Object.create(session.domains.Page), {
captureScreenshot: (params: ScreenshotParams = {}) => {
const { attachToContext, ...cdpParams } = params
if (attachToContext === false) localOnlyScreenshotParams.add(cdpParams)
return session.domains.Page.captureScreenshot(cdpParams)
},
})
const domains = Object.assign(Object.create(session.domains), { Page: page })
const snippetSession = new Proxy(session, {
get(target, property) {
if (property === "Page") return page
if (property === "domains") return domains
const value = Reflect.get(target, property, target)
return typeof value === "function" ? value.bind(target) : value
},
})

// Screenshot tap. Subscribes to the Session's call-result stream for
// the duration of this execute() call; every successful
// `Page.captureScreenshot` is collected (drained into `attachments[]`
Expand All @@ -209,7 +234,7 @@ export const make = Effect.fn("BrowserExecute.make")(function* (dataDir: string)
const mime = screenshotMime(p.format)
const ext = screenshotExt(p.format)
const idx = seq++
screenshots.push({ mime, base64: r.data })
if (!localOnlyScreenshotParams.has(p)) screenshots.push({ mime, base64: r.data })
if (dumpDir) {
const filename = `${ctx.sessionID}-${startedAt}-${String(idx).padStart(3, "0")}.${ext}`
fs.mkdir(dumpDir, { recursive: true })
Expand All @@ -219,7 +244,7 @@ export const make = Effect.fn("BrowserExecute.make")(function* (dataDir: string)
})

const ran = yield* Effect.tryPromise({
try: () => wrapped(session, snippetConsole),
try: () => wrapped(snippetSession, snippetConsole),
catch: (err) => new Error(`browser_execute snippet threw: ${err instanceof Error ? err.stack ?? err.message : String(err)}`),
}).pipe(Effect.ensuring(Effect.sync(() => unsubscribe())))

Expand Down
17 changes: 11 additions & 6 deletions packages/bcode-browser/test/browser-execute.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -117,21 +117,22 @@ test.skipIf(!enabled)("workspace import inside a snippet", async () => {
expect(JSON.parse(result.result)).toBe("bcode-be")
})

test.skipIf(!enabled)("Page.captureScreenshot is collected into result.screenshots", async () => {
test.skipIf(!enabled)("Page.captureScreenshot can stay out of model context", async () => {
const result = await Effect.runPromise(
Effect.scoped(
Effect.gen(function* () {
const impl = yield* BrowserExecute.make(dataDir)
return yield* impl.execute(
{
description: "Capture two screenshots",
description: "Capture context screenshots",
code: `await session.Page.enable();
const loaded = session.waitFor("Page.loadEventFired", { timeoutMs: 5000 });
await session.Page.navigate({ url: "data:text/html,<title>shot</title><body>hi" });
await loaded;
const a = await session.Page.captureScreenshot({ format: "png" });
const b = await session.Page.captureScreenshot({ format: "jpeg", quality: 50 });
return { aLen: a.data.length, bLen: b.data.length };`,
const local = await session.Page.captureScreenshot({ format: "webp", attachToContext: false });
return { aLen: a.data.length, bLen: b.data.length, localLen: local.data.length };`,
},
{ sessionID, workspaceDir },
)
Expand All @@ -141,7 +142,10 @@ test.skipIf(!enabled)("Page.captureScreenshot is collected into result.screensho
expect(result.screenshots).toHaveLength(2)
expect(result.screenshots[0]!.mime).toBe("image/png")
expect(result.screenshots[1]!.mime).toBe("image/jpeg")
// base64 must round-trip back to non-empty bytes for both shots.
// The local-only screenshot still returned data to the snippet but was not
// collected into model-context attachments.
expect(JSON.parse(result.result).localLen).toBeGreaterThan(0)
// Attached base64 must round-trip back to non-empty bytes for both shots.
expect(Buffer.from(result.screenshots[0]!.base64, "base64").length).toBeGreaterThan(0)
expect(Buffer.from(result.screenshots[1]!.base64, "base64").length).toBeGreaterThan(0)
})
Expand All @@ -151,20 +155,21 @@ test.skipIf(!enabled)("BCODE_SCREENSHOT_DIR dumps screenshots to disk", async ()
const prev = process.env.BCODE_SCREENSHOT_DIR
process.env.BCODE_SCREENSHOT_DIR = dump
try {
await Effect.runPromise(
const result = await Effect.runPromise(
Effect.scoped(
Effect.gen(function* () {
const impl = yield* BrowserExecute.make(dataDir)
return yield* impl.execute(
{
description: "Dump screenshot to disk",
code: `await session.Page.captureScreenshot({ format: "png" });`,
code: `await session.Page.captureScreenshot({ format: "png", attachToContext: false });`,
},
{ sessionID, workspaceDir },
)
}),
),
)
expect(result.screenshots).toHaveLength(0)
// Disk dump is fire-and-forget; give it a tick to land.
await new Promise((r) => setTimeout(r, 150))
const files = await fs.readdir(dump)
Expand Down
2 changes: 1 addition & 1 deletion packages/opencode/src/tool/browser-execute.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ Usage:
- Use this tool whenever the task requires driving a real browser.
- Use this tool to read webpages that block the webfetch tool.
- IMPORTANT: you MUST use the skill tool first to load the `browser-execute` skill. This tool will fail if you did not read those directions first.
- Returns console output from the snippet; screenshots taken attach automatically as images.
- Screenshots attach automatically; `Page.captureScreenshot({ attachToContext: false })` returns one only to the snippet and does not save it.
Loading