Pi Web prototype: macOS computer_use/UI tree and current-run edited-files widget

index.ts

import { execFile as execFileCallback } from "node:child_process";
import { readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { promisify } from "node:util";
import { randomUUID } from "node:crypto";
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
import { StringEnum } from "@earendil-works/pi-ai";
import { Type } from "typebox";

const execFile = promisify(execFileCallback);

function required(value: string | undefined, name: string): string {
	if (!value?.trim()) throw new Error(`${name} is required for this action`);
	return value.trim();
}

export default function interactiveTools(pi: ExtensionAPI) {
	const editedFiles = new Set<string>();

	pi.on("agent_start", (_event, ctx) => {
		editedFiles.clear();
		ctx.ui.setWidget("turn-edited-files", undefined);
	});

	pi.on("tool_result", (event, ctx) => {
		if (event.isError || (event.toolName !== "edit" && event.toolName !== "write")) return;
		const path = (event.input as { path?: unknown }).path;
		if (typeof path !== "string" || !path.trim()) return;
		editedFiles.add(path);
		ctx.ui.setWidget(
			"turn-edited-files",
			["本轮编辑文件", ...Array.from(editedFiles, (file) => `• ${file}`)],
			{ placement: "aboveEditor" },
		);
	});

	async function runAppleScript(source: string, args: string[] = []): Promise<string> {
		const { stdout } = await execFile("/usr/bin/osascript", ["-e", source, ...args], { timeout: 30_000 });
		return stdout.trim();
	}

	async function readAccessibilityTree(app: string | undefined, maxDepth: number, maxNodes: number, visibleOnly: boolean): Promise<string> {
		const source = String.raw`
function safe(fn, fallback = null) {
  try {
    const value = fn();
    return value === undefined ? fallback : value;
  } catch (_) {
    return fallback;
  }
}
function scalar(value) {
  try {
    if (value === null || value === undefined) return null;
    if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
    if (Array.isArray(value)) return value.map(scalar);
    return String(value);
  } catch (_) {
    return null;
  }
}
function run(argv) {
  const requestedApp = argv[0] || "";
  const maxDepth = Number(argv[1]);
  const maxNodes = Number(argv[2]);
  const visibleOnly = argv[3] !== "false";
  const systemEvents = Application("System Events");
  const processes = systemEvents.applicationProcesses();
  let process;
  if (requestedApp) {
    process = processes.find(p => safe(() => p.name()) === requestedApp)
      || processes.find(p => String(safe(() => p.name(), "")).toLowerCase() === requestedApp.toLowerCase() && safe(() => p.windows.length, 0) > 0);
  } else {
    process = processes.find(p => safe(() => p.frontmost(), false));
  }
  if (!process) throw new Error("Application process not found: " + (requestedApp || "frontmost"));
  const windows = safe(() => process.windows(), []);
  const root = windows[0] || process;
  const rootProperties = safe(() => root.properties(), {});
  const viewportPosition = rootProperties.position || [0, 0];
  const viewportSize = rootProperties.size || [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER];
  const viewport = {
    left: viewportPosition[0],
    top: viewportPosition[1],
    right: viewportPosition[0] + viewportSize[0],
    bottom: viewportPosition[1] + viewportSize[1]
  };
  const nodes = [];
  let truncated = false;
  function intersectsViewport(properties) {
    const position = properties.position;
    const size = properties.size;
    if (!Array.isArray(position) || !Array.isArray(size) || size[0] <= 0 || size[1] <= 0) return true;
    return position[0] + size[0] > viewport.left
      && position[1] + size[1] > viewport.top
      && position[0] < viewport.right
      && position[1] < viewport.bottom;
  }
  function walk(element, depth, path) {
    if (nodes.length >= maxNodes) { truncated = true; return; }
    const properties = safe(() => element.properties(), {});
    if (visibleOnly && depth > 0 && !intersectsViewport(properties)) return;
    const value = scalar(properties.value);
    nodes.push({
      path,
      depth,
      role: scalar(properties.role),
      subrole: scalar(properties.subrole),
      name: scalar(properties.name || properties.title),
      description: scalar(properties.description || properties.accessibilityDescription || properties.roleDescription),
      value: typeof value === "string" && value.length > 240 ? value.slice(0, 240) + "…" : value,
      help: scalar(properties.help),
      position: scalar(properties.position),
      size: scalar(properties.size),
      enabled: scalar(properties.enabled),
      focused: scalar(properties.focused)
    });
    if (depth >= maxDepth) return;
    const children = safe(() => element.uiElements(), []);
    for (let index = children.length - 1; index >= 0; index--) {
      walk(children[index], depth + 1, path + "." + index);
      if (nodes.length >= maxNodes) { truncated = true; break; }
    }
  }
  walk(root, 0, "0");
  return JSON.stringify({
    application: safe(() => process.name()),
    pid: safe(() => process.unixId()),
    window: windows[0] ? safe(() => windows[0].name()) : null,
    truncated,
    nodes
  });
}`;
		const raw = await execFile(
			"/usr/bin/osascript",
			["-l", "JavaScript", "-e", source, "--", app ?? "", String(maxDepth), String(maxNodes), String(visibleOnly)],
			{ timeout: 30_000 },
		);
		const tree = JSON.parse(raw.stdout) as {
			application: string;
			pid: number;
			window: string | null;
			truncated: boolean;
			nodes: Array<Record<string, unknown> & { path: string; depth: number; role: string | null }>;
		};
		const lines = [
			`Application: ${tree.application} (pid ${tree.pid})`,
			`Window: ${tree.window ?? "(none)"}`,
			`Nodes: ${tree.nodes.length}${tree.truncated ? " (truncated)" : ""}`,
			"",
		];
		for (const node of tree.nodes) {
			const fields = ["name", "description", "value", "help", "position", "size", "enabled", "focused"]
				.map((key) => node[key] === null || node[key] === undefined ? "" : `${key}=${JSON.stringify(node[key])}`)
				.filter(Boolean)
				.join(" ");
			lines.push(`${"  ".repeat(node.depth)}[${node.path}] ${node.role ?? "AXUnknown"}${fields ? ` ${fields}` : ""}`);
		}
		return lines.join("\n");
	}

	async function captureScreen() {
		const path = join(tmpdir(), `pi-computer-${randomUUID()}.png`);
		try {
			await execFile("/usr/sbin/screencapture", ["-x", "-t", "png", path], { timeout: 30_000 });
			const image = await readFile(path);
			return { type: "image" as const, data: image.toString("base64"), mimeType: "image/png" };
		} finally {
			await rm(path, { force: true });
		}
	}

	const keyCodes: Record<string, number> = {
		enter: 36,
		tab: 48,
		space: 49,
		backspace: 51,
		escape: 53,
		left: 123,
		right: 124,
		down: 125,
		up: 126,
		home: 115,
		end: 119,
		pageup: 116,
		pagedown: 121,
	};

	pi.registerTool({
		name: "computer_use",
		label: "Computer",
		description:
			"Control this macOS desktop using screenshots, coordinate clicks, keyboard input, scrolling, and app launching. Actions occur on the Mac running Pi Web.",
		promptSnippet: "View and control the macOS desktop",
		promptGuidelines: [
			"Use computer_use only when agent_browser or direct tools cannot perform the requested GUI task.",
			"Use computer_use with action=ui_tree before coordinate clicks; Accessibility positions use macOS logical points while screenshots may use Retina physical pixels.",
			"Use ask_user_question before computer_use performs a destructive, financial, authentication, permission, or externally visible action when user intent is not already explicit.",
		],
		parameters: Type.Object({
			action: StringEnum(["request_permissions", "ui_tree", "screenshot", "click", "type", "key", "scroll", "open_app"] as const),
			x: Type.Optional(Type.Integer({ minimum: 0 })),
			y: Type.Optional(Type.Integer({ minimum: 0 })),
			text: Type.Optional(Type.String()),
			key: Type.Optional(Type.String()),
			modifiers: Type.Optional(Type.Array(StringEnum(["command", "control", "option", "shift"] as const))),
			delta: Type.Optional(Type.Integer({ minimum: -20, maximum: 20 })),
			app: Type.Optional(Type.String({ description: "Application name for open_app or ui_tree; ui_tree defaults to the frontmost app" })),
			maxDepth: Type.Optional(Type.Integer({ minimum: 1, maximum: 12 })),
			maxNodes: Type.Optional(Type.Integer({ minimum: 1, maximum: 500 })),
			visibleOnly: Type.Optional(Type.Boolean({ description: "For ui_tree, omit off-screen elements; defaults to true" })),
		}),
		executionMode: "sequential",
		async execute(_id, params, signal) {
			if (process.platform !== "darwin") throw new Error("computer_use currently supports macOS only");
			if (signal?.aborted) throw new Error("Computer action cancelled");

			if (params.action === "request_permissions") {
				let accessibility = false;
				try {
					accessibility = (await runAppleScript('tell application "System Events" to get UI elements enabled')) === "true";
				} catch {}
				if (!accessibility) await execFile("/usr/bin/open", ["x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility"]);
				const image = await captureScreen();
				return {
					content: [
						{ type: "text", text: accessibility ? "Accessibility is enabled. Screen capture completed." : "Accessibility is not enabled. System Settings was opened; enable access for the Node/pi-web process, then retry." },
						image,
					],
					details: { accessibility },
				};
			}

			if (params.action === "ui_tree") {
				const tree = await readAccessibilityTree(params.app, params.maxDepth ?? 10, params.maxNodes ?? 300, params.visibleOnly !== false);
				return { content: [{ type: "text", text: tree }], details: { app: params.app ?? null } };
			}

			if (params.action === "screenshot") {
				return { content: [{ type: "text", text: "macOS desktop screenshot" }, await captureScreen()], details: {} };
			}

			switch (params.action) {
				case "click":
					if (params.x === undefined || params.y === undefined) throw new Error("x and y are required for click");
					await runAppleScript(
						'on run argv\nset px to item 1 of argv as integer\nset py to item 2 of argv as integer\ntell application "System Events" to click at {px, py}\nend run',
						[String(params.x), String(params.y)],
					);
					break;
				case "type":
					await runAppleScript(
						'on run argv\ntell application "System Events" to keystroke (item 1 of argv)\nend run',
						[required(params.text, "text")],
					);
					break;
				case "key": {
					const name = required(params.key, "key").toLowerCase();
					const code = keyCodes[name];
					if (code === undefined) throw new Error(`Unsupported key: ${name}`);
					const modifiers = params.modifiers?.map((modifier) => `${modifier} down`).join(", ");
					await runAppleScript(`tell application "System Events" to key code ${code}${modifiers ? ` using {${modifiers}}` : ""}`);
					break;
				}
				case "scroll": {
					const delta = params.delta ?? -1;
					if (delta === 0) break;
					const code = delta < 0 ? keyCodes.pagedown : keyCodes.pageup;
					await runAppleScript(`tell application "System Events"\nrepeat ${Math.abs(delta)} times\nkey code ${code}\nend repeat\nend tell`);
					break;
				}
				case "open_app":
					await execFile("/usr/bin/open", ["-a", required(params.app, "app")], { timeout: 30_000 });
					break;
			}

			await new Promise((resolve) => setTimeout(resolve, 500));
			return { content: [{ type: "text", text: `Completed computer action: ${params.action}` }, await captureScreen()], details: {} };
		},
	});
}

package.json

{
  "name": "pi-macos-computer-tools",
  "private": true,
  "version": "1.0.0",
  "type": "module"
}

pi-interactive-tools-README.md

Pi macOS computer-use prototype for agegr/pi-web

Experimental global Pi extension tested with agegr/pi-web 0.8.7, Pi 0.84.1, and macOS 15.5.

This revision intentionally reuses established Pi packages instead of duplicating them:

npm install -g agent-browser@0.33.2
pi install npm:pi-agent-browser-native
pi install npm:@juicesharp/rpiv-ask-user-question

• Browser automation: agent_browser from pi-agent-browser-native.
• User elicitation: ask_user_question from @juicesharp/rpiv-ask-user-question.

The local extension now contains only capabilities not covered by those packages:

• computer_use: macOS screenshot, Accessibility UI-tree inspection, logical-coordinate click, typing, key presses, scrolling, and app launch.
• Current-run edited-files widget: displays successful edit and write targets above the Pi Web editor.

Install

mkdir -p ~/.pi/agent/extensions/interactive-tools
cp index.ts package.json ~/.pi/agent/extensions/interactive-tools/

Start a new Pi Web session or reload the current session.

Permissions and security

The prototype runs inside Pi Web's Node.js process. macOS may therefore attribute Accessibility and Screen Recording permissions to node. A signed native app/helper with a stable bundle identifier would provide a better TCC identity.

computer_use acts on the Mac running Pi Web. Do not expose a privileged Pi Web instance to an untrusted network. Use authentication plus a trusted LAN/VPN, and require explicit user intent for destructive, financial, authentication, permission, or externally visible actions.

For GitHub access, prefer the GitHub CLI keychain (gh auth login) instead of storing a plaintext token in Pi settings. Fine-grained private-repository tokens should receive only the repository and read permissions actually required.