feat: fill password fields from the Keychain or with approved agent text, and refuse typing or pasting into them otherwise

.github/workflows/check.yml

@@ -34,6 +34,8 @@ jobs:
           node-version: 24
       # Unsigned: signing belongs to each embedding app.
       - run: node dist/build.js --out build --scratch .build/native
+      # The host's caller and self-target rules; Peekaboo's own tests run with npm run peekaboo-test.
+      - run: swift test --package-path native --scratch-path .build/native --filter DesktopToolsCallersTests
       # Embedders rebuild into the same output; notices from other sources must survive.
       - run: |
           mkdir -p build/Licenses/embedder && echo kept > build/Licenses/embedder/NOTICE

CHANGELOG.md

@@ -4,6 +4,16 @@ Notable changes to Desktop Tools, newest first. Each release's GitHub release no
 
 ## [Unreleased]
 
+### Added
+
+- The Desktop Tools app asks once per agent, such as Claude Code or a terminal app, "Allow *name* to control this Mac?", and refuses every request from an agent you haven't allowed with `CALLER_NOT_APPROVED`, listings and captures included, without making it wait. Agents are recognized by their validated signature, or by their path when unsigned, and **Approved Agents** in the menu revokes, allows or forgets each one. After updating, each agent you use is asked about once. Recent Activity names the agent, such as `claude-code`, instead of `node`. Embedding apps opt in with `desktop_tools_set_caller_handler`. ([#120](https://github.com/githubnext/desktop-tools/pull/120))
+- `desktop_fill_password` fills the focused password field. It reads a Keychain item the agent names, by service or by server (an internet password fills only a web page on that server), so macOS asks you before Desktop Tools reads it and the agent never sees the password. It can also take text from the agent, which you approve fill by fill in a Desktop Tools prompt that names the agent, the app and the site, unless you turn on **Always Allow Agent-Supplied Passwords** in the menu (off by default). It is completed when the field shows the right number of characters. Results and Recent Activity never include the password or its length, and system password prompts, such as administrator authentication, are refused. Embedding apps ask the user through `desktop_tools_set_confirmation_handlers` and set the always-allow choice with `desktop_tools_set_password_autofill`. ([#122](https://github.com/githubnext/desktop-tools/pull/122))
+
+### Changed
+
+- Agent input aimed at any Desktop Tools process, including another build of it, or sent while one of its prompts or menus is open, is refused with `SELF_TARGET_REFUSED`, so only the user can answer its prompts or choose Pause All Input. Agents can still observe them. ([#120](https://github.com/githubnext/desktop-tools/pull/120))
+- **Behavior change:** while a password field has keyboard focus, `desktop_key` refuses keys that type characters and Command-V, and `desktop_insert` refuses all its routes, and `desktop_menu` refuses Paste commands (found by their Command-V shortcut, whatever the title), with `SECURE_FIELD_REQUIRES_FILL`. Nothing is sent. Return, Tab, Escape, arrows, Delete and other shortcuts still work. Passwords go in only through `desktop_fill_password` and its approval. ([#122](https://github.com/githubnext/desktop-tools/pull/122))
+
 ### Fixed
 
 - Secure text fields, such as password fields, show `attributes.subrole: "AXSecureTextField"` and no value in `desktop_inspect`, `desktop_read` and `desktop_wait`, not even their mask characters; `desktop_type`, `desktop_set_value` and `desktop_select` refuse them before dispatch instead of returning `unknown`. ([#118](https://github.com/githubnext/desktop-tools/pull/118))

dist/build.js

@@ -70,6 +70,8 @@ const patches = [
     "peekaboo-web-accessibility.patch",
     "peekaboo-finder-paste.patch",
     "peekaboo-secure-subrole.patch",
+    "peekaboo-caller-admission.patch",
+    "peekaboo-secure-fill.patch",
 ].map((name) => join(native, "patches", name));
 function spawn(command, env) {
     return spawnSync(command[0], command.slice(1), { encoding: "utf8", env: env ?? process.env });

dist/client.d.ts

@@ -15,5 +15,14 @@ export type DesktopClientResult = DesktopResult & {
     data: unknown;
 };
 export type DesktopClient = (request: DesktopRequest, signal?: AbortSignal) => Promise<DesktopClientResult>;
+/**
+ * Gives a host refusal its own code and hint, in place of Peekaboo's generic permission code and its advice to grant
+ * a permission.
+ */
+export declare function hostRefusal<T extends {
+    code: string;
+    message: string;
+    hint?: unknown;
+}>(error: T): T;
 /** Creates a client bound to one host socket and signed client; calls are independent processes. */
 export declare function createDesktop(config: DesktopConfig): DesktopClient;

dist/client.js

@@ -92,6 +92,29 @@ async function run(path, args, signal, errorJson = false, input, agent) {
         throw new Error((failed.stderr?.trim() || failed.message).slice(0, 2000), { cause: error });
     }
 }
+/** Refusals the host owns rather than Peekaboo: their message starts with the code agents should see. */
+const HOST_REFUSAL = /^(CALLER_NOT_APPROVED|SELF_TARGET_REFUSED): /;
+/** What the agent can do about each host refusal: only the user can answer it. */
+const HOST_REFUSAL_HINTS = {
+    CALLER_NOT_APPROVED: "Ask the user to choose Allow for this agent in the Desktop Tools prompt or under Approved Agents in its menu, then retry once.",
+    SELF_TARGET_REFUSED: "Ask the user to act on Desktop Tools' own prompt, menu or window; don't retry it or work around it.",
+};
+/**
+ * Gives a host refusal its own code and hint, in place of Peekaboo's generic permission code and its advice to grant
+ * a permission.
+ */
+export function hostRefusal(error) {
+    const match = HOST_REFUSAL.exec(error.message);
+    if (!match)
+        return error;
+    const code = match[1];
+    return {
+        ...error,
+        code,
+        message: error.message.slice(match[0].length),
+        ...("hint" in error ? { hint: HOST_REFUSAL_HINTS[code] } : {}),
+    };
+}
 const caller = (config) => async (args, signal, options = {}) => {
     const { client, socket } = config;
     if (!existsSync(client))
@@ -113,7 +136,9 @@ const caller = (config) => async (args, signal, options = {}) => {
         throw new Error("The native desktop client returned an unsupported response.");
     }
     if (!value.success) {
-        const error = value.error;
+        const error = value.error && hostRefusal(value.error);
+        if (error && error.code !== value.error.code)
+            throw new NativeError(error.code, `${error.code}: ${error.message}`);
         const reason = error ? `${error.code}: ${error.message}` : "Native desktop operation failed";
         const name = config.name ?? "the desktop host app";
         const permission = error?.code.toLowerCase().includes("permission")
@@ -136,6 +161,10 @@ const caller = (config) => async (args, signal, options = {}) => {
             throw new Error("Native inspection returned a different application or window. Refresh the desktop inventory and select the target again.");
         }
     }
+    const refusal = value.data.error;
+    if (typeof refusal?.code === "string" && typeof refusal.message === "string") {
+        value.data.error = hostRefusal(refusal);
+    }
     return {
         ...value.data,
         ...(value.target_receipt ? { target_receipt: value.target_receipt } : {}),
@@ -882,6 +911,17 @@ function validateAction(request) {
             ? "Set a date as ISO 8601: 2026-10-17, 2026-10-17T15:30 (local time) or 2026-10-17T15:30:00Z. Use desktop_type for text."
             : "Set a finite number, true/false or an ISO 8601 date; use desktop_type for text.");
     }
+    if (request.op === "fill-password") {
+        const named = ["service", "server", "text"].filter((key) => key in request);
+        const strings = ["service", "server", "account"].every((key) => request[key] === undefined || (typeof request[key] === "string" && request[key].length >= 1 && request[key].length <= 1024));
+        if (named.length !== 1 || !strings || (request.text !== undefined && request.account !== undefined)) {
+            throw new Error("Name one Keychain item by service or server, with an optional account, or supply text; not more than one.");
+        }
+        // The secret is never quoted back, so its problems are described without it.
+        if (request.text !== undefined && (typeof request.text !== "string" || !request.text || request.text.length > 1024)) {
+            throw new Error("The password text must contain 1 to 1024 UTF-16 code units.");
+        }
+    }
     if (request.op === "type" && (typeof request.text !== "string" || request.text.length > 8192)) {
         throw new Error("Replacement text must contain at most 8192 UTF-16 code units.");
     }
@@ -1124,25 +1164,27 @@ async function act(native, request, signal) {
                     : request;
         const operation = request.op === "menu"
             ? "menu"
-            : request.op === "dialog-answer"
-                ? "dialog"
-                : request.op === "dock"
-                    ? "dock"
-                    : request.op === "clipboard-write"
-                        ? "clipboard"
-                        : request.op === "launch"
-                            ? "launch"
-                            : request.op === "open"
-                                ? "open"
-                                : request.op === "system-key"
-                                    ? "system-key"
-                                    : request.op === "switch-space"
-                                        ? "switch-space"
-                                        : request.op === "input-source"
-                                            ? "input-source"
-                                            : isDesktopManagement(request)
-                                                ? "management"
-                                                : "action";
+            : request.op === "fill-password"
+                ? "fill"
+                : request.op === "dialog-answer"
+                    ? "dialog"
+                    : request.op === "dock"
+                        ? "dock"
+                        : request.op === "clipboard-write"
+                            ? "clipboard"
+                            : request.op === "launch"
+                                ? "launch"
+                                : request.op === "open"
+                                    ? "open"
+                                    : request.op === "system-key"
+                                        ? "system-key"
+                                        : request.op === "switch-space"
+                                            ? "switch-space"
+                                            : request.op === "input-source"
+                                                ? "input-source"
+                                                : isDesktopManagement(request)
+                                                    ? "management"
+                                                    : "action";
         data = await native([operation], signal, {
             input: JSON.stringify(input),
             onDispatch() {
@@ -1460,6 +1502,7 @@ async function execute(native, request, abort) {
             "screen",
             "click",
             "type",
+            "fill-password",
             "key",
             "system-key",
             "spaces",
@@ -1560,8 +1603,10 @@ async function execute(native, request, abort) {
             act: (action, signal) => execute(native, action, signal),
         }, request, abort ?? new AbortController().signal);
     }
+    // A password fill may wait a minute for the user to answer a Keychain prompt or a confirmation.
+    const limit = request.op === "fill-password" ? 95 : 30;
     const deadline = new AbortController();
-    const timer = setTimeout(() => deadline.abort(new Error("Native desktop operation timed out after 30 seconds.")), 30_000);
+    const timer = setTimeout(() => deadline.abort(new Error(`Native desktop operation timed out after ${limit} seconds.`)), limit * 1000);
     const signal = abort ? AbortSignal.any([abort, deadline.signal]) : deadline.signal;
     try {
         if (request.op === "menubar-items") {

dist/protocol.d.ts

@@ -249,6 +249,16 @@ export type DesktopRequest = DesktopManagement | DesktopLaunch | DesktopOpen | D
     snapshot: string;
     element: string;
     text: string;
+}
+/** Fills the focused password field from a Keychain item named by service or server, or with text after approval. */
+ | {
+    op: "fill-password";
+    snapshot: string;
+    element: string;
+    service?: string;
+    server?: string;
+    account?: string;
+    text?: string;
 } | {
     op: "insert";
     snapshot: string;
@@ -342,7 +352,7 @@ export type DesktopModifier = (typeof DESKTOP_MODIFIERS)[number];
 export type DesktopSelection = (typeof DESKTOP_SELECTIONS)[number];
 export type DesktopElementAction = (typeof DESKTOP_ELEMENT_ACTIONS)[number];
 export type DesktopAction = Extract<DesktopRequest, {
-    op: "click" | "type" | "key" | "system-key" | "switch-space" | "input-source" | "insert" | "select" | "element-action" | "set-value" | "set-cell" | "scroll" | "drag" | "hover" | "press" | "activate" | "launch" | "open" | "quit" | "close" | "focus" | "minimize" | "restore" | "move" | "resize" | "clipboard-write" | "menu" | "dialog-answer" | "hide" | "unhide" | "maximize" | "fullscreen" | "menubar-click" | "dock" | "notification-action";
+    op: "click" | "type" | "fill-password" | "key" | "system-key" | "switch-space" | "input-source" | "insert" | "select" | "element-action" | "set-value" | "set-cell" | "scroll" | "drag" | "hover" | "press" | "activate" | "launch" | "open" | "quit" | "close" | "focus" | "minimize" | "restore" | "move" | "resize" | "clipboard-write" | "menu" | "dialog-answer" | "hide" | "unhide" | "maximize" | "fullscreen" | "menubar-click" | "dock" | "notification-action";
 }>;
 export type DesktopOutcome = "completed" | "refused" | "unknown";
 export type DesktopImage = {

dist/protocol.js

@@ -102,7 +102,7 @@ export const DESKTOP_ELEMENT_ACTIONS = [
     "AXScrollToVisible",
 ];
 export function isDesktopAction(request) {
-    return request.op === "click" || request.op === "type" || request.op === "key"
+    return request.op === "click" || request.op === "type" || request.op === "fill-password" || request.op === "key"
         || request.op === "system-key" || request.op === "switch-space" || request.op === "input-source"
         || request.op === "insert" || request.op === "select" || request.op === "scroll"
         || request.op === "drag" || request.op === "hover" || request.op === "press"