Add atomic native drags

apps/desktop/native/sources/client/actions.swift

@@ -6,7 +6,7 @@ import PeekabooFoundation
 
 private struct ActionRequest: Decodable {
 	enum Operation: String, Decodable {
-		case click, type, key, select, scroll
+		case click, type, key, select, scroll, drag
 	}
 
 	let op: Operation
@@ -22,6 +22,10 @@ private struct ActionRequest: Decodable {
 	let kind: String?
 	let direction: String?
 	let amount: Int?
+	let from: PointerPoint?
+	let to: PointerPoint?
+	let button: String?
+	let duration_ms: Int?
 
 	func validate() throws {
 		guard !snapshot.isEmpty, snapshot.utf16.count <= 256 else {
@@ -39,6 +43,9 @@ private struct ActionRequest: Decodable {
 		if op != .scroll, direction != nil || amount != nil {
 			throw ActionError("Only scrolling accepts direction or amount.")
 		}
+		if op != .drag, from != nil || to != nil || button != nil || duration_ms != nil {
+			throw ActionError("Only dragging accepts endpoints, button, or duration.")
+		}
 		if op != .click, op != .scroll, point != nil {
 			throw ActionError("Only clicks or scrolling accept a target point.")
 		}
@@ -60,6 +67,15 @@ private struct ActionRequest: Decodable {
 					let amount, (1...20).contains(amount)
 				else { throw ActionError("Choose up, down, left, or right and 1 to 20 native scroll units.") }
 			}
+		case .drag:
+			guard element == nil, text == nil, key == nil, let from, let to, from != to else {
+				throw ActionError("Choose distinct normalized screenshot points for the drag endpoints.")
+			}
+			try from.validate()
+			try to.validate()
+			guard button == nil || ExactWindowHeldPointerButton(rawValue: button!) != nil,
+				ExactWindowDragRequest.durationMillisecondsRange.contains(duration_ms ?? 500)
+			else { throw ActionError("Choose a left or right drag lasting 1 to 10000 milliseconds.") }
 		case .type, .select:
 			guard let element, !element.isEmpty, element.utf16.count <= 256, key == nil else {
 				throw ActionError("Choose a literal element ID from the inspected snapshot.")
@@ -92,7 +108,7 @@ private struct ActionRequest: Decodable {
 	}
 }
 
-private struct PointerPoint: Decodable {
+private struct PointerPoint: Decodable, Equatable {
 	let x: Double
 	let y: Double
 
@@ -214,7 +230,8 @@ func nativeAction(_ client: PeekabooBridgeClient) async throws -> Data {
 			scrollWindow = try .init(identity: identity, bounds: bounds, focusedElement: context.focusedElement)
 		}
 		var point: CGPoint?
-		if request.point != nil {
+		var drag: ExactWindowDragRequest?
+		if request.point != nil || request.op == .drag {
 			// Normalized coordinates survive host image resizing; authority stays in the bridge's capture.
 			let authority = try SnapshotTargetReceiptPlanner.assemble(
 				snapshotID: request.snapshot, detectionResult: detection
@@ -223,7 +240,19 @@ func nativeAction(_ client: PeekabooBridgeClient) async throws -> Data {
 				window.bounds.contains(captured), !detection.screenshotPath.isEmpty
 			else { throw ActionError("This observation has no pixel-backed coordinate authority for its exact window.") }
 			point = try request.point?.mapped(in: authority)
-
+			if request.op == .drag {
+				// Drag validates the full capture receipt; coordinate authority intentionally omits focus.
+				let dragWindow = try UIAutomationTarget.ExactWindow(
+					identity: identity, bounds: bounds, focusedElement: context.focusedElement
+				)
+				drag = try ExactWindowDragRequest(
+					snapshotID: request.snapshot, target: dragWindow,
+					from: request.from!.mapped(in: authority), to: request.to!.mapped(in: authority),
+					durationMilliseconds: request.duration_ms ?? 500,
+					button: request.button.flatMap(ExactWindowHeldPointerButton.init(rawValue:)) ?? .left
+				)
+				try drag!.validate()
+			}
 		}
 		lease = try await client.beginSnapshotMutation(snapshotId: request.snapshot)
 		invoked = true
@@ -243,6 +272,8 @@ func nativeAction(_ client: PeekabooBridgeClient) async throws -> Data {
 				target: request.element, point: point, snapshotId: request.snapshot,
 				expectedWindow: scrollWindow!, foreground: false
 			)))
+		case .drag:
+			evidence = try await ActionEvidence(client.dragExactWindow(drag!))
 		case .type:
 			evidence = try await ActionEvidence(client.setValueWithOutcome(
 				target: request.element!, value: .string(request.text!), snapshotId: request.snapshot

apps/desktop/native/sources/desktop/desktop.swift

@@ -67,7 +67,7 @@ private final class Desktop {
 						.listApplications, .listWindows, .desktopObservation,
 						.ownsSnapshot, .getDetectionResult, .beginSnapshotMutation, .finishSnapshotMutation,
 						.targetedClick, .exactWindowTargetedClick, .setValue, .exactWindowTargetedHotkey,
-						.selectText, .targetedScroll,
+						.selectText, .targetedScroll, .exactWindowDrag,
 						.activateApplication, .focusWindow, .restoreWindow,
 					],
 					hostKind: .gui,

apps/host/src/desktop.ts

@@ -7,6 +7,7 @@ import { promisify } from "node:util";
 
 import {
 	type Desktop,
+	DESKTOP_BUTTONS,
 	DESKTOP_CLICKS,
 	DESKTOP_DIRECTIONS,
 	DESKTOP_KEYS,
@@ -319,6 +320,24 @@ function validateAction(request: DesktopAction) {
 	if (typeof request.snapshot !== "string" || !request.snapshot || request.snapshot.length > 256) {
 		throw new Error("Use the snapshot_id from a fresh desktop_inspect result.");
 	}
+	if (request.op === "drag") {
+		validatePoint(request.from);
+		validatePoint(request.to);
+		if (request.from.x === request.to.x && request.from.y === request.to.y) {
+			throw new Error("Choose distinct start and end points for a drag.");
+		}
+		if (request.button !== undefined && !DESKTOP_BUTTONS.includes(request.button)) {
+			throw new Error("Choose the left or right mouse button for a drag.");
+		}
+		if (
+			request.duration_ms !== undefined
+			&& (!Number.isInteger(request.duration_ms) || request.duration_ms < 1
+				|| request.duration_ms > 10000)
+		) {
+			throw new Error("Drag duration must be 1 to 10000 milliseconds.");
+		}
+		return;
+	}
 	if (request.op === "click" || request.op === "scroll") {
 		if ((request.element === undefined) === (request.point === undefined)) {
 			throw new Error("Choose exactly one observed element ID or normalized screenshot point.");
@@ -603,6 +622,7 @@ export const desktop: Desktop = async (request, context) => {
 			"key",
 			"select",
 			"scroll",
+			"drag",
 			"activate",
 			"focus",
 			"restore",

docs/architecture.md

@@ -178,7 +178,7 @@ Peekaboo's native bridge inside Ace's UI process, which owns macOS Accessibility
 Recording, and Event Synthesizing permissions. The host invokes a bundled client over a local Unix socket. The bridge
 accepts only the client's exact identifier signed by Ace's team; the client verifies the host's
 signing team. Tools expose application and window inventories, observation, element or screenshot
-clicks, scrolling, replacement of editable field values, text selection, and keys or shortcuts. Screenshot
+clicks, scrolling, atomic drags, replacement of editable field values, text selection, and keys or shortcuts. Screenshot
 points are normalized and mapped through the snapshot's native capture geometry. Control input uses background delivery bound to a
 snapshot's exact process, window, and controls; every dispatched action requires a new observation.
 Separate activation, focus, and restore tools accept generation-bound inventory targets, with exact

docs/desktop-tools.md

@@ -1,7 +1,7 @@
 # Native desktop tools
 
 Ace's pi harness can inspect applications and windows, activate apps, focus or restore windows, click observed controls, replace editable
-field values, select text, send keys or shortcuts, and scroll on the machine running the channel's tools. It uses
+field values, select text, send keys or shortcuts, scroll, and drag on the machine running the channel's tools. It uses
 [Peekaboo](https://github.com/openclaw/Peekaboo) for macOS Accessibility, screen capture, and
 targeted input. The model receives the accessibility text, screenshot, and action outcome. The
 same result appears in the chat's expandable tool output, including after reopening the channel.
@@ -48,6 +48,15 @@ find the client alongside Ace Helper automatically.
 - `desktop_scroll` scrolls an observed element or screenshot point `up`, `down`, `left`, or `right`.
   `amount` is 1 to 20 native units: Accessibility pages/actions or window-routed wheel ticks, depending
   on the target. It is not a pixel distance; inspect the resulting position before continuing.
+- `desktop_drag` performs one straight-line press, move, and release between `from` and `to`
+  screenshot points inside the same captured window. `button` defaults to `left`, with `right`
+  also supported. `duration_ms` defaults to 500 and accepts 1 to 10000 milliseconds. The native
+  bridge owns the whole gesture, including release cleanup after cancellation or client death.
+  No pointer button stays held across tool calls; a drag cannot cross windows. Accepted native
+  delivery does not prove that the application moved or dropped anything; verify the effect with
+  a fresh `desktop_inspect`. An inactive view may ignore the gesture, as observed during native
+  validation. If observation shows no effect, explicitly use `desktop_focus`, then inspect again
+  before deciding on another action. Ace does not automatically switch to foreground delivery.
 - `desktop_type` replaces the entire string value of one observed editable Accessibility element.
   It does not append text, send keystrokes, or use the clipboard. Fields that do not support this
   operation are refused.
@@ -104,7 +113,7 @@ Control and keyboard tools use targeted background delivery. They do not activat
 mouse or keyboard input when a background route is unavailable. Pointer actions do not move the
 physical pointer. The target app can still respond by changing its own state or opening a window.
 Modifier-clicks and long presses need a separate foreground interaction contract; they are not
-emulated with held keys or mouse buttons across calls. Literal insertion, drag-and-drop,
+emulated with held keys or mouse buttons across calls. Literal insertion,
 clipboard operations, foreground interaction, application launch, window geometry/close, menus, and dialogs remain later slices of
 [native computer use](https://github.com/githubnext/ace2/issues/8).
 

packages/channel/src/desktop.ts

@@ -43,6 +43,14 @@ export type DesktopRequest =
 		direction: DesktopDirection;
 		amount: number;
 	}
+	| {
+		op: "drag";
+		snapshot: string;
+		from: DesktopPoint;
+		to: DesktopPoint;
+		button?: DesktopButton;
+		duration_ms?: number;
+	}
 	| { op: "type"; snapshot: string; element: string; text: string }
 	| {
 		op: "select";
@@ -58,8 +66,10 @@ export type DesktopRequest =
 export type DesktopPoint = { x: number; y: number };
 export const DESKTOP_CLICKS = ["single", "double", "right", "middle", "triple"] as const;
 export const DESKTOP_DIRECTIONS = ["up", "down", "left", "right"] as const;
+export const DESKTOP_BUTTONS = ["left", "right"] as const;
 export type DesktopClick = (typeof DESKTOP_CLICKS)[number];
 export type DesktopDirection = (typeof DESKTOP_DIRECTIONS)[number];
+export type DesktopButton = (typeof DESKTOP_BUTTONS)[number];
 
 export const DESKTOP_KEYS = [
 	"enter",
@@ -132,7 +142,7 @@ export type DesktopModifier = (typeof DESKTOP_MODIFIERS)[number];
 export type DesktopSelection = (typeof DESKTOP_SELECTIONS)[number];
 export type DesktopAction = Extract<
 	DesktopRequest,
-	{ op: "click" | "type" | "key" | "select" | "scroll" | "activate" | "focus" | "restore" }
+	{ op: "click" | "type" | "key" | "select" | "scroll" | "drag" | "activate" | "focus" | "restore" }
 >;
 export type DesktopOutcome = "completed" | "refused" | "unknown";
 export type DesktopResult = {
@@ -145,7 +155,8 @@ export type Desktop = (request: DesktopRequest, context: Context) => Promise<Des
 
 export function isDesktopAction(request: DesktopRequest): request is DesktopAction {
 	return request.op === "click" || request.op === "type" || request.op === "key"
-		|| request.op === "select" || request.op === "scroll" || isDesktopManagement(request);
+		|| request.op === "select" || request.op === "scroll" || request.op === "drag"
+		|| isDesktopManagement(request);
 }
 
 export function isDesktopManagement(request: DesktopRequest): request is DesktopManagement {
@@ -287,6 +298,21 @@ export function desktop(execute: Desktop): Extension {
 				executionMode: "sequential",
 				execute: async (args, api, context) => act({ op: "scroll", ...args }, api, context),
 			}),
+			defineTool({
+				name: "desktop_drag",
+				description:
+					"Press, move in a straight line, and release inside the exact window from a fresh desktop_inspect result. Pass snapshot_id as snapshot; from/to are distinct normalized screenshot points (x from its left edge, y from its top edge, each >= 0 and < 1). button defaults to left, with right also supported. duration_ms defaults to 500 and must be 1 to 10000. The native bridge owns the whole gesture and release cleanup; no button stays held across calls. Background delivery never moves the physical pointer or switches windows. Accepted delivery does not prove a drop; verify the effect with a fresh desktop_inspect. An inactive view may ignore the gesture. If observation shows no effect, use explicit desktop_focus followed by a fresh desktop_inspect before deciding on another action. Unsupported or changed windows are refused. The snapshot is single-use; never blindly repeat an interrupted drag because part may already have run.",
+				parameters: Type.Object({
+					snapshot,
+					from: point,
+					to: point,
+					button: Type.Optional(Type.Union(DESKTOP_BUTTONS.map((value) => Type.Literal(value)))),
+					duration_ms: Type.Optional(Type.Integer({ minimum: 1, maximum: 10000 })),
+				}),
+				replay: "unsafe",
+				executionMode: "sequential",
+				execute: async (args, api, context) => act({ op: "drag", ...args }, api, context),
+			}),
 			defineTool({
 				name: "desktop_type",
 				description: