add iOS support (#358)

* ios

* tests

* docs

* real device

* better list

* fixes
This commit is contained in:
Chris Tate
2026-02-03 01:36:19 -06:00
committed by GitHub
parent 3c6ae7df9d
commit a1b992411e
18 changed files with 4197 additions and 56 deletions
+3
View File
@@ -48,3 +48,6 @@ docs/node_modules/
docs/.next/
docs/out/
docs/package-lock.json
# pnpm
.pnpm-store/
+92
View File
@@ -691,6 +691,98 @@ Core workflow:
## Integrations
### iOS Simulator
Control real Mobile Safari in the iOS Simulator for authentic mobile web testing. Requires macOS with Xcode.
**Setup:**
```bash
# Install Appium and XCUITest driver
npm install -g appium
appium driver install xcuitest
```
**Usage:**
```bash
# List available iOS simulators
agent-browser device list
# Launch Safari on a specific device
agent-browser -p ios --device "iPhone 16 Pro" open https://example.com
# Same commands as desktop
agent-browser -p ios snapshot -i
agent-browser -p ios tap @e1
agent-browser -p ios fill @e2 "text"
agent-browser -p ios screenshot mobile.png
# Mobile-specific commands
agent-browser -p ios swipe up
agent-browser -p ios swipe down 500
# Close session
agent-browser -p ios close
```
Or use environment variables:
```bash
export AGENT_BROWSER_PROVIDER=ios
export AGENT_BROWSER_IOS_DEVICE="iPhone 16 Pro"
agent-browser open https://example.com
```
| Variable | Description |
|----------|-------------|
| `AGENT_BROWSER_PROVIDER` | Set to `ios` to enable iOS mode |
| `AGENT_BROWSER_IOS_DEVICE` | Device name (e.g., "iPhone 16 Pro", "iPad Pro") |
| `AGENT_BROWSER_IOS_UDID` | Device UDID (alternative to device name) |
**Supported devices:** All iOS Simulators available in Xcode (iPhones, iPads), plus real iOS devices.
**Note:** The iOS provider boots the simulator, starts Appium, and controls Safari. First launch takes ~30-60 seconds; subsequent commands are fast.
#### Real Device Support
Appium also supports real iOS devices connected via USB. This requires additional one-time setup:
**1. Get your device UDID:**
```bash
xcrun xctrace list devices
# or
system_profiler SPUSBDataType | grep -A 5 "iPhone\|iPad"
```
**2. Sign WebDriverAgent (one-time):**
```bash
# Open the WebDriverAgent Xcode project
cd ~/.appium/node_modules/appium-xcuitest-driver/node_modules/appium-webdriveragent
open WebDriverAgent.xcodeproj
```
In Xcode:
- Select the `WebDriverAgentRunner` target
- Go to Signing & Capabilities
- Select your Team (requires Apple Developer account, free tier works)
- Let Xcode manage signing automatically
**3. Use with agent-browser:**
```bash
# Connect device via USB, then:
agent-browser -p ios --device "<DEVICE_UDID>" open https://example.com
# Or use the device name if unique
agent-browser -p ios --device "John's iPhone" open https://example.com
```
**Real device notes:**
- First run installs WebDriverAgent to the device (may require Trust prompt)
- Device must be unlocked and connected via USB
- Slightly slower initial connection than simulator
- Tests against real Safari performance and behavior
### Browserbase
[Browserbase](https://browserbase.com) provides remote browser infrastructure to make deployment of agentic browsing agents easy. Use it when running the agent-browser CLI in an environment where a local browser isn't feasible.
+49
View File
@@ -103,6 +103,12 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result<Value, ParseError
nav_cmd["headers"] = headers;
}
}
// Include iOS device info if specified (needed for auto-launch with existing daemon)
if flags.provider.as_deref() == Some("ios") {
if let Some(ref device) = flags.device {
nav_cmd["iosDevice"] = json!(device);
}
}
Ok(nav_cmd)
}
"back" => Ok(json!({ "id": id, "action": "back" })),
@@ -835,6 +841,48 @@ pub fn parse_command(args: &[String], flags: &Flags) -> Result<Value, ParseError
}
}
// === iOS-specific commands ===
"tap" => {
// Alias for click (semantic clarity for touch interfaces)
let sel = rest.get(0).ok_or_else(|| ParseError::MissingArguments {
context: "tap".to_string(),
usage: "tap <selector>",
})?;
Ok(json!({ "id": id, "action": "tap", "selector": sel }))
}
"swipe" => {
let direction = rest.get(0).ok_or_else(|| ParseError::MissingArguments {
context: "swipe".to_string(),
usage: "swipe <up|down|left|right> [distance]",
})?;
let valid_directions = ["up", "down", "left", "right"];
if !valid_directions.contains(direction) {
return Err(ParseError::InvalidValue {
message: format!("Invalid swipe direction: {}", direction),
usage: "swipe <up|down|left|right> [distance]",
});
}
let mut cmd = json!({ "id": id, "action": "swipe", "direction": direction });
if let Some(distance) = rest.get(1) {
if let Ok(d) = distance.parse::<u32>() {
cmd.as_object_mut().unwrap().insert("distance".to_string(), json!(d));
}
}
Ok(cmd)
}
"device" => {
match rest.get(0).map(|s| *s) {
Some("list") | None => {
// List available iOS simulators
Ok(json!({ "id": id, "action": "device_list" }))
}
Some(sub) => Err(ParseError::UnknownSubcommand {
subcommand: sub.to_string(),
valid_options: &["list"],
}),
}
}
_ => Err(ParseError::UnknownCommand {
command: cmd.to_string(),
}),
@@ -1376,6 +1424,7 @@ mod tests {
user_agent: None,
provider: None,
ignore_https_errors: false,
device: None,
}
}
+18
View File
@@ -215,6 +215,8 @@ pub fn ensure_daemon(
ignore_https_errors: bool,
profile: Option<&str>,
state: Option<&str>,
provider: Option<&str>,
device: Option<&str>,
) -> Result<DaemonResult, String> {
// Check if daemon is running AND responsive
if is_daemon_running(session) && daemon_ready(session) {
@@ -343,6 +345,14 @@ pub fn ensure_daemon(
cmd.env("AGENT_BROWSER_STATE", st);
}
if let Some(p) = provider {
cmd.env("AGENT_BROWSER_PROVIDER", p);
}
if let Some(d) = device {
cmd.env("AGENT_BROWSER_IOS_DEVICE", d);
}
// Create new process group and session to fully detach
unsafe {
cmd.pre_exec(|| {
@@ -410,6 +420,14 @@ pub fn ensure_daemon(
cmd.env("AGENT_BROWSER_STATE", st);
}
if let Some(p) = provider {
cmd.env("AGENT_BROWSER_PROVIDER", p);
}
if let Some(d) = device {
cmd.env("AGENT_BROWSER_IOS_DEVICE", d);
}
// CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS
const CREATE_NEW_PROCESS_GROUP: u32 = 0x00000200;
const DETACHED_PROCESS: u32 = 0x00000008;
+9
View File
@@ -18,6 +18,7 @@ pub struct Flags {
pub user_agent: Option<String>,
pub provider: Option<String>,
pub ignore_https_errors: bool,
pub device: Option<String>,
}
pub fn parse_flags(args: &[String]) -> Flags {
@@ -49,6 +50,7 @@ pub fn parse_flags(args: &[String]) -> Flags {
user_agent: env::var("AGENT_BROWSER_USER_AGENT").ok(),
provider: env::var("AGENT_BROWSER_PROVIDER").ok(),
ignore_https_errors: false,
device: env::var("AGENT_BROWSER_IOS_DEVICE").ok(),
};
let mut i = 0;
@@ -131,6 +133,12 @@ pub fn parse_flags(args: &[String]) -> Flags {
}
}
"--ignore-https-errors" => flags.ignore_https_errors = true,
"--device" => {
if let Some(d) = args.get(i + 1) {
flags.device = Some(d.clone());
i += 1;
}
}
_ => {}
}
i += 1;
@@ -165,6 +173,7 @@ pub fn clean_args(args: &[String]) -> Vec<String> {
"--user-agent",
"-p",
"--provider",
"--device",
];
for arg in args.iter() {
+2
View File
@@ -207,6 +207,8 @@ fn main() {
flags.ignore_https_errors,
flags.profile.as_deref(),
flags.state.as_deref(),
flags.provider.as_deref(),
flags.device.as_deref(),
) {
Ok(result) => result,
Err(e) => {
+121 -2
View File
@@ -78,6 +78,53 @@ pub fn print_response(resp: &Response, json_mode: bool, action: Option<&str>) {
);
return;
}
// iOS Devices
if let Some(devices) = data.get("devices").and_then(|v| v.as_array()) {
if devices.is_empty() {
println!("No iOS devices available. Open Xcode to download simulator runtimes.");
return;
}
// Separate real devices from simulators
let real_devices: Vec<_> = devices
.iter()
.filter(|d| d.get("isRealDevice").and_then(|v| v.as_bool()).unwrap_or(false))
.collect();
let simulators: Vec<_> = devices
.iter()
.filter(|d| !d.get("isRealDevice").and_then(|v| v.as_bool()).unwrap_or(false))
.collect();
if !real_devices.is_empty() {
println!("Connected Devices:\n");
for device in real_devices.iter() {
let name = device.get("name").and_then(|v| v.as_str()).unwrap_or("Unknown");
let runtime = device.get("runtime").and_then(|v| v.as_str()).unwrap_or("");
let udid = device.get("udid").and_then(|v| v.as_str()).unwrap_or("");
println!(" {} {} ({})", color::green(""), name, runtime);
println!(" {}", color::dim(udid));
}
println!();
}
if !simulators.is_empty() {
println!("Simulators:\n");
for device in simulators.iter() {
let name = device.get("name").and_then(|v| v.as_str()).unwrap_or("Unknown");
let runtime = device.get("runtime").and_then(|v| v.as_str()).unwrap_or("");
let state = device.get("state").and_then(|v| v.as_str()).unwrap_or("Unknown");
let udid = device.get("udid").and_then(|v| v.as_str()).unwrap_or("");
let state_indicator = if state == "Booted" {
color::green("")
} else {
color::dim("")
};
println!(" {} {} ({})", state_indicator, name, runtime);
println!(" {}", color::dim(udid));
}
}
return;
}
// Tabs
if let Some(tabs) = data.get("tabs").and_then(|v| v.as_array()) {
for (i, tab) in tabs.iter().enumerate() {
@@ -1551,6 +1598,68 @@ Examples:
"##
}
// === iOS Commands ===
"tap" => {
r##"
agent-browser tap - Tap an element (touch gesture)
Usage: agent-browser tap <selector>
Taps an element. This is an alias for 'click' that provides semantic clarity
for touch-based interfaces like iOS Safari.
Options:
--json Output as JSON
--session <name> Use specific session
Examples:
agent-browser tap "#submit-button"
agent-browser tap @e1
agent-browser -p ios tap "button:has-text('Sign In')"
"##
}
"swipe" => {
r##"
agent-browser swipe - Swipe gesture (iOS)
Usage: agent-browser swipe <direction> [distance]
Performs a swipe gesture on iOS Safari. The direction determines
which way the content moves (swipe up scrolls down, etc.).
Arguments:
direction up, down, left, or right
distance Optional distance in pixels (default: 300)
Options:
--json Output as JSON
--session <name> Use specific session
Examples:
agent-browser -p ios swipe up
agent-browser -p ios swipe down 500
agent-browser -p ios swipe left
"##
}
"device" => {
r##"
agent-browser device - Manage iOS simulators
Usage: agent-browser device <subcommand>
Subcommands:
list List available iOS simulators
Options:
--json Output as JSON
--session <name> Use specific session
Examples:
agent-browser device list
agent-browser -p ios device list
"##
}
_ => return false,
};
println!("{}", help.trim());
@@ -1660,7 +1769,8 @@ Options:
--proxy-bypass <hosts> Bypass proxy for these hosts (or AGENT_BROWSER_PROXY_BYPASS)
e.g., --proxy-bypass "localhost,*.internal.com"
--ignore-https-errors Ignore HTTPS certificate errors
-p, --provider <name> Cloud browser provider (or AGENT_BROWSER_PROVIDER env)
-p, --provider <name> Browser provider: ios, browserbase, kernel, browseruse
--device <name> iOS device name (e.g., "iPhone 15 Pro")
--json JSON output
--full, -f Full page screenshot
--headed Show browser window (not headless)
@@ -1671,8 +1781,10 @@ Options:
Environment:
AGENT_BROWSER_SESSION Session name (default: "default")
AGENT_BROWSER_EXECUTABLE_PATH Custom browser executable path
AGENT_BROWSER_PROVIDER Cloud browser provider
AGENT_BROWSER_PROVIDER Browser provider (ios, browserbase, kernel, browseruse)
AGENT_BROWSER_STREAM_PORT Enable WebSocket streaming on port (e.g., 9223)
AGENT_BROWSER_IOS_DEVICE Default iOS device name
AGENT_BROWSER_IOS_UDID Default iOS device UDID
Examples:
agent-browser open example.com
@@ -1684,6 +1796,13 @@ Examples:
agent-browser screenshot --full
agent-browser --cdp 9222 snapshot # Connect via CDP port
agent-browser --profile ~/.myapp open example.com # Persistent profile
iOS Simulator (requires Xcode and Appium):
agent-browser -p ios open example.com # Use default iPhone
agent-browser -p ios --device "iPhone 15 Pro" open url # Specific device
agent-browser -p ios device list # List simulators
agent-browser -p ios swipe up # Swipe gesture
agent-browser -p ios tap @e1 # Touch element
"#
);
}
+280
View File
@@ -0,0 +1,280 @@
import { CodeBlock } from "@/components/code-block";
export default function iOS() {
return (
<div className="max-w-2xl mx-auto px-4 sm:px-6 py-8 sm:py-12">
<div className="prose">
<h1>iOS Simulator</h1>
<p>
Control real Mobile Safari in the iOS Simulator for authentic mobile
web testing. Uses Appium with XCUITest for native automation.
</p>
<h2>Requirements</h2>
<ul>
<li>macOS with Xcode installed</li>
<li>iOS Simulator runtimes (download via Xcode)</li>
<li>Appium with XCUITest driver</li>
</ul>
<h2>Setup</h2>
<CodeBlock
code={`# Install Appium globally
npm install -g appium
# Install the XCUITest driver for iOS
appium driver install xcuitest`}
/>
<h2>List available devices</h2>
<p>See all iOS simulators available on your system:</p>
<CodeBlock
code={`agent-browser device list
# Output:
# Available iOS Simulators:
#
# iPhone 16 Pro (iOS 18.0)
# F21EEC0D-7618-419F-811B-33AF27A8B2FD
# iPhone 16 Pro Max (iOS 18.0)
# 50402807-C9B8-4D37-9F13-2E00E782C744
# iPad Pro 13-inch (M4) (iOS 18.0)
# 3A6C6436-B909-4593-866D-91D1062BB070
# ...`}
/>
<h2>Basic usage</h2>
<p>
Use the <code>-p ios</code> flag to enable iOS mode. The workflow is
identical to desktop:
</p>
<CodeBlock
code={`# Launch Safari on iPhone 16 Pro
agent-browser -p ios --device "iPhone 16 Pro" open https://example.com
# Get snapshot with refs (same as desktop)
agent-browser -p ios snapshot -i
# Interact using refs
agent-browser -p ios tap @e1
agent-browser -p ios fill @e2 "text"
# Take screenshot
agent-browser -p ios screenshot mobile.png
# Close session (shuts down simulator)
agent-browser -p ios close`}
/>
<h2>Mobile-specific commands</h2>
<CodeBlock
code={`# Swipe gestures
agent-browser -p ios swipe up
agent-browser -p ios swipe down
agent-browser -p ios swipe left
agent-browser -p ios swipe right
# Swipe with distance (pixels)
agent-browser -p ios swipe up 500
# Tap (alias for click, semantically clearer for touch)
agent-browser -p ios tap @e1`}
/>
<h2>Environment variables</h2>
<p>Configure iOS mode via environment variables:</p>
<CodeBlock
code={`export AGENT_BROWSER_PROVIDER=ios
export AGENT_BROWSER_IOS_DEVICE="iPhone 16 Pro"
# Now all commands use iOS
agent-browser open https://example.com
agent-browser snapshot -i
agent-browser tap @e1`}
/>
<table>
<thead>
<tr>
<th>Variable</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<code>AGENT_BROWSER_PROVIDER</code>
</td>
<td>
Set to <code>ios</code> to enable iOS mode
</td>
</tr>
<tr>
<td>
<code>AGENT_BROWSER_IOS_DEVICE</code>
</td>
<td>Device name (e.g., &quot;iPhone 16 Pro&quot;)</td>
</tr>
<tr>
<td>
<code>AGENT_BROWSER_IOS_UDID</code>
</td>
<td>Device UDID (alternative to device name)</td>
</tr>
</tbody>
</table>
<h2>Supported devices</h2>
<p>
All iOS Simulators available in Xcode are supported, including:
</p>
<ul>
<li>All iPhone models (iPhone 15, 16, 17, SE, etc.)</li>
<li>All iPad models (iPad Pro, iPad Air, iPad mini, etc.)</li>
<li>Multiple iOS versions (17.x, 18.x, etc.)</li>
</ul>
<p>
<strong>Real devices</strong> are also supported via USB connection
(see below).
</p>
<h2>Real device support</h2>
<p>
Appium can control Safari on real iOS devices connected via USB. This
requires additional one-time setup.
</p>
<h3>1. Get your device UDID</h3>
<CodeBlock
code={`# List connected devices
xcrun xctrace list devices
# Or via system profiler
system_profiler SPUSBDataType | grep -A 5 "iPhone\\|iPad"`}
/>
<h3>2. Sign WebDriverAgent (one-time)</h3>
<p>
WebDriverAgent needs to be signed with your Apple Developer
certificate to run on real devices.
</p>
<CodeBlock
code={`# Open the WebDriverAgent Xcode project
cd ~/.appium/node_modules/appium-xcuitest-driver/node_modules/appium-webdriveragent
open WebDriverAgent.xcodeproj`}
/>
<p>In Xcode:</p>
<ol>
<li>
Select the <code>WebDriverAgentRunner</code> target
</li>
<li>Go to Signing &amp; Capabilities</li>
<li>
Select your Team (requires Apple Developer account, free tier works)
</li>
<li>Let Xcode manage signing automatically</li>
</ol>
<h3>3. Use with agent-browser</h3>
<CodeBlock
code={`# Connect device via USB, then use the UDID
agent-browser -p ios --device "<DEVICE_UDID>" open https://example.com
# Or use the device name if unique
agent-browser -p ios --device "John's iPhone" open https://example.com`}
/>
<h3>Real device notes</h3>
<ul>
<li>
First run installs WebDriverAgent to the device (may require Trust
prompt on device)
</li>
<li>Device must be unlocked and connected via USB</li>
<li>Slightly slower initial connection than simulator</li>
<li>Tests against real Safari performance and behavior</li>
<li>
On first install, go to Settings &rarr; General &rarr; VPN &amp;
Device Management to trust the developer certificate
</li>
</ul>
<h2>Performance notes</h2>
<ul>
<li>
<strong>First launch:</strong> Takes 30-60 seconds to boot the
simulator and start Appium
</li>
<li>
<strong>Subsequent commands:</strong> Fast (simulator stays running)
</li>
<li>
<strong>Close command:</strong> Shuts down simulator and Appium
server
</li>
</ul>
<h2>Differences from desktop</h2>
<table>
<thead>
<tr>
<th>Feature</th>
<th>Desktop</th>
<th>iOS</th>
</tr>
</thead>
<tbody>
<tr>
<td>Browser</td>
<td>Chromium/Firefox/WebKit</td>
<td>Safari only</td>
</tr>
<tr>
<td>Tabs</td>
<td>Supported</td>
<td>Single tab only</td>
</tr>
<tr>
<td>PDF export</td>
<td>Supported</td>
<td>Not supported</td>
</tr>
<tr>
<td>Screencast</td>
<td>Supported</td>
<td>Not supported</td>
</tr>
<tr>
<td>Swipe gestures</td>
<td>Not native</td>
<td>Native support</td>
</tr>
</tbody>
</table>
<h2>Troubleshooting</h2>
<h3>Appium not found</h3>
<CodeBlock
code={`# Make sure Appium is installed globally
npm install -g appium
appium driver install xcuitest
# Verify installation
appium --version`}
/>
<h3>No simulators available</h3>
<p>
Open Xcode and download iOS Simulator runtimes from{" "}
<strong>Settings &rarr; Platforms</strong>.
</p>
<h3>Simulator won&apos;t boot</h3>
<p>
Try booting the simulator manually from Xcode or the Simulator app to
ensure it works, then retry with agent-browser.
</p>
</div>
</div>
);
}
+1
View File
@@ -14,6 +14,7 @@ const navigation = [
{ name: "Snapshots", href: "/snapshots" },
{ name: "Streaming", href: "/streaming" },
{ name: "CDP Mode", href: "/cdp-mode" },
{ name: "iOS Simulator", href: "/ios" },
{ name: "Changelog", href: "/changelog" },
];
+2
View File
@@ -55,7 +55,9 @@
},
"homepage": "https://github.com/vercel-labs/agent-browser#readme",
"dependencies": {
"node-simctl": "^7.4.0",
"playwright-core": "^1.57.0",
"webdriverio": "^9.15.0",
"ws": "^8.19.0",
"zod": "^3.22.4"
},
+1738 -7
View File
File diff suppressed because it is too large Load Diff
+26
View File
@@ -129,6 +129,32 @@ agent-browser highlight @e1 # Highlight element
agent-browser record start demo.webm # Record session
```
### iOS Simulator (Mobile Safari)
```bash
# List available iOS simulators
agent-browser device list
# Launch Safari on a specific device
agent-browser -p ios --device "iPhone 16 Pro" open https://example.com
# Same workflow as desktop - snapshot, interact, re-snapshot
agent-browser -p ios snapshot -i
agent-browser -p ios tap @e1 # Tap (alias for click)
agent-browser -p ios fill @e2 "text"
agent-browser -p ios swipe up # Mobile-specific gesture
# Take screenshot
agent-browser -p ios screenshot mobile.png
# Close session (shuts down simulator)
agent-browser -p ios close
```
**Requirements:** macOS with Xcode, Appium (`npm install -g appium && appium driver install xcuitest`)
**Real devices:** Works with physical iOS devices if pre-configured. Use `--device "<UDID>"` where UDID is from `xcrun xctrace list devices`.
## Ref Lifecycle (Important)
Refs (`@e1`, `@e2`, etc.) are invalidated when the page changes. Always re-snapshot after:
+100 -46
View File
@@ -3,10 +3,15 @@ import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { BrowserManager } from './browser.js';
import { IOSManager } from './ios-manager.js';
import { parseCommand, serializeResponse, errorResponse } from './protocol.js';
import { executeCommand } from './actions.js';
import { executeIOSCommand } from './ios-actions.js';
import { StreamServer } from './stream-server.js';
// Manager type - either desktop browser or iOS
type Manager = BrowserManager | IOSManager;
// Platform detection
const isWindows = process.platform === 'win32';
@@ -167,8 +172,12 @@ export function getStreamPortFile(session?: string): string {
/**
* Start the daemon server
* @param options.streamPort Port for WebSocket stream server (0 to disable)
* @param options.provider Provider type ('ios' for iOS Simulator, undefined for desktop)
*/
export async function startDaemon(options?: { streamPort?: number }): Promise<void> {
export async function startDaemon(options?: {
streamPort?: number;
provider?: string;
}): Promise<void> {
// Ensure socket directory exists
const socketDir = getSocketDir();
if (!fs.existsSync(socketDir)) {
@@ -178,18 +187,24 @@ export async function startDaemon(options?: { streamPort?: number }): Promise<vo
// Clean up any stale socket
cleanupSocket();
const browser = new BrowserManager();
// Determine provider from options or environment
const provider = options?.provider ?? process.env.AGENT_BROWSER_PROVIDER;
const isIOS = provider === 'ios';
// Create appropriate manager
const manager: Manager = isIOS ? new IOSManager() : new BrowserManager();
let shuttingDown = false;
// Start stream server if port is specified (or use default if env var is set)
// Note: Stream server only works with BrowserManager (desktop), not iOS
const streamPort =
options?.streamPort ??
(process.env.AGENT_BROWSER_STREAM_PORT
? parseInt(process.env.AGENT_BROWSER_STREAM_PORT, 10)
: 0);
if (streamPort > 0) {
streamServer = new StreamServer(browser, streamPort);
if (streamPort > 0 && !isIOS && manager instanceof BrowserManager) {
streamServer = new StreamServer(manager, streamPort);
await streamServer.start();
// Write stream port to file for clients to discover
@@ -233,56 +248,91 @@ export async function startDaemon(options?: { streamPort?: number }): Promise<vo
continue;
}
// Auto-launch browser if not already launched and this isn't a launch command
// Handle device_list specially - it works without a session and always uses IOSManager
if (parseResult.command.action === 'device_list') {
const iosManager = new IOSManager();
try {
const devices = await iosManager.listAllDevices();
const response = {
id: parseResult.command.id,
success: true as const,
data: { devices },
};
socket.write(serializeResponse(response) + '\n');
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
socket.write(
serializeResponse(errorResponse(parseResult.command.id, message)) + '\n'
);
}
continue;
}
// Auto-launch if not already launched and this isn't a launch/close command
if (
!browser.isLaunched() &&
!manager.isLaunched() &&
parseResult.command.action !== 'launch' &&
parseResult.command.action !== 'close'
) {
const extensions = process.env.AGENT_BROWSER_EXTENSIONS
? process.env.AGENT_BROWSER_EXTENSIONS.split(',')
.map((p) => p.trim())
.filter(Boolean)
: undefined;
if (isIOS && manager instanceof IOSManager) {
// Auto-launch iOS Safari
// Check for device in command first (for reused daemons), then fall back to env vars
const cmd = parseResult.command as { iosDevice?: string };
const iosDevice = cmd.iosDevice || process.env.AGENT_BROWSER_IOS_DEVICE;
await manager.launch({
device: iosDevice,
udid: process.env.AGENT_BROWSER_IOS_UDID,
});
} else if (manager instanceof BrowserManager) {
// Auto-launch desktop browser
const extensions = process.env.AGENT_BROWSER_EXTENSIONS
? process.env.AGENT_BROWSER_EXTENSIONS.split(',')
.map((p) => p.trim())
.filter(Boolean)
: undefined;
// Parse args from env (comma or newline separated)
const argsEnv = process.env.AGENT_BROWSER_ARGS;
const args = argsEnv
? argsEnv
.split(/[,\n]/)
.map((a) => a.trim())
.filter((a) => a.length > 0)
: undefined;
// Parse args from env (comma or newline separated)
const argsEnv = process.env.AGENT_BROWSER_ARGS;
const args = argsEnv
? argsEnv
.split(/[,\n]/)
.map((a) => a.trim())
.filter((a) => a.length > 0)
: undefined;
// Parse proxy from env
const proxyServer = process.env.AGENT_BROWSER_PROXY;
const proxyBypass = process.env.AGENT_BROWSER_PROXY_BYPASS;
const proxy = proxyServer
? {
server: proxyServer,
...(proxyBypass && { bypass: proxyBypass }),
}
: undefined;
// Parse proxy from env
const proxyServer = process.env.AGENT_BROWSER_PROXY;
const proxyBypass = process.env.AGENT_BROWSER_PROXY_BYPASS;
const proxy = proxyServer
? {
server: proxyServer,
...(proxyBypass && { bypass: proxyBypass }),
}
: undefined;
const ignoreHTTPSErrors = process.env.AGENT_BROWSER_IGNORE_HTTPS_ERRORS === '1';
await browser.launch({
id: 'auto',
action: 'launch' as const,
headless: process.env.AGENT_BROWSER_HEADED !== '1',
executablePath: process.env.AGENT_BROWSER_EXECUTABLE_PATH,
extensions: extensions,
profile: process.env.AGENT_BROWSER_PROFILE,
storageState: process.env.AGENT_BROWSER_STATE,
args,
userAgent: process.env.AGENT_BROWSER_USER_AGENT,
proxy,
ignoreHTTPSErrors: ignoreHTTPSErrors,
});
const ignoreHTTPSErrors = process.env.AGENT_BROWSER_IGNORE_HTTPS_ERRORS === '1';
await manager.launch({
id: 'auto',
action: 'launch' as const,
headless: process.env.AGENT_BROWSER_HEADED !== '1',
executablePath: process.env.AGENT_BROWSER_EXECUTABLE_PATH,
extensions: extensions,
profile: process.env.AGENT_BROWSER_PROFILE,
storageState: process.env.AGENT_BROWSER_STATE,
args,
userAgent: process.env.AGENT_BROWSER_USER_AGENT,
proxy,
ignoreHTTPSErrors: ignoreHTTPSErrors,
});
}
}
// Handle close command specially
// Handle close command specially - shuts down daemon
if (parseResult.command.action === 'close') {
const response = await executeCommand(parseResult.command, browser);
const response =
isIOS && manager instanceof IOSManager
? await executeIOSCommand(parseResult.command, manager)
: await executeCommand(parseResult.command, manager as BrowserManager);
socket.write(serializeResponse(response) + '\n');
if (!shuttingDown) {
@@ -296,7 +346,11 @@ export async function startDaemon(options?: { streamPort?: number }): Promise<vo
return;
}
const response = await executeCommand(parseResult.command, browser);
// Execute command with appropriate handler
const response =
isIOS && manager instanceof IOSManager
? await executeIOSCommand(parseResult.command, manager)
: await executeCommand(parseResult.command, manager as BrowserManager);
socket.write(serializeResponse(response) + '\n');
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
@@ -355,7 +409,7 @@ export async function startDaemon(options?: { streamPort?: number }): Promise<vo
}
}
await browser.close();
await manager.close();
server.close();
cleanupSocket();
process.exit(0);
+273
View File
@@ -0,0 +1,273 @@
/**
* iOS command execution - mirrors actions.ts but for iOS Safari via Appium.
* Provides 1:1 command parity where possible.
*/
import type { IOSManager } from './ios-manager.js';
import type { Command, Response } from './types.js';
function successResponse<T>(id: string, data: T): Response<T> {
return { id, success: true, data };
}
function errorResponse(id: string, error: string): Response {
return { id, success: false, error };
}
/**
* Execute a command on the iOS manager
*/
export async function executeIOSCommand(command: Command, manager: IOSManager): Promise<Response> {
const { id, action } = command;
try {
switch (action) {
case 'launch': {
const cmd = command as any;
await manager.launch({
device: cmd.device,
udid: cmd.udid,
});
const info = manager.getDeviceInfo();
return successResponse(id, {
launched: true,
device: info?.name ?? 'iOS Simulator',
udid: info?.udid,
});
}
case 'navigate': {
const cmd = command as any;
const result = await manager.navigate(cmd.url);
return successResponse(id, result);
}
case 'click': {
const cmd = command as any;
await manager.click(cmd.selector);
return successResponse(id, { clicked: true });
}
case 'tap': {
const cmd = command as any;
await manager.tap(cmd.selector);
return successResponse(id, { tapped: true });
}
case 'type': {
const cmd = command as any;
await manager.type(cmd.selector, cmd.text, {
delay: cmd.delay,
clear: cmd.clear,
});
return successResponse(id, { typed: true });
}
case 'fill': {
const cmd = command as any;
await manager.fill(cmd.selector, cmd.value);
return successResponse(id, { filled: true });
}
case 'screenshot': {
const cmd = command as any;
const result = await manager.screenshot({
path: cmd.path,
fullPage: cmd.fullPage,
});
return successResponse(id, result);
}
case 'snapshot': {
const cmd = command as any;
const result = await manager.getSnapshot({
interactive: cmd.interactive,
});
return successResponse(id, { snapshot: result.tree, refs: result.refs });
}
case 'scroll': {
const cmd = command as any;
await manager.scroll({
selector: cmd.selector,
x: cmd.x,
y: cmd.y,
direction: cmd.direction,
amount: cmd.amount,
});
return successResponse(id, { scrolled: true });
}
case 'swipe': {
const cmd = command as any;
await manager.swipe(cmd.direction, { distance: cmd.distance });
return successResponse(id, { swiped: true });
}
case 'evaluate': {
const cmd = command as any;
const result = await manager.evaluate(cmd.script, ...(cmd.args ?? []));
return successResponse(id, { result });
}
case 'wait': {
const cmd = command as any;
await manager.wait({
selector: cmd.selector,
timeout: cmd.timeout,
state: cmd.state,
});
return successResponse(id, { waited: true });
}
case 'press': {
const cmd = command as any;
await manager.press(cmd.key);
return successResponse(id, { pressed: true });
}
case 'hover': {
const cmd = command as any;
await manager.hover(cmd.selector);
return successResponse(id, { hovered: true });
}
case 'content': {
const cmd = command as any;
const html = await manager.getContent(cmd.selector);
return successResponse(id, { html });
}
case 'gettext': {
const cmd = command as any;
const text = await manager.getText(cmd.selector);
return successResponse(id, { text });
}
case 'getattribute': {
const cmd = command as any;
const value = await manager.getAttribute(cmd.selector, cmd.attribute);
return successResponse(id, { value });
}
case 'isvisible': {
const cmd = command as any;
const visible = await manager.isVisible(cmd.selector);
return successResponse(id, { visible });
}
case 'isenabled': {
const cmd = command as any;
const enabled = await manager.isEnabled(cmd.selector);
return successResponse(id, { enabled });
}
case 'url': {
const url = await manager.getUrl();
return successResponse(id, { url });
}
case 'title': {
const title = await manager.getTitle();
return successResponse(id, { title });
}
case 'back': {
await manager.goBack();
return successResponse(id, { navigated: 'back' });
}
case 'forward': {
await manager.goForward();
return successResponse(id, { navigated: 'forward' });
}
case 'reload': {
await manager.reload();
return successResponse(id, { reloaded: true });
}
case 'select': {
const cmd = command as any;
await manager.select(cmd.selector, cmd.values);
return successResponse(id, { selected: true });
}
case 'check': {
const cmd = command as any;
await manager.check(cmd.selector);
return successResponse(id, { checked: true });
}
case 'uncheck': {
const cmd = command as any;
await manager.uncheck(cmd.selector);
return successResponse(id, { unchecked: true });
}
case 'focus': {
const cmd = command as any;
await manager.focus(cmd.selector);
return successResponse(id, { focused: true });
}
case 'clear': {
const cmd = command as any;
await manager.clear(cmd.selector);
return successResponse(id, { cleared: true });
}
case 'count': {
const cmd = command as any;
const count = await manager.count(cmd.selector);
return successResponse(id, { count });
}
case 'boundingbox': {
const cmd = command as any;
const box = await manager.getBoundingBox(cmd.selector);
return successResponse(id, { box });
}
case 'close': {
await manager.close();
return successResponse(id, { closed: true });
}
// iOS-specific: device list
case 'device_list': {
const devices = await manager.listDevices();
return successResponse(id, { devices });
}
// Commands that don't apply to iOS Safari
case 'tab_new':
case 'tab_list':
case 'tab_switch':
case 'tab_close':
case 'window_new':
return errorResponse(
id,
`Command '${action}' is not supported on iOS Safari. Mobile Safari does not support programmatic tab management.`
);
case 'pdf':
return errorResponse(id, 'PDF generation is not supported on iOS Safari.');
case 'screencast_start':
case 'screencast_stop':
return errorResponse(id, 'Screencast is not supported on iOS (requires CDP).');
case 'recording_start':
case 'recording_stop':
case 'recording_restart':
return errorResponse(id, 'Video recording is not yet supported on iOS.');
default:
return errorResponse(id, `Unknown or unsupported iOS command: ${action}`);
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return errorResponse(id, message);
}
}
+157
View File
@@ -0,0 +1,157 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { IOSManager } from './ios-manager.js';
// Mock node-simctl
vi.mock('node-simctl', () => {
return {
Simctl: class MockSimctl {
async getDevices() {
return {
'iOS 18.0': [
{
name: 'iPhone 16 Pro',
udid: 'TEST-UDID-1234',
state: 'Shutdown',
isAvailable: true,
},
{
name: 'iPhone 16',
udid: 'TEST-UDID-5678',
state: 'Booted',
isAvailable: true,
},
{
name: 'iPad Pro',
udid: 'TEST-UDID-IPAD',
state: 'Shutdown',
isAvailable: true,
},
],
};
}
},
};
});
describe('IOSManager', () => {
let manager: IOSManager;
beforeEach(() => {
manager = new IOSManager();
});
describe('listDevices', () => {
it('should list available iOS simulators', async () => {
const devices = await manager.listDevices();
expect(devices).toHaveLength(3);
expect(devices[0]).toEqual({
name: 'iPhone 16 Pro',
udid: 'TEST-UDID-1234',
state: 'Shutdown',
runtime: 'iOS 18.0',
isAvailable: true,
isRealDevice: false,
});
});
it('should include runtime version for each device', async () => {
const devices = await manager.listDevices();
devices.forEach((device) => {
expect(device.runtime).toBe('iOS 18.0');
});
});
});
describe('isLaunched', () => {
it('should return false when browser is not launched', () => {
expect(manager.isLaunched()).toBe(false);
});
});
describe('getRefData', () => {
it('should return null for unknown refs', () => {
// Access private method via bracket notation for testing
const result = (manager as any).getRefData('@e99');
expect(result).toBeNull();
});
it('should handle @-prefixed refs', () => {
// Set up a ref in the refMap
(manager as any).refMap = {
e1: { selector: 'button', role: 'button', name: 'Submit' },
};
const result = (manager as any).getRefData('@e1');
expect(result).toEqual({ selector: 'button', role: 'button', name: 'Submit' });
});
it('should handle ref= prefixed refs', () => {
(manager as any).refMap = {
e2: { selector: 'a', role: 'link', name: 'Learn more' },
};
const result = (manager as any).getRefData('ref=e2');
expect(result).toEqual({ selector: 'a', role: 'link', name: 'Learn more' });
});
it('should handle bare ref names', () => {
(manager as any).refMap = {
e3: { selector: 'input', role: 'textbox', name: 'Email' },
};
const result = (manager as any).getRefData('e3');
expect(result).toEqual({ selector: 'input', role: 'textbox', name: 'Email' });
});
});
});
describe('IOSManager integration', () => {
// These tests require Appium and iOS Simulator to be available
// They are skipped by default and can be run manually
describe.skip('with real simulator', () => {
let manager: IOSManager;
beforeEach(() => {
// Use real implementation for integration tests
vi.resetModules();
manager = new IOSManager();
});
it('should launch Safari and navigate', async () => {
await manager.launch({ device: 'iPhone 16 Pro' });
expect(manager.isLaunched()).toBe(true);
const result = await manager.navigate('https://example.com');
expect(result.url).toContain('example.com');
expect(result.title).toBe('Example Domain');
await manager.close();
}, 120000);
it('should take screenshots', async () => {
await manager.launch({ device: 'iPhone 16 Pro' });
await manager.navigate('https://example.com');
const result = await manager.screenshot();
expect(result.base64).toBeDefined();
expect(result.base64?.length).toBeGreaterThan(1000);
await manager.close();
}, 120000);
it('should generate snapshots with refs', async () => {
await manager.launch({ device: 'iPhone 16 Pro' });
await manager.navigate('https://example.com');
const snapshot = await manager.getSnapshot();
expect(snapshot.tree).toContain('link');
expect(snapshot.tree).toContain('[ref=e1]');
expect(snapshot.refs.e1).toBeDefined();
expect(snapshot.refs.e1.role).toBe('link');
await manager.close();
}, 120000);
});
});
+1299
View File
File diff suppressed because it is too large Load Diff
+13
View File
@@ -686,6 +686,17 @@ const inputTouchSchema = baseCommandSchema.extend({
modifiers: z.number().optional(),
});
// iOS-specific schemas
const swipeSchema = baseCommandSchema.extend({
action: z.literal('swipe'),
direction: z.enum(['up', 'down', 'left', 'right']),
distance: z.number().positive().optional(),
});
const deviceListSchema = baseCommandSchema.extend({
action: z.literal('device_list'),
});
const pressSchema = baseCommandSchema.extend({
action: z.literal('press'),
key: z.string().min(1),
@@ -906,6 +917,8 @@ const commandSchema = z.discriminatedUnion('action', [
inputMouseSchema,
inputKeyboardSchema,
inputTouchSchema,
swipeSchema,
deviceListSchema,
]);
// Parse result type
+14 -1
View File
@@ -521,6 +521,17 @@ export interface InputTouchCommand extends BaseCommand {
modifiers?: number;
}
// iOS-specific commands
export interface SwipeCommand extends BaseCommand {
action: 'swipe';
direction: 'up' | 'down' | 'left' | 'right';
distance?: number;
}
export interface DeviceListCommand extends BaseCommand {
action: 'device_list';
}
// Video recording (Playwright native - requires launch-time setup)
export interface VideoStartCommand extends BaseCommand {
action: 'video_start';
@@ -931,7 +942,9 @@ export type Command =
| ScreencastStopCommand
| InputMouseCommand
| InputKeyboardCommand
| InputTouchCommand;
| InputTouchCommand
| SwipeCommand
| DeviceListCommand;
// Response types
export interface SuccessResponse<T = unknown> {