From 88f1a19a4751947da70236e28f214f6604966a62 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 15 May 2026 14:27:24 -0400 Subject: [PATCH 1/4] feat(camera): add iOS simulator camera simulation --- README.md | 13 + docs/api/rest.md | 29 + docs/cli/commands.md | 19 + docs/cli/flags.md | 9 + docs/cli/index.md | 2 + docs/guide/testing.md | 16 + package.json | 2 + packages/client/src/api/simulators.ts | 54 + packages/client/src/api/types.ts | 40 + packages/client/src/app/AppShell.tsx | 14 + .../simulators/CameraSimulationModal.tsx | 393 ++++++ .../src/features/simulators/SimulatorMenu.tsx | 11 + .../client/src/features/toolbar/Toolbar.tsx | 3 + packages/client/src/styles/components.css | 15 + .../camera/SimDeckCameraHelper-Info.plist | 22 + .../camera/SimDeckCameraHelper.entitlements | 8 + .../native/camera/SimDeckCameraHelper.m | 782 ++++++++++++ .../native/camera/SimDeckCameraInjector.m | 1066 +++++++++++++++++ .../native/camera/SimDeckCameraShared.h | 34 + packages/server/native/camera/build-helper.sh | 39 + .../server/native/camera/build-injector.sh | 38 + packages/server/src/api/routes.rs | 79 ++ packages/server/src/camera.rs | 761 ++++++++++++ packages/server/src/main.rs | 113 ++ scripts/build-cli.sh | 9 + scripts/integration/camera.mjs | 621 ++++++++++ skills/simdeck/SKILL.md | 25 + 27 files changed, 4217 insertions(+) create mode 100644 packages/client/src/features/simulators/CameraSimulationModal.tsx create mode 100644 packages/server/native/camera/SimDeckCameraHelper-Info.plist create mode 100644 packages/server/native/camera/SimDeckCameraHelper.entitlements create mode 100644 packages/server/native/camera/SimDeckCameraHelper.m create mode 100644 packages/server/native/camera/SimDeckCameraInjector.m create mode 100644 packages/server/native/camera/SimDeckCameraShared.h create mode 100755 packages/server/native/camera/build-helper.sh create mode 100755 packages/server/native/camera/build-injector.sh create mode 100644 packages/server/src/camera.rs create mode 100644 scripts/integration/camera.mjs diff --git a/README.md b/README.md index b2c54504..4fd5fe90 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ view inside the editor. - Real-time screen `describe` command using accessibility view tree - available in token-efficient format for agents - Profiling built-in: CPU, memory, disk writes, network throughput, hang signals, and stack sampling - CoreSimulator chrome asset rendering for device bezels +- iOS Simulator camera simulation from a generated pattern, local media file or stream URL, or a Mac camera source - NativeScript, React Native, Flutter, UIKit and SwiftUI runtime inspector plugins to debug app's view hierarchy live - `simdeck/test` for fast JS-based app tests that can query accessibility state and drive simulator controls @@ -118,6 +119,11 @@ simdeck install android: /path/to/app.apk simdeck uninstall com.example.App simdeck open-url https://example.com simdeck launch com.apple.Preferences +simdeck camera sources +simdeck camera start com.example.App --file /absolute/path/to/camera.mov +simdeck camera start com.example.App --webcam +simdeck camera switch --placeholder +simdeck camera stop simdeck toggle-appearance simdeck pasteboard set "hello" simdeck pasteboard get @@ -179,6 +185,13 @@ directory. Most device commands accept `[]`; when it is omitted, SimDeck uses `--device`, `SIMDECK_DEVICE`, `SIMDECK_UDID`, the saved project default, or the only booted simulator, in that order. +`camera start` runs a local camera helper, injects the SimDeck camera shim into +the target iOS simulator app, and relaunches that bundle. The source can be a +generated pattern, an absolute image or video path, an `http://`, `https://`, or +`file://` video URL, or a Mac camera selected with `--webcam [id-or-name]`. +Use the browser menu item **Camera Simulation...** for the same flow from the UI. +Camera simulation is iOS-simulator-only and requires a booted simulator. + ## JS/TS Tests ```ts diff --git a/docs/api/rest.md b/docs/api/rest.md index 87449357..e21086f0 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -115,6 +115,35 @@ Launch apps and open URLs through `/api/simulators/{udid}/action` with `{ "action": "launch", "bundleId": "com.example.App" }` or `{ "action": "openUrl", "url": "https://example.com" }`. +## Camera Simulation + +| Method | Path | Purpose | +| -------- | -------------------------------------- | ------------------------------------------- | +| `GET` | `/api/camera/webcams` | List available Mac camera sources | +| `GET` | `/api/simulators/{udid}/camera` | Get camera helper status | +| `POST` | `/api/simulators/{udid}/camera` | Start helper and optionally relaunch an app | +| `POST` | `/api/simulators/{udid}/camera/source` | Switch the running helper source | +| `DELETE` | `/api/simulators/{udid}/camera` | Stop the camera helper | + +Start request: + +```json +{ + "bundleId": "com.example.App", + "mirror": "off", + "source": { + "kind": "video", + "arg": "/absolute/path/to/feed.mov" + } +} +``` + +Source `kind` is `placeholder`, `image`, `video`, or `webcam`. Image and video +sources require `arg`; local files must be absolute paths. Video sources also +accept `http://`, `https://`, and `file://` URLs. `webcam` can omit `arg` to use +the first available Mac camera, or pass a camera ID/name from `/api/camera/webcams`. +`mirror` is `auto`, `on`, or `off`. + ## Performance iOS simulator app processes run as host macOS processes. These endpoints expose host-process telemetry for matching simulator app PIDs. diff --git a/docs/cli/commands.md b/docs/cli/commands.md index 0e1e918b..213b8431 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -80,6 +80,25 @@ simdeck open-url https://example.com simdeck toggle-appearance ``` +## Camera Simulation + +```sh +simdeck camera sources +simdeck camera start com.example.App --file /absolute/path/to/feed.mov --mirror off +simdeck camera start com.example.App --webcam +simdeck camera switch --placeholder +simdeck camera switch --file /absolute/path/to/frame.png +simdeck camera status +simdeck camera stop +``` + +`camera start` is iOS-simulator-only. It starts the local camera helper, +relaunches the target bundle with the SimDeck camera injector, and makes +`AVCaptureDevice`, `AVCaptureVideoDataOutput`, `AVCapturePhotoOutput`, and +`AVCaptureVideoPreviewLayer` consume the simulated feed. `camera switch` changes +the running helper source without relaunching the app. Media files must use +absolute paths; URL sources are treated as video streams. + ## Inspect UI ```sh diff --git a/docs/cli/flags.md b/docs/cli/flags.md index d8b4cf74..79741f39 100644 --- a/docs/cli/flags.md +++ b/docs/cli/flags.md @@ -88,6 +88,15 @@ button label. | `pasteboard set` | `--stdin`, `--file` | | `batch` | `--step`, `--file`, `--stdin`, `--continue-on-error` | +## Camera Simulation + +| Command | Flags | +| --------------- | ---------------------------------------------------------------------------------- | +| `camera start` | `--file `, `--webcam [id]`, `--mirror auto\|on\|off` | +| `camera switch` | `--file `, `--webcam [id]`, `--placeholder`, `--mirror auto\|on\|off` | +| `camera status` | none | +| `camera stop` | none | + ## Exit codes | Code | Meaning | diff --git a/docs/cli/index.md b/docs/cli/index.md index 973962d6..39bbad48 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -43,6 +43,8 @@ simdeck install /path/to/App.app simdeck install /path/to/App.ipa simdeck launch com.example.App simdeck open-url https://example.com +simdeck camera start com.example.App --file /absolute/path/to/feed.mov +simdeck camera stop simdeck tap --label "Continue" --wait-timeout-ms 5000 simdeck tap --id com.apple.settings.screenTime --expect-id BackButton simdeck tap "Continue" diff --git a/docs/guide/testing.md b/docs/guide/testing.md index df847b77..00c7621a 100644 --- a/docs/guide/testing.md +++ b/docs/guide/testing.md @@ -64,6 +64,21 @@ simdeck maestro test flow.yaml --artifacts-dir artifacts/maestro Supported commands include `launchApp`, `openLink`, `tapOn`, `inputText`, `eraseText`, `pressKey`, `assertVisible`, `assertNotVisible`, `scrollUntilVisible`, `swipe`, `takeScreenshot`, and `waitForAnimationToEnd`. Unsupported Maestro commands fail clearly so the flow can be adjusted or the compatibility layer can be expanded. +## Camera Apps + +For iOS apps that use `AVFoundation`, start camera simulation before running the +camera workflow: + +```sh +simdeck camera start com.example.App --file /absolute/path/to/feed.mov --mirror off +simdeck camera switch --placeholder +simdeck camera stop +``` + +The browser UI exposes the same controls from **Camera Simulation...** in the +simulator menu. Webcam forwarding is available with `--webcam` when macOS has an +available camera and has granted the helper permission. + ## Repository tests Normal unit and client tests: @@ -79,6 +94,7 @@ npm run build:cli npm run build:client npm run test:integration:fixture npm run test:integration:cli +npm run test:integration:camera ``` Verbose iOS run: diff --git a/package.json b/package.json index 3bb89751..0e281501 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "scripts/studio-provider-bridge.mjs", "scripts/postinstall.mjs", "build/simdeck-bin", + "build/camera/", "packages/client/dist/", "packages/simdeck-test/dist/" ], @@ -68,6 +69,7 @@ "test:integration:cli": "node scripts/integration/cli.mjs", "test:integration:cli:verbose": "SIMDECK_INTEGRATION_VERBOSE=1 SIMDECK_INTEGRATION_SHOW_SIMULATOR=1 node scripts/integration/cli.mjs", "test:integration:service": "node scripts/integration/service-restart.mjs", + "test:integration:camera": "node scripts/integration/camera.mjs", "test:integration:fixture": "node scripts/integration/prebuild-fixture.mjs", "test:integration:js-api": "node scripts/integration/js-api.mjs", "test:integration:webrtc": "node scripts/integration/webrtc.mjs", diff --git a/packages/client/src/api/simulators.ts b/packages/client/src/api/simulators.ts index 98cd4fbf..1e31e932 100644 --- a/packages/client/src/api/simulators.ts +++ b/packages/client/src/api/simulators.ts @@ -2,6 +2,9 @@ import { apiRequest } from "./client"; import type { AccessibilitySourcePreference, AccessibilityTreeResponse, + CameraStartRequest, + CameraStatusResponse, + CameraWebcamsResponse, ChromeDevToolsTargetDiscovery, ChromeProfile, CreateSimulatorRequest, @@ -175,6 +178,57 @@ export async function sampleSimulatorProcess( ); } +export async function fetchCameraWebcams( + options: RequestInit = {}, +): Promise { + return apiRequest("/api/camera/webcams", options); +} + +export async function fetchCameraStatus( + udid: string, + options: RequestInit = {}, +): Promise { + return apiRequest( + `/api/simulators/${encodeURIComponent(udid)}/camera`, + options, + ); +} + +export async function startCameraSimulation( + udid: string, + payload: CameraStartRequest, +): Promise { + return apiRequest( + `/api/simulators/${encodeURIComponent(udid)}/camera`, + { + body: JSON.stringify(payload), + method: "POST", + }, + ); +} + +export async function switchCameraSimulationSource( + udid: string, + payload: Pick, +): Promise { + return apiRequest( + `/api/simulators/${encodeURIComponent(udid)}/camera/source`, + { + body: JSON.stringify(payload), + method: "POST", + }, + ); +} + +export async function stopCameraSimulation( + udid: string, +): Promise { + return apiRequest( + `/api/simulators/${encodeURIComponent(udid)}/camera`, + { method: "DELETE" }, + ); +} + export async function fetchWebKitTargets( udid: string, options: RequestInit = {}, diff --git a/packages/client/src/api/types.ts b/packages/client/src/api/types.ts index 052d3551..dc170b85 100644 --- a/packages/client/src/api/types.ts +++ b/packages/client/src/api/types.ts @@ -218,6 +218,46 @@ export interface InstallUploadResponse { udid: string; } +export type CameraSourceKind = "placeholder" | "image" | "video" | "webcam"; + +export interface CameraSourceRequest { + kind: CameraSourceKind; + arg?: string; +} + +export interface CameraStartRequest { + bundleId?: string; + source: CameraSourceRequest; + mirror?: "auto" | "on" | "off"; +} + +export interface CameraWebcam { + id: string; + name: string; + position?: string; +} + +export interface CameraWebcamsResponse { + webcams: CameraWebcam[]; +} + +export interface CameraStatusResponse { + ok?: boolean; + udid?: string; + alive: boolean; + source?: CameraSourceKind | string; + arg?: string; + sourceLabel?: string; + mirror?: "auto" | "on" | "off" | string; + helperPid?: number; + bundleIds?: string[]; + width?: number; + height?: number; + sequence?: number; + helperLogPath?: string; + error?: string; +} + export interface SimulatorForegroundApp { appName?: string | null; bundleIdentifier?: string | null; diff --git a/packages/client/src/app/AppShell.tsx b/packages/client/src/app/AppShell.tsx index 3848b3a7..07f35ba6 100644 --- a/packages/client/src/app/AppShell.tsx +++ b/packages/client/src/app/AppShell.tsx @@ -73,6 +73,7 @@ import type { ViewMode, } from "../features/viewport/types"; import { useViewportLayout } from "../features/viewport/useViewportLayout"; +import { CameraSimulationModal } from "../features/simulators/CameraSimulationModal"; import { NewSimulatorModal } from "../features/simulators/NewSimulatorModal"; import { nextViewportWheelPanState } from "../features/viewport/viewportWheel"; import { @@ -491,6 +492,7 @@ export function AppShell({ const [menuOpen, setMenuOpen] = useState(false); const [simulatorMenuOpen, setSimulatorMenuOpen] = useState(false); const [newSimulatorOpen, setNewSimulatorOpen] = useState(false); + const [cameraSimulationOpen, setCameraSimulationOpen] = useState(false); const [localError, setLocalError] = useState(""); const [captureStatus, setCaptureStatus] = useState( null, @@ -2872,6 +2874,10 @@ export function AppShell({ } }} onInstallAppPrompt={openInstallAppPicker} + onOpenCameraSimulation={() => { + setMenuOpen(false); + setCameraSimulationOpen(true); + }} onOpenAppSwitcher={() => { if (!selectedSimulator) { return; @@ -2991,6 +2997,14 @@ export function AppShell({ open={newSimulatorOpen && !hideSimulatorSelection} selectedSimulator={selectedSimulator} /> + setCameraSimulationOpen(false)} + open={cameraSimulationOpen} + selectedSimulator={selectedSimulator} + /> void; + open: boolean; + selectedSimulator: SimulatorMetadata | null; +} + +type SourceMode = "placeholder" | "webcam" | "media"; +type MirrorMode = "auto" | "on" | "off"; + +export function CameraSimulationModal({ + foregroundBundleId, + onClose, + open, + selectedSimulator, +}: CameraSimulationModalProps) { + const [bundleId, setBundleId] = useState(""); + const [sourceMode, setSourceMode] = useState("placeholder"); + const [mediaPath, setMediaPath] = useState(""); + const [webcamId, setWebcamId] = useState(""); + const [mirror, setMirror] = useState("auto"); + const [status, setStatus] = useState(null); + const [webcams, setWebcams] = useState([]); + const [isLoading, setIsLoading] = useState(false); + const [isApplying, setIsApplying] = useState(false); + const [isStopping, setIsStopping] = useState(false); + const [error, setError] = useState(""); + + const udid = selectedSimulator?.udid ?? ""; + const canApply = Boolean( + selectedSimulator?.isBooted && + bundleId.trim() && + (sourceMode !== "media" || mediaPath.trim()), + ); + + useEffect(() => { + if (!open) { + return; + } + setBundleId(foregroundBundleId ?? ""); + setError(""); + setIsApplying(false); + setIsStopping(false); + void refreshStatus(); + void refreshWebcams(); + }, [foregroundBundleId, open, udid]); + + useEffect(() => { + if (!open) { + return; + } + function handleKeyDown(event: KeyboardEvent) { + if (event.key === "Escape") { + onClose(); + } + } + window.addEventListener("keydown", handleKeyDown); + return () => window.removeEventListener("keydown", handleKeyDown); + }, [onClose, open]); + + const activeBundleText = useMemo(() => { + const bundleIds = status?.bundleIds ?? []; + if (bundleIds.length === 0) { + return status?.alive ? "helper running" : "not running"; + } + return bundleIds.join(", "); + }, [status]); + + if (!open) { + return null; + } + + async function refreshStatus() { + if (!udid) { + setStatus(null); + return; + } + setIsLoading(true); + try { + const nextStatus = await fetchCameraStatus(udid); + setStatus(nextStatus); + if (nextStatus.mirror === "on" || nextStatus.mirror === "off") { + setMirror(nextStatus.mirror); + } + if ( + nextStatus.source === "webcam" || + nextStatus.source === "placeholder" + ) { + setSourceMode(nextStatus.source); + } else if ( + nextStatus.source === "image" || + nextStatus.source === "video" + ) { + setSourceMode("media"); + } + if (nextStatus.arg) { + if (nextStatus.source === "webcam") { + setWebcamId(nextStatus.arg); + } else if ( + nextStatus.source === "image" || + nextStatus.source === "video" + ) { + setMediaPath(nextStatus.arg); + } + } + } catch (statusError) { + setStatus(null); + setError( + statusError instanceof Error + ? statusError.message + : "Unable to load camera status.", + ); + } finally { + setIsLoading(false); + } + } + + async function refreshWebcams() { + try { + const response = await fetchCameraWebcams(); + setWebcams(response.webcams ?? []); + setWebcamId((current) => current || response.webcams?.[0]?.id || ""); + } catch { + setWebcams([]); + } + } + + function requestSource(): { kind: CameraSourceKind; arg?: string } { + if (sourceMode === "webcam") { + return { kind: "webcam", arg: webcamId || undefined }; + } + if (sourceMode === "media") { + const value = mediaPath.trim(); + const kind: CameraSourceKind = looksLikeVideo(value) ? "video" : "image"; + return { kind, arg: value }; + } + return { kind: "placeholder" }; + } + + async function apply(event: FormEvent) { + event.preventDefault(); + if (!selectedSimulator?.isBooted) { + setError( + "Boot the selected simulator before enabling camera simulation.", + ); + return; + } + if (!bundleId.trim()) { + setError( + "Enter the app bundle identifier to relaunch with camera simulation.", + ); + return; + } + setIsApplying(true); + setError(""); + try { + const nextStatus = await startCameraSimulation(udid, { + bundleId: bundleId.trim(), + mirror, + source: requestSource(), + }); + setStatus(nextStatus); + } catch (applyError) { + setError( + applyError instanceof Error + ? applyError.message + : "Unable to start camera simulation.", + ); + } finally { + setIsApplying(false); + } + } + + async function switchSourceOnly() { + if (!status?.alive) { + return; + } + setIsApplying(true); + setError(""); + try { + const nextStatus = await switchCameraSimulationSource(udid, { + mirror, + source: requestSource(), + }); + setStatus(nextStatus); + } catch (switchError) { + setError( + switchError instanceof Error + ? switchError.message + : "Unable to switch camera source.", + ); + } finally { + setIsApplying(false); + } + } + + async function stop() { + setIsStopping(true); + setError(""); + try { + const nextStatus = await stopCameraSimulation(udid); + setStatus(nextStatus); + } catch (stopError) { + setError( + stopError instanceof Error + ? stopError.message + : "Unable to stop camera simulation.", + ); + } finally { + setIsStopping(false); + } + } + + return ( +
{ + if (event.target === event.currentTarget) { + onClose(); + } + }} + > +
+
+
+ +
+
+ + + +
+
+ + {sourceMode === "media" ? ( + + ) : null} + {sourceMode === "webcam" ? ( + + ) : null} + +

+ {isLoading + ? "Loading camera status..." + : `Status: ${activeBundleText}`} +

+
+ {error ?

{error}

: null} +
+ +
+ + + + + +
+
+
+ ); +} + +function looksLikeVideo(value: string): boolean { + if (/^https?:\/\//i.test(value)) { + return true; + } + return /\.(mp4|m4v|mov|qt|avi|mkv|webm|mpg|mpeg|3gp|3g2)$/i.test(value); +} diff --git a/packages/client/src/features/simulators/SimulatorMenu.tsx b/packages/client/src/features/simulators/SimulatorMenu.tsx index 10e7ef71..0347f7bf 100644 --- a/packages/client/src/features/simulators/SimulatorMenu.tsx +++ b/packages/client/src/features/simulators/SimulatorMenu.tsx @@ -24,6 +24,7 @@ interface SimulatorMenuProps { onDismissKeyboard: () => void; onHome: () => void; onInstallAppPrompt: () => void; + onOpenCameraSimulation: () => void; onOpenAppSwitcher: () => void; onOpenBundlePrompt: () => void; onOpenUrlPrompt: () => void; @@ -63,6 +64,7 @@ export function SimulatorMenu({ onDismissKeyboard, onHome, onInstallAppPrompt, + onOpenCameraSimulation, onOpenAppSwitcher, onOpenBundlePrompt, onOpenUrlPrompt, @@ -273,6 +275,15 @@ export function SimulatorMenu({ ? "Stop Recording" : "Start Recording"} +