fix(limrun): list the tabs of a collapsed iOS tab bar in snapshots

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift

@@ -412,6 +412,13 @@ extension RunnerTests {
     return area < containerArea * 0.9 && containerFrame.contains(CGPoint(x: frame.midX, y: frame.midY))
   }
 
+  /// Whether a collapsed tab container's items sit in one row: their vertical midpoints spread no
+  /// more than modest jitter or a short two-row wrap allows, which still rejects unrelated controls.
+  static func collapsedTabRowFits(midpoints: [Double], containerHeight: Double) -> Bool {
+    let rowSpread = (midpoints.max() ?? 0) - (midpoints.min() ?? 0)
+    return rowSpread <= max(24.0, containerHeight * 0.6)
+  }
+
   /// Whether an element names something an agent can address: a label, identifier or value that is
   /// not blank. Candidate types carry no placeholder.
   static func hasCollapsedTabContent(_ attributes: XCUIElementAttributes) -> Bool {
@@ -449,9 +456,9 @@ extension RunnerTests {
 
     if candidates.count < 2 { return [] }
     let rowMidpoints = candidates.map { $0.rect.y + ($0.rect.height / 2) }
-    let rowSpread = (rowMidpoints.max() ?? 0) - (rowMidpoints.min() ?? 0)
-    // Allow modest vertical jitter and short two-row wraps while still rejecting unrelated controls.
-    if rowSpread > max(24.0, Double(containerFrame.height) * 0.6) { return [] }
+    if !Self.collapsedTabRowFits(midpoints: rowMidpoints, containerHeight: Double(containerFrame.height)) {
+      return []
+    }
 
     var seen = Set<String>()
     let uniqueCandidates = candidates.filter { node in

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotAcquisitionTests.swift

@@ -66,5 +66,67 @@ extension RunnerTests {
     XCTAssertFalse(predicate.evaluate(with: ElementAttributesFixture(.button, CGRect(x: 30, y: 780, width: 100, height: 30), label: "Left")))
     XCTAssertEqual(matched, [], "a refused element's attributes are not recorded")
   }
+
+  /// The Limrun provider applies the same geometry to its tab bar hit-tests and reads this table too.
+  @MainActor
+  func testCollapsedTabGeometryMatchesGoldenFixture() throws {
+    let fixtureURL = URL(fileURLWithPath: #filePath)
+      .deletingLastPathComponent()
+      .deletingLastPathComponent()
+      .deletingLastPathComponent()
+      .deletingLastPathComponent()
+      .deletingLastPathComponent()
+      .deletingLastPathComponent()
+      .appendingPathComponent("contracts/fixtures/ios-collapsed-tab-geometry.json")
+    let fixture = try JSONDecoder().decode(CollapsedTabGeometryFixture.self, from: Data(contentsOf: fixtureURL))
+    XCTAssertFalse(fixture.containers.isEmpty)
+    XCTAssertFalse(fixture.candidates.isEmpty)
+    XCTAssertFalse(fixture.rows.isEmpty)
+    for row in fixture.containers {
+      XCTAssertEqual(isCollapsedTabContainer(ElementAttributesFixture(.tabBar, row.frame.cgRect)), row.expected, row.name)
+    }
+    for row in fixture.candidates {
+      XCTAssertEqual(
+        Self.isCollapsedTabCandidate(type: .button, frame: row.frame.cgRect, in: row.container.cgRect),
+        row.expected, row.name
+      )
+    }
+    for row in fixture.rows {
+      XCTAssertEqual(
+        Self.collapsedTabRowFits(midpoints: row.midpoints, containerHeight: row.container.height),
+        row.expected, row.name
+      )
+    }
+  }
+}
+
+private struct CollapsedTabGeometryFixture: Decodable {
+  struct Rect: Decodable {
+    let x: Double
+    let y: Double
+    let width: Double
+    let height: Double
+    var cgRect: CGRect { CGRect(x: x, y: y, width: width, height: height) }
+  }
+  struct Container: Decodable {
+    let name: String
+    let frame: Rect
+    let expected: Bool
+  }
+  struct Candidate: Decodable {
+    let name: String
+    let container: Rect
+    let frame: Rect
+    let expected: Bool
+  }
+  struct Row: Decodable {
+    let name: String
+    let container: Rect
+    let midpoints: [Double]
+    let expected: Bool
+  }
+  let containers: [Container]
+  let candidates: [Candidate]
+  let rows: [Row]
 }
 #endif

contracts/fixtures/ios-collapsed-tab-geometry.json

@@ -0,0 +1,93 @@
+{
+  "containers": [
+    {
+      "name": "a full-width iPhone tab bar",
+      "frame": { "x": 0, "y": 791, "width": 402, "height": 83 },
+      "expected": true
+    },
+    {
+      "name": "exactly 1.75 times as wide as it is tall",
+      "frame": { "x": 0, "y": 700, "width": 350, "height": 200 },
+      "expected": true
+    },
+    {
+      "name": "narrower than 160 points",
+      "frame": { "x": 0, "y": 834, "width": 150, "height": 40 },
+      "expected": false
+    },
+    {
+      "name": "narrower than 1.75 times its height",
+      "frame": { "x": 0, "y": 694, "width": 300, "height": 180 },
+      "expected": false
+    },
+    {
+      "name": "an empty frame",
+      "frame": { "x": 0, "y": 874, "width": 402, "height": 0 },
+      "expected": false
+    }
+  ],
+  "candidates": [
+    {
+      "name": "an item centered inside the container",
+      "container": { "x": 20, "y": 800, "width": 360, "height": 31 },
+      "frame": { "x": 30, "y": 800, "width": 100, "height": 30 },
+      "expected": true
+    },
+    {
+      "name": "the container's own frame",
+      "container": { "x": 20, "y": 800, "width": 360, "height": 31 },
+      "frame": { "x": 20, "y": 800, "width": 360, "height": 31 },
+      "expected": false
+    },
+    {
+      "name": "a wrapper centered on the container",
+      "container": { "x": 20, "y": 800, "width": 360, "height": 31 },
+      "frame": { "x": 0, "y": 515, "width": 400, "height": 600 },
+      "expected": false
+    },
+    {
+      "name": "an item overlapping the container with its center outside",
+      "container": { "x": 20, "y": 800, "width": 360, "height": 31 },
+      "frame": { "x": 30, "y": 780, "width": 100, "height": 30 },
+      "expected": false
+    },
+    {
+      "name": "an item with an empty frame",
+      "container": { "x": 20, "y": 800, "width": 360, "height": 31 },
+      "frame": { "x": 30, "y": 800, "width": 0, "height": 30 },
+      "expected": false
+    }
+  ],
+  "rows": [
+    {
+      "name": "items in one row",
+      "container": { "x": 0, "y": 791, "width": 402, "height": 83 },
+      "midpoints": [822, 822, 822],
+      "expected": true
+    },
+    {
+      "name": "a spread within 0.6 of a tall container",
+      "container": { "x": 0, "y": 791, "width": 402, "height": 83 },
+      "midpoints": [800, 849],
+      "expected": true
+    },
+    {
+      "name": "a spread past 0.6 of a tall container",
+      "container": { "x": 0, "y": 791, "width": 402, "height": 83 },
+      "midpoints": [800, 850],
+      "expected": false
+    },
+    {
+      "name": "a spread of 24 points in a short container",
+      "container": { "x": 0, "y": 800, "width": 402, "height": 31 },
+      "midpoints": [805, 829],
+      "expected": true
+    },
+    {
+      "name": "a spread past 24 points in a short container",
+      "container": { "x": 0, "y": 800, "width": 402, "height": 31 },
+      "midpoints": [805, 830],
+      "expected": false
+    }
+  ]
+}

packages/capture-kit/src/ios-snapshot-acquisition.ts

@@ -1,6 +1,7 @@
 import type { SnapshotRuntimeAcquiredResult } from '@agent-device/contracts/interactor-types';
 import type {
   IosAcquisitionResidue,
+  IosCollapsedTabBarProbeFailure,
   IosProviderAcquisitionCapabilities,
   IosProviderAcquisitionProducer,
   IosSnapshotEvidenceAvailability,
@@ -75,6 +76,7 @@ export function iosSnapshotTruncationEvidence(
 function deriveIosSnapshotAcquisitionResidue(
   producer: IosProviderAcquisitionCapabilities,
   viewport: IosViewportEvidence,
+  collapsedTabBarProbeFailure: IosCollapsedTabBarProbeFailure | undefined,
 ): readonly IosAcquisitionResidue[] {
   const residue: IosAcquisitionResidue[] = [];
   if (producer.hittabilityEvidence === 'unavailable') {
@@ -92,6 +94,9 @@ function deriveIosSnapshotAcquisitionResidue(
   if (viewport.kind === 'missing') {
     residue.push({ kind: 'missing-viewport', reason: viewport.reason });
   }
+  if (collapsedTabBarProbeFailure) {
+    residue.push({ kind: 'collapsed-tab-bar-probe-failed', reason: collapsedTabBarProbeFailure });
+  }
   return Object.freeze(residue);
 }
 
@@ -102,6 +107,11 @@ export function createIosSnapshotAcquisition(
     viewport: IosViewportEvidence;
     keyboard?: SnapshotKeyboardBandFact;
     lineage: IosSnapshotLineage;
+    /**
+     * Set when the provider left a collapsed tab bar unexpanded: a hit-test was rejected or timed
+     * out, or the hit-test budget ran out before a candidate with a tab bar's height was checked.
+     */
+    collapsedTabBarProbeFailure?: IosCollapsedTabBarProbeFailure;
   }>,
 ): SnapshotRuntimeAcquiredResult {
   const producer = IOS_PROVIDER_ACQUISITION_CAPABILITY_VALUES[input.producer];
@@ -114,7 +124,11 @@ export function createIosSnapshotAcquisition(
       viewport: input.viewport,
       ...(input.keyboard ? { keyboard: input.keyboard } : {}),
       lineage: input.lineage,
-      residue: deriveIosSnapshotAcquisitionResidue(producer, input.viewport),
+      residue: deriveIosSnapshotAcquisitionResidue(
+        producer,
+        input.viewport,
+        input.collapsedTabBarProbeFailure,
+      ),
     },
   };
 }

packages/capture-kit/src/ios-snapshot-planning.ts

@@ -96,5 +96,7 @@ function residueIdentity(residue: IosAcquisitionResidue): string {
       return JSON.stringify({ kind: residue.kind, fact: residue.fact });
     case 'fallback-source':
       return JSON.stringify({ kind: residue.kind, producer: residue.producer });
+    case 'collapsed-tab-bar-probe-failed':
+      return JSON.stringify({ kind: residue.kind, reason: residue.reason });
   }
 }

packages/capture-kit/src/snapshot/ios-snapshot-runtime.ts

@@ -106,6 +106,11 @@ function snapshotWarnings(residue: readonly IosAcquisitionResidue[]): { warnings
           `iOS snapshot acquisition did not provide a valid viewport (${entry.reason}); retry with --raw to inspect the acquired tree, while regular presentation requires viewport evidence.`,
         ];
       }
+      if (entry.kind === 'collapsed-tab-bar-probe-failed') {
+        return [
+          `iOS snapshot could not read the tabs of a collapsed tab bar (${entry.reason}); the bar is listed without its tabs.${entry.reason === 'budget-exhausted' ? '' : ' Retry the snapshot.'}`,
+        ];
+      }
       if (entry.kind !== 'unavailable-fact') return [];
       const warning = IOS_SNAPSHOT_FACT_WARNINGS[entry.fact];
       return warning ? [warning] : [];

packages/contracts/src/ios-snapshot.ts

@@ -124,6 +124,15 @@ export type IosHittabilityEvidence =
 
 export type IosProviderPrunedField = 'nodes' | 'depth' | 'scope' | 'interactive-only';
 
+/**
+ * Why a provider left a collapsed tab bar unexpanded:
+ * - `rejected`: a hit-test across the bar failed.
+ * - `timed-out`: a hit-test across the bar did not answer in time.
+ * - `budget-exhausted`: the hit-test budget ran out before a candidate with a tab bar's height
+ *   was checked.
+ */
+export type IosCollapsedTabBarProbeFailure = 'rejected' | 'timed-out' | 'budget-exhausted';
+
 export type IosAcquisitionResidue =
   | Readonly<{
       kind: 'provider-pruned';
@@ -152,6 +161,10 @@ export type IosAcquisitionResidue =
   | Readonly<{
       kind: 'fallback-source';
       producer: IosSnapshotProducer;
+    }>
+  | Readonly<{
+      kind: 'collapsed-tab-bar-probe-failed';
+      reason: IosCollapsedTabBarProbeFailure;
     }>;
 
 type IosSnapshotAcquisitionForIntent<Intent extends IosAcquisitionIntent> = Readonly<{