ci: smoke-run each adapter's desktop example against a local fixture site

.github/scripts/examples-smoke/run-example.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Usage: run-example.sh <name> <command...>; checks the zips the run wrote against smoke/expectations/<name>.json.
+# The exit code is passed to the checker rather than gating here, because most examples fail on purpose.
+set -uo pipefail
+
+NAME="${1:?usage: run-example.sh <name> <command...>}"
+shift
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT="$(cd "$HERE/../../.." && pwd)"
+OUT="${SMOKE_OUT:-${RUNNER_TEMP:-/tmp}/examples-smoke}/$NAME"
+
+rm -rf "$OUT"
+mkdir -p "$OUT"
+# Found by age, not path: each adapter resolves its output dir from the test file, the config or the cwd.
+MARKER="$OUT/.started"
+touch "$MARKER"
+
+echo "::group::$NAME: $*"
+(cd "$ROOT" && "$@") 2>&1 | tee "$OUT/run.log"
+STATUS=${PIPESTATUS[0]}
+echo "::endgroup::"
+echo "$NAME exited $STATUS"
+
+# Only examples/ and packages/ are searched, never .claude/worktrees or $OUT itself; a numbered directory
+# per zip keeps two identically named zips from overwriting each other.
+COUNT=0
+while IFS= read -r -d '' ZIP; do
+  COUNT=$((COUNT + 1))
+  mkdir -p "$OUT/$COUNT"
+  cp "$ZIP" "$OUT/$COUNT/"
+  echo "collected ${ZIP#"$ROOT"/} as $COUNT/"
+done < <(find "$ROOT/examples" "$ROOT/packages" \
+  \( -name node_modules -o -name .claude -o -path "$OUT" \) -prune \
+  -o -type f -name 'trace-*.zip' -newer "$MARKER" -print0)
+
+cd "$ROOT" && pnpm -s smoke:assert --expect "$NAME" --exit-code "$STATUS" \
+  --log "$OUT/run.log" "$OUT"

.github/workflows/ci.yml

@@ -96,6 +96,91 @@ jobs:
       - name: 🧩 Component Tests
         run: pnpm test:ui --maxInstances 2
 
+  # Runs one desktop example per adapter in trace mode against a local fixture
+  # site and checks each trace zip against
+  # packages/conformance/smoke/expectations/. The conformance suite covers the
+  # adapters against a fake browser; this covers the examples users copy, in a
+  # real headless Chrome (preinstalled on the runner image; each framework
+  # resolves its own driver). Mobile and Safari stay manual.
+  examples-smoke:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      # FIXTURE_DEFAULT_HOST/PORT in packages/conformance/smoke/constants.ts.
+      DEVTOOLS_EXAMPLE_BASE_URL: http://127.0.0.1:4567
+      DEVTOOLS_MODE: trace
+      DEVTOOLS_OPEN: '0'
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Use Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version-file: .nvmrc
+          # the runner cache can hold a 24.x below the engines floor
+          check-latest: true
+      - name: Cache pnpm modules
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        env:
+          cache-name: cache-pnpm-modules
+        with:
+          path: ~/.pnpm-store
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+      - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
+        with:
+          version: 10.26.1
+          run_install: false
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+      - name: 📦 Bundle
+        run: pnpm -r --workspace-concurrency=1 build
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: '3.13'
+      - name: Install the Python adapter
+        run: pip install -e 'packages/selenium-devtools-py[test]'
+      - name: Chrome version
+        run: google-chrome --version
+      - name: Start the fixture site
+        id: fixture
+        run: |
+          nohup pnpm -s smoke:fixtures > "$RUNNER_TEMP/fixture-site.log" 2>&1 &
+          for _ in $(seq 1 30); do
+            curl -sf "$DEVTOOLS_EXAMPLE_BASE_URL/login" > /dev/null && exit 0
+            sleep 1
+          done
+          cat "$RUNNER_TEMP/fixture-site.log"
+          exit 1
+      # One step per example so a red step names it; each runs even when an
+      # earlier one failed.
+      - name: 🧪 WebdriverIO + Mocha (examples/wdio/mocha)
+        if: ${{ !cancelled() && steps.fixture.outcome == 'success' }}
+        run: bash .github/scripts/examples-smoke/run-example.sh wdio-mocha pnpm demo:wdio:mocha
+      - name: 🧪 Selenium + Mocha (examples/selenium-js/mocha)
+        if: ${{ !cancelled() && steps.fixture.outcome == 'success' }}
+        run: bash .github/scripts/examples-smoke/run-example.sh selenium-mocha pnpm --filter @wdio/selenium-devtools example:mocha
+      - name: 🧪 Nightwatch BDD (examples/nightwatch/bdd)
+        if: ${{ !cancelled() && steps.fixture.outcome == 'success' }}
+        run: bash .github/scripts/examples-smoke/run-example.sh nightwatch-bdd pnpm demo:nightwatch
+      # The committed pytest.ini keeps only failing tests' archives; one archive
+      # for the whole run is what the expectations describe.
+      - name: 🧪 Python + pytest (examples/selenium-py/pytest)
+        if: ${{ !cancelled() && steps.fixture.outcome == 'success' }}
+        run: >-
+          bash .github/scripts/examples-smoke/run-example.sh python-pytest
+          python -m pytest examples/selenium-py/pytest
+          -o devtools_trace_policy=on -o devtools_trace_granularity=session
+      - name: Upload traces and run logs
+        if: failure()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: examples-smoke
+          path: |
+            ${{ runner.temp }}/examples-smoke/
+            ${{ runner.temp }}/fixture-site.log
+          if-no-files-found: ignore
+
   # The workspace develops against one WebdriverIO major, but the service and
   # @wdio/elements declare peers on two. Each published major is installed from
   # npm into a scratch project beside the packed tarballs, so a peer conflict, a

CLAUDE.md

@@ -29,6 +29,8 @@ Run from repo root unless noted.
 | `pnpm demo:wdio` / `pnpm demo:nightwatch` / `pnpm demo:selenium` | Run the per-framework example projects. Useful for manual verification of UI or runtime changes. **No package pins a browser driver**, deliberately: a pinned `chromedriver` rots against whatever Chrome the developer has, and because pnpm puts the package's `node_modules/.bin` on `PATH`, Selenium Manager finds it there, prefers it over resolving, and on a major mismatch only *warns* before returning it anyway. The session then fails to start. With nothing on `PATH` both adapters resolve a matching driver (Nightwatch's Chrome service builder sets `requiresDriverBinary: false` and passes an unset `server_path` through to the same resolver), and nightwatch declares `chromedriver` an optional peer. |
 | `pnpm demo:wdio:mobile` / `:selenium:mobile` / `:nightwatch:mobile` / `:python:mobile` | The same, against Appium. All four build the same capability bag and drive the Clock app that ships with every Android system image (keying a duration on the timer keypad, never starting a timer), so a native example needs no `.apk`. `DEVTOOLS_MOBILE_PLATFORM=ios` runs the iOS spec instead — all four adapters — which drives Settings because the simulator ships no Clock, from a separate spec per platform rather than a branch. The simulator is chosen by udid and defaults to whichever is already booted, and an unmatched `IOS_DEVICE_NAME` is refused: naming one that does not exist makes the XCUITest driver create and boot it, every run, rather than fail. That refusal and the booted-simulator preflight are **local** policy — `xcrun simctl` enumerates local simulators and nothing else — so `IOS_UDID` and a non-local `APPIUM_HOST` both bypass them, or a real device and a cloud grid would be refused a run they were correctly configured for. `DEVTOOLS_MOBILE=web` drives the device's own browser on both platforms — Chrome on Android, Safari on iOS, which the XCUITest driver serves without a chromedriver. `examples/MOBILE.md` holds the prerequisites and the `DEVTOOLS_MOBILE` / `APPIUM_APP` switches; `DEVTOOLS_MODE=trace` flips any demo to trace mode. |
 | `pnpm dev` | Run all packages in parallel dev mode. |
+| `pnpm smoke:fixtures [--port N]` | Serve the local fixture site the desktop examples drive (a copy of the-internet's login flow: `/`, `/login`, `/secure`, `/logout`; default `http://127.0.0.1:4567`). Every desktop login example reads `DEVTOOLS_EXAMPLE_BASE_URL` (trailing slash ignored); unset, they drive the public site. |
+| `pnpm smoke:assert --expect <name> [--exit-code N] [--log file] <zip-or-dir>...` | Check trace zips against `packages/conformance/smoke/expectations/<name>.json`: the listed rows under each test's own tracing group (or the whole run for an entry with `scope: "run"` and a `reason`), an error row on each deliberately failing test, a retried test's attempt count with its other checks read from its final attempt alone (`attempts`), title fragments such as a failed wait's condition, a call site on every row of a per-test entry (at least one for a run-scoped entry) and every call site inside the example's own `sourceFiles`, the run's exit code, and no wait timeout, in any adapter's wording (`TIMEOUT_PATTERNS` in `smoke/constants.ts`), in the trace's stdout or the run log other than the lines of a test marked `allowTimeoutError`. |
 | `pnpm docs:capabilities` | Render the per-adapter capability tables from `packages/conformance/src/features.ts` into `README.md`; `--website <webdriverio checkout>` also renders the docs-site tables, `--check` fails instead of writing, and `--parity` lists every feature some adapter lacks plus the evidence counts. |
 | `python3 packages/selenium-devtools-py/scripts/changes.py next-version` | The version a Python-adapter release would publish, from the fragments pending in `changes/`. `check --base <ref>` is the CI gate; `apply` is what the release runs. |
 
@@ -199,6 +201,10 @@ For UI or runtime changes, `examples/<framework>/` is the verification harness.
 
 When CI can't run an example (no real browser), the PR description says so explicitly.
 
+### Examples smoke job
+
+`ci.yml`'s `examples-smoke` job runs one desktop example per adapter (WDIO mocha, Selenium mocha, Nightwatch BDD, Python pytest) in trace mode in the runner's headless Chrome, against the fixture site (`pnpm smoke:fixtures`) via `DEVTOOLS_EXAMPLE_BASE_URL`, so no third-party page change can break it. `.github/scripts/examples-smoke/run-example.sh` collects the zips each run wrote and hands them to `pnpm smoke:assert`; each red step names its example and each `FAIL` line names the expectation, and the zips and run logs are uploaded as the `examples-smoke` artifact. It checks that capture still produces the rows the examples imply, not how they render, so dashboard visuals, mobile and Safari still need a person. A change that alters an example's rows updates its expectation file in the same PR.
+
 The venv's editable install of `selenium_devtools` points at the primary checkout, so in a worktree every Python example, and the backend and page collector it resolves through `_find_monorepo_backend`, come from `main` unless `PYTHONPATH=packages/selenium-devtools-py/src` is exported first. A Python run that reports the old behaviour from a worktree is usually this, not the change.
 
 ### Skipping tests that depend on workspace-internal build artifacts

examples/README.md

@@ -46,6 +46,27 @@ runner directly — `example:mocha`, `example:jest`, `example:cucumber`, or
 `DEVTOOLS_TRACE_GRANULARITY` and `DEVTOOLS_TRACE_POLICY` walk the rest of the
 ladder, so each runner needs only one config.
 
+## Local fixture site
+
+`pnpm smoke:fixtures` serves a copy of the-internet's login flow (`/`, `/login`,
+`/secure`, `/logout`, same selectors, titles and flash texts) on
+`http://127.0.0.1:4567`. `DEVTOOLS_EXAMPLE_BASE_URL` points an example at it
+instead of the public site:
+
+```
+pnpm smoke:fixtures &
+DEVTOOLS_EXAMPLE_BASE_URL=http://127.0.0.1:4567 DEVTOOLS_MODE=trace pnpm demo:wdio:mocha
+```
+
+Every desktop login example reads it (WebdriverIO, Nightwatch and Selenium in
+each runner, and both Python login cases; a trailing slash is ignored). Unset,
+they drive the public site as before. `web_form.py` drives selenium.dev's form
+page instead and the mobile examples read `DEVTOOLS_MOBILE_URL`. CI's `examples-smoke` job runs one
+of each in trace mode against the fixture and checks the zips with
+`pnpm smoke:assert`; the expected rows per test live in
+`packages/conformance/smoke/expectations/`, so a change to one of those
+examples' steps updates its expectation file too.
+
 ## Mobile
 
 Every adapter has a `mobile/` example, and all four build the same capability

examples/nightwatch/bdd/smoke-test.js

@@ -17,7 +17,9 @@
  *   pnpm demo:nightwatch:retry    (rung 5 — adds --retries 1)
  */
 
-const BASE_URL = 'https://the-internet.herokuapp.com'
+const BASE_URL = (
+  process.env.DEVTOOLS_EXAMPLE_BASE_URL || 'https://the-internet.herokuapp.com'
+).replace(/\/+$/, '')
 
 // Survives Nightwatch's testcase retry so the flaky test fails once, then passes.
 let flakyAttempts = 0

examples/nightwatch/cucumber/features/step_definitions/login.steps.js

@@ -5,7 +5,10 @@
 // functions so `this` binds to the World.
 const { Given, When, Then } = require('@cucumber/cucumber')
 
-const LOGIN_URL = 'https://the-internet.herokuapp.com/login'
+const BASE_URL = (
+  process.env.DEVTOOLS_EXAMPLE_BASE_URL || 'https://the-internet.herokuapp.com'
+).replace(/\/+$/, '')
+const LOGIN_URL = `${BASE_URL}/login`
 
 Given(/^I am on the login page$/, async function () {
   await this.browser.url(LOGIN_URL)

examples/selenium-js/cucumber/features/support/steps.js

@@ -2,8 +2,12 @@ import { strict as assert } from 'node:assert'
 import { Given, When, Then } from '@cucumber/cucumber'
 import { By, until } from 'selenium-webdriver'
 
+const BASE_URL = (
+  process.env.DEVTOOLS_EXAMPLE_BASE_URL || 'https://the-internet.herokuapp.com'
+).replace(/\/+$/, '')
+
 Given('I am on the login page', async function () {
-  await this.driver.get('https://the-internet.herokuapp.com/login')
+  await this.driver.get(`${BASE_URL}/login`)
 })
 
 When(

examples/selenium-js/jest/example.js

@@ -7,7 +7,10 @@
 import { DevTools } from '@wdio/selenium-devtools'
 import { Builder, By, until } from 'selenium-webdriver'
 
-const LOGIN_URL = 'https://the-internet.herokuapp.com/login'
+const BASE_URL = (
+  process.env.DEVTOOLS_EXAMPLE_BASE_URL || 'https://the-internet.herokuapp.com'
+).replace(/\/+$/, '')
+const LOGIN_URL = `${BASE_URL}/login`
 const VALID_USERNAME = 'tomsmith'
 const VALID_PASSWORD = 'SuperSecretPassword!'
 

examples/selenium-js/mocha/example.js

@@ -3,8 +3,9 @@
  *
  * Walk the live/trace ladder by editing ONLY the DevTools.configure({...})
  * block below. The suite carries a passing pair, an always-failing test (for
- * retain-on-failure), and a flaky fail-then-pass test with a per-test retry
- * (for on-first-retry / attempt capture).
+ * retain-on-failure), a wait that times out (a failed wait row), and a flaky
+ * fail-then-pass test with a per-test retry (for on-first-retry / attempt
+ * capture).
  *
  * Run from the package root:  pnpm example:mocha
  */
@@ -27,7 +28,10 @@ DevTools.configure({
   headless: true
 })
 
-const LOGIN_URL = 'https://the-internet.herokuapp.com/login'
+const BASE_URL = (
+  process.env.DEVTOOLS_EXAMPLE_BASE_URL || 'https://the-internet.herokuapp.com'
+).replace(/\/+$/, '')
+const LOGIN_URL = `${BASE_URL}/login`
 
 // Survives Mocha's in-process retry so the flaky test fails once, then passes.
 let flakyAttempts = 0
@@ -54,10 +58,10 @@ describe('selenium-devtools smoke test', function () {
   })
 
   it('navigates and reads the page title', async function () {
-    await driver.get('https://example.org')
+    await driver.get(BASE_URL)
     await driver.sleep(1500)
     const title = await driver.getTitle()
-    assert.match(title, /Example/i)
+    assert.match(title, /The Internet/)
   })
 
   it('fails on a wrong heading (retain-on-failure target)', async function () {
@@ -68,15 +72,23 @@ describe('selenium-devtools smoke test', function () {
     assert.equal(text, 'This Is Not The Heading')
   })
 
+  it('times out waiting for an element the page never renders (failed wait target)', async function () {
+    await driver.get(LOGIN_URL)
+    await driver.wait(
+      until.elementLocated(By.css('#devtools-never-rendered')),
+      1000
+    )
+  })
+
   it('flaky: fails the first attempt, then passes (retry target)', async function () {
     this.retries(1)
-    await driver.get('https://example.com')
+    await driver.get(LOGIN_URL)
     await driver.sleep(1000)
     flakyAttempts += 1
     if (flakyAttempts === 1) {
       throw new Error('intentional first-attempt failure — should retry')
     }
     const title = await driver.getTitle()
-    assert.match(title, /Example/i)
+    assert.match(title, /The Internet/)
   })
 })

examples/selenium-py/pytest/test_login_pytest.py

@@ -43,14 +43,18 @@
 — which also exercises the adapter's per-driver capture state.
 """
 
+import os
+
 import pytest
 from selenium import webdriver
 from selenium.webdriver.chrome.options import Options
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 
-BASE_URL = "https://the-internet.herokuapp.com"
+BASE_URL = (
+    os.environ.get("DEVTOOLS_EXAMPLE_BASE_URL") or "https://the-internet.herokuapp.com"
+).rstrip("/")
 USERNAME = "tomsmith"
 PASSWORD = "SuperSecretPassword!"
 TIMEOUT = 5

examples/selenium-py/scripts/login.py

@@ -16,14 +16,18 @@
 screencast .webm) lands in ``test-results/`` beside this file.
 """
 
+import os
+
 import selenium_devtools as devtools
 from selenium import webdriver
 from selenium.webdriver.chrome.options import Options
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 
-BASE_URL = "https://the-internet.herokuapp.com"
+BASE_URL = (
+    os.environ.get("DEVTOOLS_EXAMPLE_BASE_URL") or "https://the-internet.herokuapp.com"
+).rstrip("/")
 USERNAME = "tomsmith"
 PASSWORD = "SuperSecretPassword!"
 TIMEOUT = 5

examples/wdio/mocha/retry/flaky.e2e.ts

@@ -7,9 +7,13 @@ import { browser, expect } from '@wdio/globals'
 // the retry attempt is captured and is distinct from a failure.
 let attempts = 0
 
+const BASE_URL = (
+  process.env.DEVTOOLS_EXAMPLE_BASE_URL || 'https://the-internet.herokuapp.com'
+).replace(/\/+$/, '')
+
 describe('Flaky (passes on retry)', () => {
   it('fails the first attempt, then passes', async () => {
-    await browser.url('https://the-internet.herokuapp.com/login')
+    await browser.url(`${BASE_URL}/login`)
     attempts += 1
     if (attempts === 1) {
       throw new Error('intentional first-attempt failure — should retry')

examples/wdio/pageobjects/page.ts

@@ -1,5 +1,9 @@
 import { browser } from '@wdio/globals'
 
+const BASE_URL = (
+  process.env.DEVTOOLS_EXAMPLE_BASE_URL || 'https://the-internet.herokuapp.com'
+).replace(/\/+$/, '')
+
 /**
  * main page object containing all methods, selectors and functionality
  * that is shared across all page objects
@@ -10,6 +14,6 @@ export default class Page {
    * @param path path of the sub page (e.g. /path/to/page.html)
    */
   public open(path: string) {
-    return browser.url(`https://the-internet.herokuapp.com/${path}`)
+    return browser.url(`${BASE_URL}/${path}`)
   }
 }

package.json

@@ -22,6 +22,8 @@
     "demo:python:login": "python3 examples/selenium-py/scripts/login.py",
     "demo:python:pytest": "python3 -m pytest examples/selenium-py/pytest/",
     "demo:python:mobile": "python3 examples/selenium-py/mobile/${DEVTOOLS_MOBILE_PLATFORM:-android}/*.py",
+    "smoke:fixtures": "tsx packages/conformance/smoke/serve-fixtures.ts",
+    "smoke:assert": "tsx packages/conformance/smoke/assert-trace.ts",
     "dev": "pnpm --parallel dev",
     "preview": "pnpm --parallel preview",
     "docs:capabilities": "tsx packages/conformance/src/capability-matrix.ts",