Name a node:assert target from value provenance in the WDIO service

.changeset/assert-target-per-test.md

@@ -0,0 +1,7 @@
+---
+'@wdio/devtools-service': patch
+'@wdio/selenium-devtools': patch
+'@wdio/nightwatch-devtools': patch
+---
+
+A `node:assert` row only names an element read in the same test (or Cucumber scenario). An assertion in a later test, such as `assert.ok(true)`, no longer boxes an element an earlier test read the same value from. With the WebdriverIO service, a `browser.getTitle()` issued while one of the service's own title reads is still pending keeps its row in the Actions list.

.changeset/service-assert-target.md

@@ -0,0 +1,5 @@
+---
+'@wdio/devtools-service': patch
+---
+
+A `node:assert` row names the element whose read produced the asserted value, and none for a page-level value: `assert.notEqual(await browser.getTitle(), 'Secure Area')` no longer boxes the last element the test resolved. The test's own `browser.getTitle()` is now a `Page.getTitle` row in the Actions list, as in the other adapters.

packages/conformance/src/allowlist.ts

@@ -3,7 +3,6 @@ import {
   CONDITION_WAIT_READS,
   HELPER_WAIT_READ,
   KNOWN_DEBT,
-  PAGE_TITLE_TEST,
   RULES
 } from './constants.js'
 import { anyOf, hunk } from './gap-matchers.js'
@@ -17,9 +16,8 @@ import type {
   Violation
 } from './types.js'
 
-const [WAITS, PAGE_TITLE, MISSING] = [
+const [WAITS, MISSING] = [
   'waits on what the page shows',
-  PAGE_TITLE_TEST,
   'times out waiting for a missing element'
 ].map(scenarioIndex)
 
@@ -63,41 +61,12 @@ export const KNOWN_GAPS: KnownGap[] = [
     timingDependent: true
   },
   ...BIDI_MODES.flatMap((bidi) => [
-    ...serviceGaps(bidi),
     ...nightwatchGaps(bidi),
     ...seleniumGaps(bidi),
     ...pythonGaps(bidi)
   ])
 ]
 
-function serviceGaps(bidi: BidiMode): KnownGap[] {
-  const adapter = 'devtools-service'
-  const reason = KNOWN_DEBT.serviceAssertTarget
-  return [
-    {
-      adapter,
-      bidi,
-      rules: [RULES.pageAssertNamesNoElement],
-      only: (message) =>
-        message === 'assert.notEqual on the page title names h2',
-      reason
-    },
-    {
-      adapter,
-      bidi,
-      rules: [RULES.agreeOnActions],
-      only: hunk(
-        PAGE_TITLE,
-        'Element.getText h2',
-        adapter,
-        'Assert.notEqual h2',
-        'Page.getTitle, Assert.notEqual'
-      ),
-      reason
-    }
-  ]
-}
-
 /** Nightwatch's scenario gaps in one mode, each an adapter bug with its Known debt entry. */
 function nightwatchGaps(bidi: BidiMode): KnownGap[] {
   const adapter = 'nightwatch-devtools'

packages/conformance/src/constants.ts

@@ -203,7 +203,6 @@ const SELENIUM_WAIT_DEBT =
 
 /** Why each scenario gap exists: the limitation, and the Known debt entry that describes it. */
 export const KNOWN_DEBT = {
-  serviceAssertTarget: `The WebdriverIO service stamps a node:assert row with the last element it resolved and hides the user's own getTitle, so an assertion on the page title boxes the heading read before it. ${debt('The WDIO service boxes the last element it resolved for an assertion on the page title', '#481')}`,
   conditionWait: `A passing condition wait shows as its reads in Selenium and Python, which have no row for the wait itself, as one Browser.waitForFunction row in the WebdriverIO service, which hides the reads inside it, and as both in Nightwatch. ${debt('A condition wait shows its reads in some adapters and one wait row in others', '#464')}`,
   seleniumWait: `A successful driver.wait() has no row of its own: its polls are findElements rows, which the trace vocabulary leaves unmapped. ${debt(SELENIUM_WAIT_DEBT, '#464')}`,
   seleniumFailedWait: `A timed-out driver.wait() is a wait row named after its condition, mapped to Browser.waitForFunction with no target, where the other adapters show Element.waitForExist on the selector. ${debt(SELENIUM_WAIT_DEBT, '#464')}`,

packages/core/src/read-value-locators.ts

@@ -59,6 +59,11 @@ export function rememberReadValue(
   }
 }
 
+/** Called at each test start: an earlier test's read would otherwise box its element for a later `assert.ok(true)`. */
+export function clearReadValues(): void {
+  selectorByReadValue.clear()
+}
+
 /** The element locator this value was read from — undefined when an element read
  *  wasn't its most recent producer. */
 export function selectorForReadValue(value: unknown): string | undefined {

packages/core/tests/read-value-locators.test.ts

@@ -1,5 +1,6 @@
 import { describe, it, expect } from 'vitest'
 import {
+  clearReadValues,
   rememberReadValue,
   resolveAssertTargetFromArgs,
   selectorForReadValue
@@ -144,3 +145,18 @@ describe('resolveAssertTargetFromArgs', () => {
     ).toBe('#handle')
   })
 })
+
+describe('clearReadValues', () => {
+  it('forgets every producer, so a later assertion on the same value names nothing', () => {
+    rememberReadValue(true, '#remember-me')
+    rememberReadValue('Example Domain', undefined)
+    clearReadValues()
+    expect(selectorForReadValue(true)).toBeUndefined()
+    expect(resolveAssertTargetFromArgs([true])).toBeUndefined()
+    // The no-element sentinel goes too: a cleared value no longer ends the lookup.
+    rememberReadValue('expected-after-clear', '#fresh')
+    expect(
+      resolveAssertTargetFromArgs(['Example Domain', 'expected-after-clear'])
+    ).toBe('#fresh')
+  })
+})

packages/nightwatch-devtools/src/helpers/assertCapture.ts

@@ -16,18 +16,24 @@ const log = logger('@wdio/nightwatch-devtools:assertCapture')
 /**
  * Patch node:assert once per process. Getters are read at capture time — the
  * capturer is created lazily in session init and the test UID changes per test.
+ * `scopeReadValues` runs before each lookup so an assert that precedes its
+ * test's first command still cannot resolve against an earlier test's reads.
  */
 export function wireAssertCapture(
   getCapturer: () => SessionCapturer | undefined,
-  getTestUid: () => string | undefined
+  getTestUid: () => string | undefined,
+  scopeReadValues: () => void = () => {}
 ): void {
   patchNodeAssert(
     (cmd) => captureAssert(getCapturer(), getTestUid(), cmd),
     (level, message) => log[level](message),
     // No handle resolver: Nightwatch commands take selector strings, so an
     // element object never reaches an assert — only the value's provenance,
     // recorded by the command path (BrowserProxy), can name the target.
-    (args) => resolveAssertTargetFromArgs(args)
+    (args) => {
+      scopeReadValues()
+      return resolveAssertTargetFromArgs(args)
+    }
   )
 }
 

packages/nightwatch-devtools/src/helpers/browserProxy.ts

@@ -16,6 +16,7 @@ import { NativeAssertRecorder } from './nativeAssertRecorder.js'
 import {
   RetryTracker,
   beginInputDispatch,
+  clearReadValues,
   rememberReadValue,
   toError
 } from '@wdio/devtools-core'
@@ -67,6 +68,9 @@ export class BrowserProxy {
    */
   private browser: NightwatchBrowser | null = null
 
+  /** The test core's value registry currently holds reads for; see {@link scopeReadValuesToRowTest}. */
+  private readValueTestUid: string | undefined
+
   constructor(
     private readonly sessionCapturer: SessionCapturer,
     private testManager: TestManager,
@@ -104,6 +108,20 @@ export class BrowserProxy {
     return this.rowTest()?.uid
   }
 
+  /**
+   * Forget the values an earlier test read once the running test changes. The
+   * lifecycle reset below fires once per MODULE on the BDD interface, so this
+   * per-`it` live read is what keeps one `it`'s reads from naming another's
+   * assertion target. Called before every record and every lookup.
+   */
+  scopeReadValuesToRowTest(): void {
+    const uid = this.rowTestUid()
+    if (uid !== this.readValueTestUid) {
+      clearReadValues()
+      this.readValueTestUid = uid
+    }
+  }
+
   /** Per-command-scope dedup state only. Cucumber calls this per STEP, so it
    *  must not touch the native-assert buffer — that buffer spans the whole
    *  scenario and is drained once, at the scenario's pre-quit hook. */
@@ -118,6 +136,8 @@ export class BrowserProxy {
   resetTestTracking(): void {
     this.resetCommandTracking()
     this.assertRecorder.clear()
+    clearReadValues()
+    this.readValueTestUid = undefined
   }
 
   /** Hand off this test's recorded native-assertion calls and clear the
@@ -437,6 +457,7 @@ export class BrowserProxy {
     callbackResult: unknown
   ): unknown {
     const result = serializeCommandResult(callbackResult, methodName)
+    this.scopeReadValuesToRowTest()
     rememberReadValue(result, commandTargetSelector(methodName, logArgs))
     return result
   }

packages/nightwatch-devtools/src/index.ts

@@ -400,7 +400,8 @@ class NightwatchDevToolsPlugin {
         // Rows are tagged per-`it` (BrowserProxy.rowTestUid); `#currentTestUid`
         // — the lifecycle test, which the BDD interface pins to the module's
         // first `it` — is only the fallback, and stays the artifact/slice key.
-        () => this.browserProxy?.rowTestUid() ?? this.#currentTestUid()
+        () => this.browserProxy?.rowTestUid() ?? this.#currentTestUid(),
+        () => this.browserProxy?.scopeReadValuesToRowTest()
       )
     }
   }

packages/nightwatch-devtools/tests/assertCapture.test.ts

@@ -3,6 +3,7 @@ import assert from 'node:assert'
 import {
   ASSERT_PATCHED_SYMBOL,
   TRACKED_ASSERT_METHODS,
+  clearReadValues,
   rememberReadValue
 } from '@wdio/devtools-core'
 import { wireAssertCapture } from '../src/helpers/assertCapture.js'
@@ -58,11 +59,13 @@ function makeFakeCapturer() {
 const live: {
   fake?: ReturnType<typeof makeFakeCapturer>
   uid?: string
+  scope?: () => void
 } = {}
 beforeAll(() => {
   wireAssertCapture(
     () => live.fake?.capturer,
-    () => live.uid
+    () => live.uid,
+    () => live.scope?.()
   )
 })
 
@@ -155,4 +158,19 @@ describe('node:assert target selector', () => {
       expect.objectContaining({ selector: '#subject' })
     )
   })
+
+  it('scopes the registry to the running test before looking a value up', () => {
+    // An assert that precedes its test's first command must not resolve against
+    // the reads of the test before it.
+    rememberReadValue(true, '#remember-me')
+    live.scope = clearReadValues
+    try {
+      const row = capture(() => {
+        assert.ok(true)
+      })
+      expect(row.selector).toBeUndefined()
+    } finally {
+      live.scope = undefined
+    }
+  })
 })

packages/nightwatch-devtools/tests/browserProxy.test.ts

@@ -318,6 +318,53 @@ describe('BrowserProxy read-value provenance', () => {
     proxied(failing).getText('#gone')
     expect(selectorForReadValue('Timed out on <#gone>')).toBeUndefined()
   })
+
+  // Regression: the registry outlived the test that read a value, so a later
+  // test's `assert.ok(true)` boxed an element an earlier test had read `true` from.
+  describe('scoped to one test', () => {
+    function proxyForRunningTest(running: { uid: string }) {
+      const { capturer } = makeCapturer()
+      const proxy = new BrowserProxy(
+        capturer,
+        makeTestManager(() => running as TestStats),
+        () => ({ uid: 'module-first-it' })
+      )
+      const browser = makeReadBrowser(true)
+      proxy.wrapBrowserCommands(browser)
+      return {
+        proxy,
+        browser: browser as unknown as Record<
+          string,
+          (...a: unknown[]) => unknown
+        >
+      }
+    }
+
+    it('forgets an earlier `it`s reads once the running `it` changes (BDD, one hook per module)', () => {
+      const running = { uid: 'it-1' }
+      const { proxy, browser } = proxyForRunningTest(running)
+      browser.getText('#remember-me')
+      expect(selectorForReadValue(true)).toBe('#remember-me')
+      running.uid = 'it-2'
+      // What the node:assert resolver calls before its lookup.
+      proxy.scopeReadValuesToRowTest()
+      expect(selectorForReadValue(true)).toBeUndefined()
+    })
+
+    it('keeps the reads of the `it` still running', () => {
+      const { proxy, browser } = proxyForRunningTest({ uid: 'it-1' })
+      browser.getText('#remember-me')
+      proxy.scopeReadValuesToRowTest()
+      expect(selectorForReadValue(true)).toBe('#remember-me')
+    })
+
+    it('forgets every read at a test unit reset (a cucumber scenario, a module)', () => {
+      const { proxy, browser } = proxyForRunningTest({ uid: 'it-1' })
+      browser.getText('#remember-me')
+      proxy.resetTestTracking()
+      expect(selectorForReadValue(true)).toBeUndefined()
+    })
+  })
 })
 
 describe('BrowserProxy row metadata', () => {

packages/selenium-devtools-py/changes/assertion-target-per-test.md

@@ -0,0 +1,5 @@
+---
+patch
+---
+
+An assertion row only names an element read in the same pytest test. An assertion in a later test no longer boxes an element an earlier test read the same value from.

packages/selenium-devtools-py/src/selenium_devtools/pytest_plugin.py

@@ -15,7 +15,7 @@
 from typing import Dict, Optional
 
 import selenium_devtools as devtools
-from . import assertions, frames, rerun
+from . import assertions, frames, read_value_locators, rerun
 from .constants import ENV_OPT_IN, ENV_PORT, LOGGER_NAME
 from .utils import now_ms
 
@@ -531,6 +531,7 @@ def pytest_runtest_logstart(nodeid, location) -> None:  # noqa: ANN001
     if not _opted_in():
         return
     _registry.mark_start(nodeid)
+    read_value_locators.reset()
     capturer = devtools.get_capturer()
     if capturer is None:
         return

packages/selenium-devtools-py/src/selenium_devtools/read_value_locators.py

@@ -26,7 +26,8 @@
 
 
 def reset() -> None:
-    """Forget every value, so a re-enable cannot attribute a previous run's read."""
+    """Forget every value: at a re-enable, and at each test start so a later
+    test's ``assert True`` cannot box an element an earlier test read."""
     _selectors.clear()
 
 

packages/selenium-devtools-py/tests/test_read_value_locators.py

@@ -177,6 +177,21 @@ def test_a_containment_looks_up_the_observed_side(self):
         )
         self.assertEqual(self._assert_row()["selector"], "h1")
 
+    def test_a_test_start_forgets_the_previous_test_s_reads(self):
+        # Regression: a later test's assertion on a value an earlier test read
+        # boxed the earlier test's element.
+        plugin.pytest_runtest_logstart("test_x.py::test_one", ("test_x.py", 1, "test_one"))
+        self.driver.text(self.driver.find("#flash"))
+        plugin.pytest_runtest_logstart("test_x.py::test_two", ("test_x.py", 5, "test_two"))
+        self._passing_assert("You logged in", "You logged in", 'expected == "You logged in"')
+        self.assertNotIn("selector", self._assert_row())
+
+    def test_a_read_within_the_test_still_names_its_element(self):
+        plugin.pytest_runtest_logstart("test_x.py::test_one", ("test_x.py", 1, "test_one"))
+        greeting = self.driver.text(self.driver.find("#flash"))
+        self._passing_assert(greeting, "You logged in", 'greeting == "You logged in"')
+        self.assertEqual(self._assert_row()["selector"], "#flash")
+
     def test_a_re_enable_forgets_the_previous_run_s_reads(self):
         self.driver.text(self.driver.find("#flash"))
         instrumentation.uninstall()

packages/selenium-devtools/src/test-management.ts

@@ -17,7 +17,7 @@ import type { SessionCapturer } from './session.js'
 import type { TestReporter } from './reporter.js'
 import type { SuiteManager } from './helpers/suiteManager.js'
 import type { DetectedRunner, TestStats } from './types.js'
-import type { RetryTracker } from '@wdio/devtools-core'
+import { clearReadValues, type RetryTracker } from '@wdio/devtools-core'
 
 const log = logger('@wdio/selenium-devtools:test-management')
 
@@ -118,6 +118,7 @@ export function startTest(
   name: string,
   meta: StartTestMeta = {}
 ): void {
+  clearReadValues()
   if (!ctx.testFilePath && meta.file) {
     ctx.testFilePath = meta.file
   }
@@ -171,6 +172,7 @@ export function startScenario(
   name: string,
   meta: StartScenarioMeta = {}
 ): void {
+  clearReadValues()
   if (!ctx.suiteManager || !ctx.testReporter) {
     ctx.pendingScenario = { name, ...meta }
     return