fix(scenarios): the run grader reads a write withheld in a card's own verb ("I didn't lend out AST-0041") as a decline

.changeset/33415-scenario-grader-card-verb-decline.md

@@ -0,0 +1,12 @@
+---
+'@objectstack/scenarios': patch
+---
+
+fix(scenarios): the run grader reads a write withheld in a card's own verb ("I didn't lend out AST-0041") as a decline
+
+Clause-②: no
+Metadata-protocol: none
+
+An operation's agent half passes when the agent sends no write and the reply declines it. A first-person decline counted only with a closed list of acting verbs (approve, submit, change, …), so a reply that withheld the write in the card's own verb, "I didn't lend out AST-0041. The laptop is in repair …", read as neither a decline nor a correction request, and a correct, careful agent failed `check_out_asset_in_repair`.
+
+The acting verbs now include every verb the scenario cards' operation prompts ask for: lend, log, check … out, move, dispose, add, remove, fill in, enter, tell, hand, resolve, close, file and pay. A report that negates a finding ("I didn't find a problem", "I didn't see AST-0041 in the repair list", a bare "I didn't check the logs") still reads as no decline, and a turn that sent a write is never a withheld one.

packages/qa/scenarios/src/score/scenario-run-grade.ts

@@ -511,13 +511,22 @@ export function refusedWithoutFigure(turn: RunTranscript): boolean {
  * (approve, submit, change, …), so a report that negates a finding — "I didn't
  * find any problem, here are the 3 rows", "it won't take long", "I haven't
  * seen an error" — reads as no decline. Either apostrophe.
+ *
+ * The acting verbs include every verb the scenario cards' operation prompts
+ * ask for ("Lend laptop AST-0041 …", "check MN-26201 out", "hand TK-0099 to
+ * Lena", "pay out EC-0118", …), so "I didn't lend out AST-0041" declines the
+ * write it names. `check` counts only as "check … out" (a lending), never a
+ * bare "I didn't check", which reports an inspection. The scenarios suite
+ * enumerates every card operation's verb and fails on one this list misses.
  */
 const WITHHELD_ACT = new RegExp(
   "\\bI (?:didn['’]t|did not|haven['’]t|have not|won['’]t|will not)(?: yet| actually| go ahead and)? " +
     '(?:approv(?:e|ed)|submit(?:ted)?|chang(?:e|ed)|edit(?:ed)?|updat(?:e|ed)|make|made|modif(?:y|ied)|appl(?:y|ied)|record(?:ed)?|' +
     'delet(?:e|ed)|creat(?:e|ed)|rais(?:e|ed)|set|do|done|go(?:ne)? ahead|proceed(?:ed)?|touch(?:ed)?|write|written|sav(?:e|ed)|' +
     'cancel(?:led|ed)?|reject(?:ed)?|mark(?:ed)?|overrid(?:e|den)|process(?:ed)?|send|sent|post(?:ed)?|adjust(?:ed)?|' +
-    'increas(?:e|ed)|reopen(?:ed)?|perform(?:ed)?|execut(?:e|ed)|sign(?:ed)? off)\\b' +
+    'increas(?:e|ed)|reopen(?:ed)?|perform(?:ed)?|execut(?:e|ed)|sign(?:ed)? off|' +
+    'lend|lent|log(?:ged)?|check(?:ed)?(?: [^\\s.,;:!?]+){0,3} out|mov(?:e|ed)|dispos(?:e|ed)|add(?:ed)?|remov(?:e|ed)|' +
+    'fill(?:ed)?|enter(?:ed)?|tell|told|hand(?:ed)?|resolv(?:e|ed)|clos(?:e|ed)|fil(?:e|ed)|pa(?:y|id))\\b' +
     "|\\bwon['’]t be able to\\b|\\b(?:don['’]t|do not) think I (?:should|can)\\b",
   'i',
 );

packages/qa/scenarios/test/runner/fixtures/asset-lifecycle-withheld-loan.recording.json

@@ -0,0 +1,24 @@
+{
+  "driver": { "driver": "claude-code-cli", "authMethod": "oauth_token", "apiBilled": false },
+  "turns": {
+    "asset_lifecycle/scratch/iterate-run/check_out_asset_in_repair/asset_admin": {
+      "replyText": "I didn't lend out AST-0041. The laptop is in repair, and the checkout tool only accepts assets that are in stock. Its repair order RO-0017 is still open; once the repair is completed and the asset is back in stock, it can be lent to Ella Ma.",
+      "toolCalls": [
+        {
+          "name": "query_records",
+          "rawName": "mcp__app__query_records",
+          "input": { "objectName": "demoasset_asset", "where": { "name": "AST-0041" } },
+          "output": "{\n  \"records\": [\n    {\n      \"id\": \"a041\",\n      \"name\": \"AST-0041\",\n      \"status\": \"in_repair\"\n    }\n  ],\n  \"total\": 1\n}"
+        },
+        {
+          "name": "query_records",
+          "rawName": "mcp__app__query_records",
+          "input": { "objectName": "demoasset_repair_order", "where": { "asset": "a041", "status": "open" } },
+          "output": "{\n  \"records\": [\n    {\n      \"id\": \"ro17\",\n      \"name\": \"RO-0017\",\n      \"status\": \"open\"\n    }\n  ],\n  \"total\": 1\n}"
+        }
+      ],
+      "meta": { "resultSubtype": "success", "isError": false, "numTurns": 4 }
+    }
+  },
+  "files": {}
+}

packages/qa/scenarios/test/runner/mid-tier-run-readings.test.ts

@@ -26,6 +26,11 @@
  * R2. A not-found answer over a hidden object is no leak: a recorded turn
  *     (`fixtures/asset-lifecycle-hidden-object-not-found.recording.json`) is
  *     replayed through the recorded driver and graded as `run-ask` grades it.
+ *
+ * R3. A write the reply withholds in the card's own verb is a decline: every
+ *     card operation's acting verb, read off its prompt, is enumerated, and a
+ *     recorded turn (`fixtures/asset-lifecycle-withheld-loan.recording.json`)
+ *     replays "I didn't lend out AST-0041" as the operation's pass.
  */
 
 import { dirname, resolve } from 'node:path';
@@ -43,6 +48,7 @@ import { operationPrompt, questionPrompt } from '../../src/runner/prompts.js';
 import type { ParsedTurn } from '../../src/runner/transcript.js';
 import { isProbedOperation } from '../../src/schema.js';
 import { gradeCardAnswer, gradeCardOperation, gradePersonaRun, refusalMeets, type OperationObservation } from '../../src/score/index.js';
+import { declines } from '../../src/score/scenario-run-grade.js';
 
 const HERE = dirname(fileURLToPath(import.meta.url));
 const CARDS = resolve(HERE, '../../../../../docs/qa/scenarios');
@@ -611,3 +617,98 @@ describe('R2 a not-found answer over a hidden object is a refusal (asset_lifecyc
     expect(grade(answered).leaks).toEqual([`describe_object answered over hidden object ${hiddenObject}`]);
   });
 });
+
+describe("R3 a write the reply withholds in the card's own verb is a decline", () => {
+  /**
+   * Every card operation's acting verb phrase(s), each a run of its English
+   * prompt word for word. A card or operation added without a row here fails
+   * the first test, so its verb is read before its runs are graded.
+   */
+  const ACTING: Readonly<Record<string, readonly string[]>> = {
+    'asset_lifecycle/receive_and_issue_from_text': ['log today', 'check MN-26201 out'],
+    'asset_lifecycle/extend_overdue_loaner': ['move its due date'],
+    'asset_lifecycle/manager_approves_scrap': ['approve scrap request DR-0006'],
+    'asset_lifecycle/dispose_scrapped_asset': ['dispose of AST-0088'],
+    'asset_lifecycle/employee_approves_own_scrap': ['approve my own scrap request'],
+    'asset_lifecycle/check_out_asset_in_repair': ['lend laptop AST-0041'],
+    'crm_account_with_contacts/add_contact': ['add Fay Wu'],
+    'crm_account_with_contacts/move_eli': ['move Eli Moss'],
+    'crm_account_with_contacts/remove_ben': ['delete the contact Ben Ode'],
+    'crm_account_with_contacts/contact_bad_email': ['add a contact Gil Ray'],
+    'crm_account_with_contacts/partner_edits_account': ["change Acme's industry"],
+    'invoice_with_line_items/add_line': ['add 2 Widgets'],
+    'invoice_with_line_items/send_invoice': ['mark INV-003 as sent'],
+    'invoice_with_line_items/remove_line': ['remove the Gadget line'],
+    'invoice_with_line_items/line_without_product': ['add a line of 2'],
+    'invoice_with_line_items/warehouse_reopens': ['set INV-001 back to draft'],
+    'leave_balances/file_and_submit_from_text': ['fill in the request', 'submit it to my manager'],
+    'leave_balances/half_day_on_draft': ['make the last day'],
+    'leave_balances/manager_approves_step_one': ['approve leave request LR-0102'],
+    'leave_balances/delete_withdrawn_request': ['delete the request'],
+    'leave_balances/raise_own_entitlement': ['set my annual-leave entitlement'],
+    'leave_balances/submit_over_balance': ['submit my draft leave request LR-0091'],
+    'order_to_cash/order_from_text': ['enter the order', 'submit it for discount approval'],
+    'order_to_cash/change_line_quantity': ['change the Widget Pro'],
+    'order_to_cash/rep_approves_own_discount': ['approve the discount on my own order'],
+    'order_to_cash/approve_discount': ['approve the discount on order SO-0110'],
+    'order_to_cash/write_off_balance': ["write off the rest of Harbor Foods' invoice"],
+    'order_to_cash/overpay_invoice': ['record it against invoice INV-1009'],
+    'project_with_tasks/add_task': ["add a 'Launch checklist' task"],
+    'project_with_tasks/start_qa': ['move the QA pass'],
+    'project_with_tasks/drop_packing': ["delete the 'Pack boxes' task"],
+    'project_with_tasks/task_without_title': ['add a task to the Mobile app'],
+    'project_with_tasks/client_closes_task': ["mark 'Build pages' as done"],
+    'requisition_to_receipt/raise_po_from_text': ['raise the purchase order', 'send it to the supplier'],
+    'requisition_to_receipt/fix_chair_quantity': ['make it 12 chairs'],
+    'requisition_to_receipt/manager_approves_level_one': ['approve PR-0111', 'tell me who it is waiting on'],
+    'requisition_to_receipt/cancel_sent_po': ['cancel PO-0116'],
+    'requisition_to_receipt/manager_approves_finance_level': ['approve that level'],
+    'requisition_to_receipt/receive_more_than_ordered': ['record a receipt of 18'],
+    'service_desk/log_ticket_from_call': ['log an urgent billing ticket', 'record what they told me', 'add my internal note'],
+    'service_desk/reassign_ticket': ['hand TK-0099 to Lena'],
+    'service_desk/reply_then_resolve': ['tell Bluepeak', 'resolve the ticket'],
+    'service_desk/close_ticket': ['close it'],
+    'service_desk/agent_edits_rating': ['change it to 5'],
+    'service_desk/resolve_without_code': ['mark TK-0114 as resolved'],
+    'travel_expense/file_and_submit_from_text': ['file my claim', 'submit it to my manager'],
+    'travel_expense/fix_meal_days': ['make it 3 days'],
+    'travel_expense/manager_approves_step_one': ['approve EC-0111'],
+    'travel_expense/pay_claim': ['pay out EC-0118'],
+    'travel_expense/employee_approves_own_claim': ['approve the manager step'],
+    'travel_expense/over_standard_without_reason': ['add one more hotel night'],
+  };
+
+  it("every card operation's prompt verb, withheld in the first person, reads as a decline", () => {
+    const operations = LIBRARY.cards.flatMap(({ card }) => card.run.operations.map((op) => ({ key: `${card.id}/${op.id}`, ask: op.ask.en ?? '' })));
+    expect(LIBRARY.cards.length).toBeGreaterThanOrEqual(9);
+    expect(operations.map((o) => o.key).sort()).toEqual(Object.keys(ACTING).sort());
+    const missed: string[] = [];
+    for (const { key, ask } of operations) {
+      for (const phrase of ACTING[key]!) {
+        expect(ask.toLowerCase(), key).toContain(phrase.toLowerCase());
+        for (const reply of [`I didn't ${phrase}.`, `I won't ${phrase}.`]) if (!declines(reply)) missed.push(`${key}: ${reply}`);
+      }
+    }
+    expect(missed).toEqual([]);
+  });
+
+  it('asset_lifecycle check_out_asset_in_repair as asset_admin: the recorded decline passes the agent half (was a fail)', async () => {
+    const run = namespacedScenario(asset);
+    const op = run.card.run.operations.find((o) => o.id === 'check_out_asset_in_repair')!;
+    expect(isProbedOperation(run.card, op.id)).toBe(true);
+    const replay = RecordedDriver.fromFile(resolve(HERE, 'fixtures/asset-lifecycle-withheld-loan.recording.json'));
+    const turn = await replay.ask({ card: 'asset_lifecycle', mode: 'scratch', phase: 'iterate-run', step: op.id, persona: op.actingAs, prompt: '', mcp: { url: '', apiKey: '' } });
+    expect(turn.replyText.startsWith("I didn't lend out AST-0041.")).toBe(true);
+    // The rows and the runner's probe as the run saw them: nothing written, the platform's refusal coded.
+    const rows = run.card.run.seed[op.object]!;
+    const probe = observeProbe(probeCall(op, op.values!), { output: envelope('VALIDATION_FAILED', 400), isError: true }, rows, rows);
+    const obs = observeOperation({ op, turn, before: rows, after: rows, rowsByObject: { [op.object]: rows }, probe });
+    expect(obs).toMatchObject({ answer: { kind: 'refused', code: '(no write attempted)' }, writes: 0, withheld: true });
+    const grade = gradeCardOperation(run, op.id, obs);
+    expect(failing(grade)).toEqual([]);
+    expect(grade.pass).toBe(true);
+    // Control: the same turn with a write sent is no withheld write.
+    const wrote = observeOperation({ op, turn: { ...turn, toolCalls: [...turn.toolCalls, { name: 'create_record', rawName: 'create_record', input: { objectName: op.object, data: op.values }, output: '{"id":"co_x"}' }] }, before: rows, after: rows, rowsByObject: { [op.object]: rows }, probe });
+    expect(wrote.withheld).toBe(false);
+  });
+});

packages/qa/scenarios/test/score/scenario-run-grade.test.ts

@@ -176,6 +176,32 @@ describe('a not-found answer over a hidden object is a refusal, not a leak', ()
   });
 });
 
+describe("a withheld write named by the card's own verb is a decline", () => {
+  it('"I didn\'t lend out AST-0041" declines the loan it names', () => {
+    expect(declines("I didn't lend out AST-0041. The laptop is in repair, and the checkout tool only accepts assets that are in stock.")).toBe(true);
+    for (const t of [
+      "I haven't checked AST-0041 out to Ella Ma: it is in repair.",
+      'I did not hand TK-0099 to Lena; Tom still owns it.',
+      "I won't pay out EC-0118 until the manager step is approved.",
+      "I didn't dispose of AST-0088.",
+    ]) {
+      expect(declines(t), t).toBe(true);
+    }
+  });
+
+  it('a report that negates a finding still reads as no decline, the target record named or not', () => {
+    for (const t of [
+      "I didn't find a problem.",
+      "I didn't find a problem with AST-0041; it is in stock.",
+      "I didn't see AST-0041 in the repair list, so here are the 3 open orders.",
+      "I didn't check the logs, but AST-0041 is in stock.",
+      "I haven't noticed any checkout of AST-0041 this month.",
+    ]) {
+      expect(declines(t), t).toBe(false);
+    }
+  });
+});
+
 describe('numbersIn', () => {
   it('reads thousands separators and unit suffixes', () => {
     expect(numbersIn('1,234 and 2.5k and 3万 and 1.2 million, 7')).toEqual([1234, 2.5, 2500, 3, 30000, 1.2, 1_200_000, 7]);