fix(scenarios): the run grader reads a hook's declared refusal off the door's declaredCode

.changeset/33497-scenario-grader-declared-code.md

@@ -0,0 +1,12 @@
+---
+'@objectstack/scenarios': patch
+---
+
+fix(scenarios): the run grader reads a hook's declared refusal off the door's `declaredCode`
+
+Clause-②: no
+Metadata-protocol: none
+
+The MCP door serves a hook's own refusal code demoted off `code`: a hook that refuses with `ASSET_NOT_IN_STOCK` at 409 reaches the agent as `{ error: { code: 'RESOURCE_CONFLICT', status: 409, declaredCode: 'ASSET_NOT_IN_STOCK' } }`. The runner kept only `code` and `status`, so the grader saw a platform conflict and failed a card whose app refused exactly as it should (`check_out_asset_in_repair`, a declared `VALIDATION_FAILED`).
+
+A refused `ObservedAnswer` now carries the envelope's `declaredCode` when the door served one, from a write call, the last read on the object and the runner's probe alike, in the nested and the flat envelope. `refusalMeets` reads it two ways: a declared code equal to `declaredCode` meets exactly, and a declared `VALIDATION_FAILED` is met by a `declaredCode` at any 4xx. A catalogue code with no `declaredCode` still never meets a different declared code, and a `declaredCode` at 5xx or with no status does not meet. The grade's detail names what met ("refused RESOURCE_CONFLICT (409), the hook's declared refusal ASSET_NOT_IN_STOCK").

packages/qa/scenarios/README.md

@@ -67,9 +67,12 @@ gradeCardAnswer(card, { question: 'account_count', persona: 'sdr', turn }); // a
   asks for a correction) and the platform half, `platform:` checks (the runner's own write of
   the declared values, the probe, was refused with that code and stored nothing); so is every
   other `refused` create or update, permission oversteps included (`isProbedOperation`). A
-  declared `VALIDATION_FAILED` is met by an app-defined code at any 4xx status, or by the
-  app's flow refusal (`refusalMeets`); a platform catalogue code other than the declared one
-  never is. The platform half is not measured, out of passed and total with the reason in the
+  declared `VALIDATION_FAILED` is met by the app's own refusal at any 4xx status, or by the
+  app's flow refusal (`refusalMeets`). A hook's declared refusal reaches the MCP data tools
+  with `code` set to the status's catalogue code (`RESOURCE_CONFLICT` at 409) and the hook's
+  own code beside it as `declaredCode` (`ASSET_NOT_IN_STOCK`); the runner keeps both, and a
+  declared code meets its `declaredCode` exactly. A platform catalogue code other than the
+  declared one, with no `declaredCode`, never meets it. The platform half is not measured, out of passed and total with the reason in the
   scorecard's findings, when the probe met the build's own lock: refused 403
   `PERMISSION_DENIED`, nothing written, and the build locks a field the probe writes for the
   persona and declares an action on the object (`buildLockOf`; ruling A on #31999 item 9).

packages/qa/scenarios/src/runner/observe.ts

@@ -139,17 +139,19 @@ export function writeCallsOn(turn: Pick<ParsedTurn, 'toolCalls'>, object: string
  * not a coded refusal (or nothing on the object was called).
  */
 export function lastReadRefusal(turn: Pick<ParsedTurn, 'toolCalls'>, object: string): string | undefined {
-  return lastReadRefusalOf(turn, object)?.code;
+  const output = lastReadRefusalOutput(turn, object);
+  return output === undefined ? undefined : refusalOf(output)?.code;
 }
 
-function lastReadRefusalOf(turn: Pick<ParsedTurn, 'toolCalls'>, object: string): ReturnType<typeof refusalOf> {
+/** The output of the last read on `object` when it carries a coded refusal, else `undefined`. */
+function lastReadRefusalOutput(turn: Pick<ParsedTurn, 'toolCalls'>, object: string): unknown {
   const reads = turn.toolCalls.filter((c) => {
     if ((WRITE_TOOLS as readonly string[]).includes(c.name) || c.output === undefined) return false;
     const input = (typeof c.input === 'object' && c.input !== null ? c.input : {}) as Record<string, unknown>;
     return input.objectName === object;
   });
   const last = reads[reads.length - 1];
-  return last === undefined ? undefined : refusalOf(last.output);
+  return last !== undefined && refusalOf(last.output) !== undefined ? last.output : undefined;
 }
 
 /**
@@ -205,13 +207,54 @@ export function withheldWrite(turn: ParsedTurn, object: string, writes: number):
 
 const failed = (c: RunnerToolCall) => c.isError === true || refusalCodeOf(c.output) !== undefined || flowRefusalOf(c.output) !== undefined;
 
-const refusedBy = (r: NonNullable<ReturnType<typeof refusalOf>>): ObservedAnswer =>
-  r.status !== undefined ? { kind: 'refused', code: r.code, status: r.status } : { kind: 'refused', code: r.code };
+/**
+ * The `declaredCode` of the envelope {@link refusalOf} reads in `output` — the
+ * producer's own unregistered code the door demoted off `code` (#31407: the
+ * MCP data tools' `{ error: { code, message, status, declaredCode } }`, the
+ * resource's flat `{ error, code, status, declaredCode }`) — or `undefined`
+ * when the door served none. The same envelope as {@link refusalOf}: the
+ * nested `error` object when it carries a string `code`, else the flat body.
+ */
+function declaredCodeOf(output: unknown): string | undefined {
+  let v: unknown = output;
+  if (typeof output === 'string') {
+    v = undefined;
+    // The same two readings as `refusalOf`: the whole text, else from its first `{`.
+    for (const text of [output, output.slice(Math.max(0, output.indexOf('{')))]) {
+      try {
+        v = JSON.parse(text);
+        break;
+      } catch {
+        // try the next reading
+      }
+    }
+  }
+  if (typeof v !== 'object' || v === null || Array.isArray(v)) return undefined;
+  const body = v as Record<string, unknown>;
+  const nested = body.error;
+  const env = typeof nested === 'object' && nested !== null && !Array.isArray(nested) && typeof (nested as Record<string, unknown>).code === 'string'
+    ? (nested as Record<string, unknown>)
+    : body;
+  return typeof env.declaredCode === 'string' && env.declaredCode !== '' ? env.declaredCode : undefined;
+}
+
+/** The refused answer of an output carrying a coded refusal: its `code`, `status` and `declaredCode`, each only when served. */
+function refusedBy(output: unknown): ObservedAnswer | undefined {
+  const r = refusalOf(output);
+  if (r === undefined) return undefined;
+  const declaredCode = declaredCodeOf(output);
+  return {
+    kind: 'refused',
+    code: r.code,
+    ...(r.status !== undefined ? { status: r.status } : {}),
+    ...(declaredCode !== undefined ? { declaredCode } : {}),
+  };
+}
 
 /** A write call's refusal by rule 3: its coded refusal, or its flow's declared one; `undefined` when it carries neither. */
 function refusalAnswerOf(c: RunnerToolCall): ObservedAnswer | undefined {
-  const coded = refusalOf(c.output);
-  if (coded !== undefined) return refusedBy(coded);
+  const coded = refusedBy(c.output);
+  if (coded !== undefined) return coded;
   const text = flowRefusalOf(c.output);
   if (text === undefined) return undefined;
   return text === '' ? { kind: 'refused', code: flowRefused } : { kind: 'refused', code: flowRefused, note: `the flow answered ${JSON.stringify(text)}` };
@@ -224,8 +267,8 @@ export function observedAnswer(turn: ParsedTurn, object: string, writes: number)
   const refusals = calls.map(refusalAnswerOf).filter((r) => r !== undefined);
   if (refusals.some((r) => r.kind === 'refused' && r.code === CONFIRMATION_REQUIRED_CODE)) return { kind: 'pending_confirmation' };
   if (refusals.length > 0) return refusals[refusals.length - 1]!;
-  const readRefusal = calls.length === 0 ? lastReadRefusalOf(turn, object) : undefined;
-  if (readRefusal !== undefined) return refusedBy(readRefusal);
+  const readRefusal = calls.length === 0 ? refusedBy(lastReadRefusalOutput(turn, object)) : undefined;
+  if (readRefusal !== undefined) return readRefusal;
   if (calls.length === 0 && asksToConfirm(turn.replyText)) return { kind: 'pending_confirmation' };
   if (calls.length > 0) {
     const last = calls[calls.length - 1]!;

packages/qa/scenarios/src/score/scenario-operation-grade.ts

@@ -85,6 +85,13 @@ export type ObservedAnswer =
       code: string;
       /** The envelope's HTTP `status`, when it carried one. */
       status?: number;
+      /**
+       * The envelope's `declaredCode`, when the door served one: the producer's
+       * own unregistered code (a hook's declared refusal, `ASSET_NOT_IN_STOCK`),
+       * demoted off `code`, which then holds the status's catalogue code
+       * (`RESOURCE_CONFLICT` at 409). Absent when the door served none.
+       */
+      declaredCode?: string;
       /** What the runner read of an answer with no code: the door's text, and whose throw it names. */
       note?: string;
     }
@@ -192,37 +199,46 @@ function isValidationCode(code: string): boolean {
 }
 
 /**
- * The refusal met the declared code. Exactly that code, or — for a declared
- * `VALIDATION_FAILED` — the app's own refusal (ruling A on #31999 item 9,
- * comment 6764953017, widening #31145 item 2), read from the door's answer
- * alone:
+ * The refusal met the declared code. Exactly that code, as the door's `code`
+ * or as its `declaredCode`, or — for a declared `VALIDATION_FAILED` — the
+ * app's own refusal (ruling A on #31999 item 9, comment 6764953017, widening
+ * #31145 item 2), read from the door's answer alone:
  *
- *   - a code that is none of the platform's (it fails the spec's `ErrorCode`
- *     catalogue, ADR-0112) at any client-error status (4xx): the app's own
- *     rule answered, whatever status it chose — a hook's declared refusal
+ *   - a `declaredCode` at any client-error status (4xx): the app's own rule
+ *     answered, whatever status it chose. A hook's declared refusal
  *     (`HookSchema.refusals`, `OVER_RECEIPT` at 422, `ASSET_NOT_IN_STOCK` at
- *     409) reaches the MCP door with its own code;
+ *     409) reaches the MCP data tools with `code` set to the status's
+ *     catalogue code (`VALIDATION_ERROR` at 422, `RESOURCE_CONFLICT` at 409)
+ *     and its own code beside it as `declaredCode` (the door's demotion of an
+ *     unregistered code, #31407), whatever the catalogue `code`;
+ *   - a `code` that is none of the platform's (it fails the spec's `ErrorCode`
+ *     catalogue, ADR-0112) at any 4xx status;
  *   - the flow's declared refusal ({@link flowRefused}): the build routed the
  *     write through an action whose flow refused it.
  *
- * Platform codes stay strict: a catalogue code other than the declared one
- * never meets it at any status (a denial at 403, a missing row at 404, a
- * conflict at 409, an error at 500). Nor does an app code with no status or a
- * 5xx one, nor the runner's other parenthesized labels.
+ * Platform codes stay strict: a catalogue code with no `declaredCode`, other
+ * than the declared one, never meets it at any status (a denial at 403, a
+ * missing row at 404, a conflict at 409, an error at 500). Nor does an app
+ * code or a `declaredCode` with no status or a 5xx one, nor the runner's other
+ * parenthesized labels.
  */
 export function refusalMeets(declared: string, answer: ObservedAnswer): boolean {
   if (answer.kind !== 'refused') return false;
-  if (answer.code === declared) return true;
+  if (answer.code === declared || answer.declaredCode === declared) return true;
   if (!isValidationCode(declared)) return false;
   if (answer.code === flowRefused) return true;
   if (answer.status === undefined || answer.code.startsWith('(')) return false;
+  const clientError = answer.status >= 400 && answer.status < 500;
+  if (answer.declaredCode !== undefined) return clientError;
   if (ErrorCode.safeParse(answer.code).success) return false;
-  return answer.status >= 400 && answer.status < 500;
+  return clientError;
 }
 
 const refusalText = (declared: string, a: ObservedAnswer): string => {
   if (!refusalMeets(declared, a) || a.kind !== 'refused' || a.code === declared) return answerText(a);
-  return `${answerText(a)}, ${a.code === flowRefused ? "the app's flow refusal" : 'an app-defined refusal'}`;
+  if (a.code === flowRefused) return `${answerText(a)}, the app's flow refusal`;
+  if (a.declaredCode !== undefined) return `${answerText(a)}, the hook's declared refusal ${a.declaredCode}`;
+  return `${answerText(a)}, an app-defined refusal`;
 };
 
 /** The door refused the probe as a field lock refuses: `PERMISSION_DENIED` at 403. */