fix(mcp): validate_expression keeps the hidden __search companion out of scope

.changeset/mcp-validate-expression-search-companion.md

@@ -0,0 +1,16 @@
+---
+'@objectstack/mcp': patch
+'@objectstack/spec': patch
+'@objectstack/objectql': patch
+---
+
+fix(mcp): `validate_expression` no longer lists the hidden `__search` search companion in `inScope.fields`, and a read of it is refused as `os validate` refuses it
+
+Clause-②: yes (widening) — `@objectstack/spec/data` gains the `SEARCH_COMPANION_FIELD` constant (`'__search'`); `@objectstack/objectql` re-exports the same value it exported before.
+Metadata-protocol: none
+
+Where pinyin search is on, the object an MCP caller is served carries the hidden `__search` companion column beside its declared fields. `validate_expression` took its field list from that object, so it named `__search` in `inScope.fields` and answered `ok` for an expression such as `record.__search != ''`. Every record read strips that column, and `os validate` judges an expression against the declared and injected columns only, so the same rule failed there with `unknown-field`.
+
+The tool now leaves the companion out of the fields it judges and lists. Every other field, injected system columns such as `organization_id` among them, is listed and judged as before.
+
+The companion's name now lives in the protocol, as `SEARCH_COMPANION_FIELD` in `@objectstack/spec/data`, so a layer that only needs the name does not have to load the engine to read it. `@objectstack/objectql` provisions and strips the column under that same constant and still exports it under the same name, so engine callers change nothing. `@objectstack/mcp` reads it from the spec and gains no runtime dependency on the engine.

packages/mcp/package.json

@@ -38,6 +38,7 @@
     "zod": "^4.6.6"
   },
   "devDependencies": {
+    "@objectstack/objectql": "workspace:*",
     "@types/node": "^26.6.5",
     "typescript": "^6.0.3",
     "vitest": "^4.1.11"

packages/mcp/src/mcp-http-tools.ts

@@ -48,7 +48,7 @@ import type { EngineAggregateOptions } from '@objectstack/spec/data';
 // entry IS this node, so this door and the REST query door (which parses
 // `FindDataRequestSchema`, whose `aggregations` are this node) judge one shape
 // with one refusal.
-import { AggregationNodeSchema } from '@objectstack/spec/data';
+import { AggregationNodeSchema, SEARCH_COMPANION_FIELD } from '@objectstack/spec/data';
 // [#23529] The catalogue's spelling of `PERMISSION_DENIED`, the code whose
 // `details` `errorResultFromThrown` withholds. Imported, never hand-spelled.
 import { StandardErrorCode, type ErrorCode } from '@objectstack/spec/api';
@@ -2632,8 +2632,16 @@ export function registerObjectTools(
           const fieldDefs = Array.isArray(def.fields) ? (def.fields as Array<Record<string, unknown>>) : [];
           const fields: string[] = [];
           const fieldTypes: Record<string, string> = {};
+          // The hidden search companion is served beside the declared fields
+          // (the registry stamps it where pinyin search is on), but no
+          // expression may read it: every read strips it, and `os validate`
+          // judges against the declared and injected columns, which never
+          // include it. Left in, it is listed in `inScope.fields` and a read
+          // of it answers ok here while the gate refuses it. The name is the
+          // protocol's one constant, the same one the engine provisions under.
           for (const f of fieldDefs) {
             if (typeof f?.name !== 'string') continue;
+            if (f.name === SEARCH_COMPANION_FIELD) continue;
             fields.push(f.name);
             if (typeof f?.type === 'string') fieldTypes[f.name] = f.type;
           }

packages/mcp/src/mcp-validate-expression.test.ts

@@ -16,8 +16,9 @@ import {
 } from '@objectstack/lint';
 
 import { applyInjectedSystemColumns } from '@objectstack/metadata-core';
+import { provisionSearchCompanion, SEARCH_COMPANION_FIELD } from '@objectstack/objectql/core';
 
-import { MCPServerRuntime } from './mcp-server-runtime.js';
+import { MCPServerRuntime, describeServedObject } from './mcp-server-runtime.js';
 import type { McpDataBridge } from './mcp-http-tools.js';
 
 /**
@@ -919,3 +920,61 @@ describe('validate_expression judges the field being authored at the formula sit
     expect((json as { result: { isError?: boolean } }).result.isError).toBe(true);
   });
 });
+
+describe('validate_expression keeps the hidden search companion out of scope', () => {
+  // A field-scoped search object as the registry serves it with pinyin search
+  // on: the companion stamped beside the declared fields, then projected the
+  // way the transports project it.
+  const authored = {
+    name: 'sc_contact',
+    label: 'Contact',
+    fields: {
+      name: { type: 'text', label: 'Name', required: true },
+      email: { type: 'email', label: 'Email' },
+    },
+  };
+  const served = provisionSearchCompanion(applyInjectedSystemColumns(authored));
+  const bridge = (): McpDataBridge => ({
+    ...makeBridge(),
+    async listObjects() {
+      return [{ name: 'sc_contact', label: 'Contact', fieldCount: Object.keys(served.fields).length }];
+    },
+    async describeObject(name: string) {
+      return name === 'sc_contact' ? describeServedObject(served as never) : null;
+    },
+  });
+
+  let runtime: MCPServerRuntime;
+  beforeEach(() => {
+    runtime = new MCPServerRuntime({ name: 'objectstack-test', version: '9.9.9' });
+  });
+
+  it('the served object really carries the companion (precondition)', () => {
+    expect(Object.keys(served.fields)).toContain(SEARCH_COMPANION_FIELD);
+  });
+
+  it('inScope.fields lists the ordinary fields and no companion', async () => {
+    const r = await validate(runtime, bridge(), { objectName: 'sc_contact', expression: "record.name != ''", site: 'validation' });
+    expect(r.ok).toBe(true);
+    expect(r.inScope.fields).toContain('name');
+    expect(r.inScope.fields).toContain('email');
+    expect(r.inScope.fields).toContain('created_at');
+    expect(r.inScope.fields).not.toContain(SEARCH_COMPANION_FIELD);
+  });
+
+  it('a read of the companion is refused as a field the object does not have, as os validate refuses it', async () => {
+    const expression = `record.${SEARCH_COMPANION_FIELD} != ''`;
+    const gate = validateStackExpressions({
+      objects: [{ ...authored, validations: [{ name: AUTHORED_RULE_NAME, type: 'script', condition: expression, message: 'm' }] }],
+    }).filter((i) => i.where.includes(`'${AUTHORED_RULE_NAME}'`));
+    expect(gate.map((i) => i.severity ?? 'error')).toEqual(['error']);
+    expect(gate[0]!.message).toContain(SEARCH_COMPANION_FIELD);
+
+    for (const site of ['validation', 'formula', 'flow_condition'] as const) {
+      const r = await validate(runtime, bridge(), { objectName: 'sc_contact', expression, site });
+      expect(r.ok, site).toBe(false);
+      const refused = r.errors.filter((e: { code?: string }) => e.code === 'unknown-field');
+      expect(refused.map((e: { params?: { field?: string } }) => e.params?.field), site).toEqual([SEARCH_COMPANION_FIELD]);
+    }
+  });
+});

packages/mcp/vitest.config.ts

@@ -57,10 +57,18 @@ export default defineConfig({
     // which drives the stdio exit through the shared ADR-0106 case table: the
     // table is a VALUE import too, and judging this transport against a stale
     // build of it would be the same silent wrong-tree verdict.
+    //
+    // The `@objectstack/objectql/core` entry (the lean engine entry, ADR-0076)
+    // is for `mcp-validate-expression.test.ts`, which provisions the search
+    // companion with the engine's own `provisionSearchCompanion` and reads its
+    // name from `SEARCH_COMPANION_FIELD`: VALUE imports, judged against the
+    // source in the checkout, not a build of it. Only the test loads the
+    // engine; the tool reads the name from `@objectstack/spec/data`.
     alias: [
       { find: /^@objectstack\/metadata-core$/, replacement: path.resolve(__dirname, '../metadata-core/src/index.ts') },
       { find: /^@objectstack\/metadata-core\/testing$/, replacement: path.resolve(__dirname, '../metadata-core/src/testing.ts') },
       { find: /^@objectstack\/lint$/, replacement: path.resolve(__dirname, '../lint/src/index.ts') },
+      { find: /^@objectstack\/objectql\/core$/, replacement: path.resolve(__dirname, '../objectql/src/core.ts') },
     ],
   },
 });

packages/objectql/src/search-companion.ts

@@ -40,16 +40,17 @@
  * the same column.
  */
 
-import { resolveDisplayField, TITLE_ELIGIBLE_TYPES } from '@objectstack/spec/data';
+import { resolveDisplayField, SEARCH_COMPANION_FIELD, TITLE_ELIGIBLE_TYPES } from '@objectstack/spec/data';
 import { SystemFieldName } from '@objectstack/spec/system';
 import { isFederatedObject } from './federated-object.js';
 
 /**
- * Name of the hidden companion column. Double-underscore prefixed so it can
- * never collide with an author-declared field (machine names are snake_case
- * words) and reads as platform plumbing in raw rows.
+ * Name of the hidden companion column. The name is the protocol's
+ * (`@objectstack/spec/data`), re-exported here so engine callers keep
+ * importing it from the engine; see the declaration there for why it lives in
+ * the spec.
  */
-export const SEARCH_COMPANION_FIELD = '__search';
+export { SEARCH_COMPANION_FIELD };
 
 /**
  * Normalizers applied to the companion column. Generalization seam: pinyin is

packages/spec/api-surface/data.json

@@ -725,6 +725,7 @@
     "SEARCHABLE_TEXTUAL_TYPES (const)",
     "SEARCH_AUTO_EXCLUDED_FIELDS (const)",
     "SEARCH_AUTO_EXCLUDED_TYPES (const)",
+    "SEARCH_COMPANION_FIELD (const)",
     "SEARCH_FLAG_EXCLUDED_TYPES (const)",
     "SEARCH_VIRTUAL_TYPES (const)",
     "SECRET_MASK (const)",

packages/spec/export-origins/data.json

@@ -713,6 +713,7 @@
     "SEARCHABLE_TEXTUAL_TYPES": "src/data/search-fields.ts#SEARCHABLE_TEXTUAL_TYPES (const)",
     "SEARCH_AUTO_EXCLUDED_FIELDS": "src/data/search-fields.ts#SEARCH_AUTO_EXCLUDED_FIELDS (const)",
     "SEARCH_AUTO_EXCLUDED_TYPES": "src/data/search-fields.ts#SEARCH_AUTO_EXCLUDED_TYPES (const)",
+    "SEARCH_COMPANION_FIELD": "src/data/search-fields.ts#SEARCH_COMPANION_FIELD (const)",
     "SEARCH_FLAG_EXCLUDED_TYPES": "src/data/search-fields.ts#SEARCH_FLAG_EXCLUDED_TYPES (const)",
     "SEARCH_VIRTUAL_TYPES": "src/data/search-fields.ts#SEARCH_VIRTUAL_TYPES (const)",
     "SECRET_MASK": "src/data/secret-mask.ts#SECRET_MASK (const)",

packages/spec/src/data/search-fields.ts

@@ -359,3 +359,20 @@ export function resolveSearchFields(opts: SearchFieldResolutionOptions): string[
   }
   return allowed;
 }
+
+/**
+ * Name of the hidden search companion column (#2486): the lowercase,
+ * normalised copy of an object's searchable text that the engine provisions
+ * where pinyin search is on, and the one extra `$contains` clause above scans.
+ * It is platform plumbing: every record read strips it, `resolveSearchFields`
+ * never returns it, and no expression may read it.
+ *
+ * ONE name for every layer that must recognise the column: the engine
+ * (`@objectstack/objectql` provisions and strips it under this name and
+ * re-exports it) and the MCP `validate_expression` tool (which leaves it out
+ * of the fields an expression may name). It lives here, not in the engine, so
+ * a layer that only needs the name does not have to load the engine to read it.
+ * Double-underscore prefixed so it can never collide with an author-declared
+ * field (machine names are snake_case words).
+ */
+export const SEARCH_COMPANION_FIELD = '__search';