Skip to content

Commit 128f4e2

Browse files
Merge pull request #127 from ScriptedAlchemy/t3/tool-output-noise
fix: readable tool results and less noise from pull_request_failures
2 parents 735ed97 + 5f4ff6a commit 128f4e2

7 files changed

Lines changed: 186 additions & 13 deletions

File tree

‎apps/web/src/components/ChatView.browser.test.tsx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,8 @@ it("an interrupted image-only first send of a new thread says it may not have be
284284
.element()
285285
.dispatchEvent(new ClipboardEvent("paste", { clipboardData, bubbles: true, cancelable: true }));
286286
await expect.element(page.getByRole("img", { name: "shot.png" })).toBeVisible();
287+
// The thumbnail is visible while compression still blocks sending.
288+
await expect.element(page.getByRole("button", { name: "Preview shot.png" })).toBeEnabled();
287289
await userEvent.keyboard("{Enter}");
288290
await vi.waitFor(() => expect(launch.calls).toHaveLength(1));
289291
await userEvent.keyboard("next draft");

‎packages/client-runtime/src/work-log/itemDetail.test.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,3 +182,27 @@ describe("MCP app tool output", () => {
182182
expect(turnItemOutputText(bounded)).toBe("chart\n[image]");
183183
});
184184
});
185+
186+
describe("MCP results with the SDK's _meta", () => {
187+
const meta = { "io.modelcontextprotocol/serverInfo": { name: "T3 Code", version: "1" } };
188+
const structuredContent = {
189+
check: "Server tests 7",
190+
message: "expected [ 60 ]\nto equal [ 59 ]",
191+
};
192+
it.each([
193+
[
194+
"text and structured content",
195+
{ content: JSON.stringify(structuredContent), structuredContent },
196+
],
197+
["text blocks only", { content: [{ type: "text", text: "Server tests: 7 passed" }] }],
198+
[
199+
"text blocks and structured content",
200+
{ content: [{ type: "text", text: JSON.stringify(structuredContent) }], structuredContent },
201+
],
202+
])("reads %s the same with or without _meta", (_shape, result) => {
203+
const plain = turnItemOutputText(toolItem(result));
204+
const withMeta = turnItemOutputText(toolItem({ ...result, _meta: meta }));
205+
expect(withMeta).toBe(plain);
206+
expect(withMeta).not.toContain("serverInfo");
207+
});
208+
});

‎packages/client-runtime/src/work-log/itemDetail.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,10 @@ function textFromBlocks(
4040
if (typeof resource.text === "string") return resource.text;
4141
if (typeof resource.uri === "string") return resource.uri;
4242
}
43-
const keys = Object.keys(value).filter((key) => key !== "isError" && key !== "is_error");
43+
// MCP SDKs stamp `_meta` with server info on every result; it is never the tool's output.
44+
const keys = Object.keys(value).filter(
45+
(key) => key !== "isError" && key !== "is_error" && key !== "_meta",
46+
);
4447
if (keys.length === 1 && keys[0] === "content")
4548
return textFromBlocks(value.content, depth + 1, imageBlocks);
4649
if (keys.length === 2 && keys.includes("content") && keys.includes("structuredContent")) {

‎packages/source-control-core/src/server/pullRequestChecks.test.ts‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,45 @@ describe("check log excerpts", () => {
227227
lines: 1,
228228
},
229229
{ chunks: ["one\ntw", "o\nthree"], text: "one\ntwo\nthree", lines: 3 },
230+
{
231+
chunks: [
232+
[
233+
" ❯ ~effect/Effect/successCont ../../node_modules/.pnpm/effect@4.1.3/node_modules/effect/dist/esm/Effect.js:1:1",
234+
" ❯ FiberImpl.runLoop ../../node_modules/.pnpm/effect@4.1.3/node_modules/effect/dist/esm/internal/fiberRuntime.js:1:1",
235+
"##[start-action display=Install dependencies;id=install]",
236+
"##[end-action id=install;outcome=success;conclusion=success;duration_ms=900]",
237+
"Post job cleanup.",
238+
"Cleaning up orphan processes",
239+
" ❯ runAssertion src/example.test.ts:10:5",
240+
].join("\n"),
241+
],
242+
text: " ❯ runAssertion src/example.test.ts:10:5",
243+
lines: 7,
244+
},
245+
{
246+
chunks: [
247+
[
248+
"##[error]TypeError: boom",
249+
" at Object.<anonymous> (/home/runner/work/app/node_modules/foo/index.js:1:1)",
250+
" at Object.<anonymous> (/home/runner/work/app/src/build.js:2:2)",
251+
].join("\n"),
252+
],
253+
text: "##[error]TypeError: boom\n at Object.<anonymous> (/home/runner/work/app/src/build.js:2:2)",
254+
lines: 3,
255+
},
256+
{
257+
chunks: [
258+
[
259+
"##[start-action display=Install dependencies;id=install]",
260+
"##[end-action id=install;outcome=success;conclusion=success;duration_ms=900]",
261+
"##[start-action display=Build;id=build]",
262+
"##[error]The action has timed out.",
263+
"##[end-action id=build;outcome=failure;conclusion=failure;duration_ms=1250]",
264+
].join("\n"),
265+
],
266+
text: "##[error]The action has timed out.\n##[end-action id=build;outcome=failure;conclusion=failure;duration_ms=1250]",
267+
lines: 5,
268+
},
230269
])("strips formatting and counts lines across chunks %#", ({ chunks, text, lines }) => {
231270
expect(excerptCheckLog(chunks)).toEqual({
232271
_tag: "excerpt",
@@ -236,6 +275,42 @@ describe("check log excerpts", () => {
236275
});
237276
});
238277

278+
it("does not let noise consume error context or tail slots", () => {
279+
const noise = [
280+
" ❯ FiberImpl.runLoop ../../node_modules/.pnpm/effect@4.1.3/node_modules/effect/dist/esm/internal/fiberRuntime.js:1:1",
281+
"##[start-action display=Install dependencies;id=install]",
282+
"##[end-action id=install;outcome=success;conclusion=success;duration_ms=900]",
283+
"Post job cleanup.",
284+
"Cleaning up orphan processes",
285+
];
286+
const expected = [
287+
"##[group]Run vp test",
288+
"before 1",
289+
"before 2",
290+
"before 3",
291+
"##[error]Assertion failed",
292+
"after 1",
293+
"after 2",
294+
"after 3",
295+
...Array.from({ length: 20 }, (_, i) => `tail ${i}`),
296+
];
297+
const result = excerptCheckLog([
298+
[
299+
...expected.slice(0, 4),
300+
...noise,
301+
...expected.slice(4, 5),
302+
...noise,
303+
...expected.slice(5),
304+
...noise,
305+
].join("\n"),
306+
]);
307+
308+
expect(result.text).toBe(expected.join("\n"));
309+
expect(result.totalLines).toBe(expected.length + noise.length * 3);
310+
// Every real line is present, so dropped noise alone does not mark the excerpt truncated.
311+
expect(result.truncated).toBe(false);
312+
});
313+
239314
it("keeps error context, its Run header and the tail within both budgets", () => {
240315
const lines = [
241316
"##[group]Run vp test",

‎packages/source-control-core/src/server/pullRequestChecks.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,17 @@ export const CHECK_LOG_EXCERPT_BUDGET = {
117117
} as const;
118118
type Line = { index: number; text: string };
119119

120+
// A failed composite step's end marker is the only record of which nested step failed.
121+
const isLogNoise = (text: string) =>
122+
(/^\s*(?:❯|at)\s/.test(text) && /node_modules[\\/]/.test(text)) ||
123+
/^##\[start-action /.test(text) ||
124+
/^##\[end-action .*outcome=(?:success|skipped)\b/.test(text) ||
125+
/^(?:Post job cleanup\.|Cleaning up orphan processes)/.test(text);
126+
120127
export class CheckLogExcerpt {
121128
private pending = "";
122129
private totalLines = 0;
130+
private noiseLines = 0;
123131
private clipped = false;
124132
private afterError = 0;
125133
private header: Line | undefined;
@@ -150,6 +158,10 @@ export class CheckLogExcerpt {
150158
.replace(/^\d{4}-\d\d-\d\dT[\d:.]+Z\s?/, "")
151159
.replace(/\r$/, ""),
152160
};
161+
if (isLogNoise(line.text)) {
162+
this.noiseLines++;
163+
return;
164+
}
153165
// Step names do not appear in the log; a section failed when it carries ##[error].
154166
if (line.text.startsWith("##[group]Run ")) {
155167
if (this.failedSection) this.failedTail = this.tail;
@@ -208,7 +220,7 @@ export class CheckLogExcerpt {
208220
_tag: "excerpt",
209221
text: lines.join("\n"),
210222
totalLines: this.totalLines,
211-
truncated: this.clipped || lines.length < this.totalLines,
223+
truncated: this.clipped || lines.length < this.totalLines - this.noiseLines,
212224
};
213225
}
214226
}

‎packages/source-control-github/src/server/GitHubRest.test.ts‎

Lines changed: 53 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -722,11 +722,35 @@ describe("GitHub check failures", () => {
722722
check_run_url: "https://api.github.com/repos/acme/web/check-runs/123",
723723
steps: [{ name: "Run lint", conclusion: "failure" }],
724724
};
725+
const exitOneAnnotation = {
726+
path: ".github",
727+
start_line: 0,
728+
annotation_level: "failure",
729+
message: "Process completed with exit code 1.",
730+
};
725731

726732
it.effect("sees the signed log redirect itself and follows it without Authorization", () => {
727733
const { layer, calls, fetch } = fetchHarness((url) => {
728734
if (url.includes("/annotations"))
729735
return json([
736+
{
737+
path: ".github",
738+
start_line: 0,
739+
annotation_level: "failure",
740+
message: "Process completed with exit code 1.",
741+
},
742+
{
743+
path: ".github",
744+
start_line: 0,
745+
annotation_level: "failure",
746+
message: "Process completed with exit code 137.",
747+
},
748+
{
749+
path: ".github/workflows/ci.yml",
750+
start_line: 0,
751+
annotation_level: "failure",
752+
message: "Process completed with exit code 1.",
753+
},
730754
{ path: "a.ts", start_line: 4, annotation_level: "failure", message: "bad code" },
731755
{ path: "budget.json", start_line: 0, annotation_level: "failure", message: "grew" },
732756
]);
@@ -736,7 +760,9 @@ describe("GitHub check failures", () => {
736760
headers: { location: "https://blob.example/log?signature=secret" },
737761
});
738762
if (url.startsWith("https://blob.example/"))
739-
return new Response("2026-10-30T12:00:00Z ##[error]bad code\n");
763+
return new Response(
764+
"2026-10-30T12:00:00Z ##[error]bad code\n##[error]Process completed with exit code 1.\n",
765+
);
740766
return json(job);
741767
});
742768
return Effect.gen(function* () {
@@ -746,10 +772,16 @@ describe("GitHub check failures", () => {
746772
attempt: 1,
747773
failedSteps: ["Run lint"],
748774
annotations: [
775+
{ path: ".github", startLine: 0, message: "Process completed with exit code 137." },
749776
{ path: "a.ts", startLine: 4, message: "bad code" },
750777
{ path: "budget.json", startLine: 0, message: "grew" },
751778
],
752-
log: { _tag: "excerpt", text: "##[error]bad code", totalLines: 1, truncated: false },
779+
log: {
780+
_tag: "excerpt",
781+
text: "##[error]bad code\n##[error]Process completed with exit code 1.",
782+
totalLines: 2,
783+
truncated: false,
784+
},
753785
});
754786
expect(result).not.toHaveProperty("required");
755787
const authorization = (call: (typeof calls)[number]) =>
@@ -770,7 +802,7 @@ describe("GitHub check failures", () => {
770802
it.effect("reads an untouched job from an earlier attempt after a rerun", () => {
771803
const { layer } = harness((request) =>
772804
request.url.includes("/annotations")
773-
? json([])
805+
? json([exitOneAnnotation])
774806
: request.url.endsWith("/logs")
775807
? new Response("##[error]bad code\n")
776808
: json(job),
@@ -783,6 +815,9 @@ describe("GitHub check failures", () => {
783815
});
784816
expect(result).toMatchObject({
785817
attempt: 1,
818+
annotations: [
819+
{ path: ".github", startLine: 0, message: "Process completed with exit code 1." },
820+
],
786821
log: { _tag: "excerpt", text: "##[error]bad code" },
787822
});
788823
}).pipe(Effect.provide(layer));
@@ -816,14 +851,27 @@ describe("GitHub check failures", () => {
816851
])("returns $reason for $name without extra reads", (test) => {
817852
const { layer, requests } = harness((request) =>
818853
request.url.includes("/annotations")
819-
? json([])
854+
? json([exitOneAnnotation])
820855
: request.url.endsWith("/logs")
821856
? json({ message: "Unavailable" }, { status: test.logStatus })
822857
: json(test.job),
823858
);
824859
return Effect.gen(function* () {
825860
const read = yield* makeGitHubCheckFailureReader;
826-
expect((yield* read(input)).log).toEqual({ _tag: "unavailable", reason: test.reason });
861+
const result = yield* read(input);
862+
expect(result.log).toEqual({ _tag: "unavailable", reason: test.reason });
863+
expect(result.annotations).toEqual(
864+
test.requests === 3
865+
? [
866+
{
867+
path: ".github",
868+
startLine: 0,
869+
level: "failure",
870+
message: exitOneAnnotation.message,
871+
},
872+
]
873+
: [],
874+
);
827875
expect(requests).toHaveLength(test.requests);
828876
}).pipe(Effect.provide(layer));
829877
});

‎packages/source-control-github/src/server/gitHubCheckFailures.ts‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -165,12 +165,21 @@ export const makeGitHubCheckFailureReader = Effect.gen(function* () {
165165
status: JOB_CONCLUSIONS[job.conclusion ?? ""] ?? "neutral",
166166
attempt: job.run_attempt,
167167
failedSteps: failedSteps.slice(0, 20).map((name) => name.slice(0, 300)),
168-
annotations: annotations.slice(0, 20).map((annotation) => ({
169-
path: annotation.path.slice(0, 1024),
170-
startLine: annotation.start_line,
171-
level: annotation.annotation_level,
172-
message: annotation.message.slice(0, 300),
173-
})),
168+
annotations: annotations
169+
.filter(
170+
// Only drop exit 1 when the delivered excerpt already contains it.
171+
(annotation) =>
172+
annotation.message !== "Process completed with exit code 1." ||
173+
log._tag !== "excerpt" ||
174+
!log.text.includes(annotation.message),
175+
)
176+
.slice(0, 20)
177+
.map((annotation) => ({
178+
path: annotation.path.slice(0, 1024),
179+
startLine: annotation.start_line,
180+
level: annotation.annotation_level,
181+
message: annotation.message.slice(0, 300),
182+
})),
174183
log,
175184
} satisfies PullRequestCheckFailure;
176185
});

0 commit comments

Comments
 (0)