fix(automate): read visual-log screenshots in fetchAutomationScreenshots

src/tools/automate-utils/fetch-screenshots.ts

@@ -32,45 +32,93 @@ async function extractScreenshotUrls(
       ? response.data
       : JSON.stringify(response.data);
 
+  return parseScreenshotUrls(text);
+}
+
+const LOG_PREFIX = String.raw`^\d{4}-\d{2}-\d{2} \d{1,2}:\d{1,2}:\d{1,2}:\d{1,3} `;
+const LINE_TYPE_PATTERN = new RegExp(`${LOG_PREFIX}([A-Z_]+) `);
+const SCREENSHOT_REQUEST_PATTERN = new RegExp(
+  String.raw`${LOG_PREFIX}REQUEST \[[^\]]+\] GET /session/[^/\s]+/screenshot(\s|$)`,
+);
+const RESPONSE_VALUE_PATTERN = new RegExp(
+  String.raw`${LOG_PREFIX}RESPONSE \{"value"\s*:\s*"(https://[^"]+)"`,
+);
+const DEBUG_SCREENSHOT_PATTERN = new RegExp(
+  String.raw`${LOG_PREFIX}DEBUG (https://\S+)$`,
+);
+const TRUSTED_HOST_PATTERN =
+  /(^|\.)browserstack\.com$|\.s3([.-][a-z0-9-]+)*\.amazonaws\.com$|\.backblazeb2\.com$/;
+
+function isTrustedScreenshotUrl(value: string): boolean {
+  try {
+    const url = new URL(value);
+    return url.protocol === "https:" && TRUSTED_HOST_PATTERN.test(url.hostname);
+  } catch {
+    return false;
+  }
+}
+
+export function parseScreenshotUrls(text: string): string[] {
   const urls: string[] = [];
-  const SCREENSHOT_PATTERN = /REQUEST.*GET.*\/screenshot/;
-  const RESPONSE_VALUE_PATTERN = /"value"\s*:\s*"([^"]+)"/;
+  const lines = text.split("\n").map((line) => line.trimEnd());
 
-  // Split logs into lines and process them
-  const lines = text.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
 
-  for (let i = 0; i < lines.length - 1; i++) {
-    const currentLine = lines[i];
-    const nextLine = lines[i + 1];
+    const debugMatch = line.match(DEBUG_SCREENSHOT_PATTERN);
+    if (debugMatch) {
+      urls.push(debugMatch[1]);
+      continue;
+    }
 
-    if (SCREENSHOT_PATTERN.test(currentLine)) {
-      const match = nextLine.match(RESPONSE_VALUE_PATTERN);
-      if (match && match[1]) {
-        urls.push(match[1]);
+    if (!SCREENSHOT_REQUEST_PATTERN.test(line)) {
+      continue;
+    }
+    for (let j = i + 1; j < lines.length; j++) {
+      const lineType = lines[j].match(LINE_TYPE_PATTERN)?.[1];
+      if (lineType === "REQUEST") {
+        break;
+      }
+      if (lineType === "RESPONSE") {
+        const valueMatch = lines[j].match(RESPONSE_VALUE_PATTERN);
+        if (valueMatch) {
+          urls.push(valueMatch[1]);
+        }
+        break;
       }
     }
   }
 
-  return urls;
+  return [...new Set(urls)].filter(isTrustedScreenshotUrl);
 }
 
-async function fetchAndCompress(
-  url: string,
-): Promise<{ url: string; base64: string }> {
+function detectImageMimeType(buffer: Buffer): string {
+  if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {
+    return "image/jpeg";
+  }
+  return "image/png";
+}
+
+export type Screenshot = { url: string; base64: string; mimeType: string };
+
+async function fetchAndCompress(url: string): Promise<Screenshot> {
   const response = await apiClient.get({ url, responseType: "arraybuffer" });
   // Axios returns response.data as a Buffer for binary data
-  const base64 = Buffer.from(response.data).toString("base64");
-  return { url, base64: await maybeCompressBase64(base64) };
+  const buffer = Buffer.from(response.data);
+  const base64 = await maybeCompressBase64(buffer.toString("base64"));
+  return {
+    url,
+    base64,
+    mimeType: detectImageMimeType(Buffer.from(base64, "base64")),
+  };
 }
 
 //Converts screenshot URLs to base64 encoded images
-async function convertUrlsToBase64(
-  urls: string[],
-): Promise<Array<{ url: string; base64: string }>> {
+async function convertUrlsToBase64(urls: string[]): Promise<Screenshot[]> {
   // Hosted pods are memory-constrained: process one image at a time. Local runs
   // keep the concurrent path.
   if (globalConfig.REMOTE_MCP) {
-    const screenshots: Array<{ url: string; base64: string }> = [];
+    const screenshots: Screenshot[] = [];
     for (const url of urls) {
       screenshots.push(await fetchAndCompress(url));
     }

src/tools/automate.ts

@@ -36,7 +36,7 @@ export async function fetchAutomationScreenshotsTool(
         content: [
           {
             type: "text",
-            text: "No screenshots found in the session or some unexpected error occurred",
+            text: "No screenshots found in the session logs. Run with browserstack.debug: true to capture visual-log screenshots.",
           },
         ],
         isError: true,
@@ -46,7 +46,7 @@ export async function fetchAutomationScreenshotsTool(
     const results = screenshots.map((screenshot, index) => ({
       type: "image" as const,
       data: screenshot.base64,
-      mimeType: "image/png",
+      mimeType: screenshot.mimeType,
       _meta: { url: screenshot.url, index: index + 1 },
     }));
 

tests/tools/automate.test.ts

@@ -21,8 +21,8 @@ describe("fetchAutomationScreenshotsTool", () => {
 
   it("SUCCESS: returns screenshots as image content", async () => {
     (fetchAutomationScreenshots as Mock).mockResolvedValue([
-      { base64: "abc123", url: "https://example.com/1.png" },
-      { base64: "def456", url: "https://example.com/2.png" },
+      { base64: "abc123", url: "https://example.com/1.png", mimeType: "image/png" },
+      { base64: "def456", url: "https://example.com/2.jpeg", mimeType: "image/jpeg" },
     ]);
 
     const result = await fetchAutomationScreenshotsTool(
@@ -34,6 +34,8 @@ describe("fetchAutomationScreenshotsTool", () => {
     expect(result.content.length).toBe(3); // 1 text + 2 images
     expect(result.content[0].type).toBe("text");
     expect(result.content[1].type).toBe("image");
+    expect(result.content[1].mimeType).toBe("image/png");
+    expect(result.content[2].mimeType).toBe("image/jpeg");
   });
 
   it("SUCCESS: returns isError when no screenshots found", async () => {

tests/tools/fetch-screenshots.test.ts

@@ -0,0 +1,177 @@
+import { describe, it, expect, vi, beforeEach, Mock } from "vitest";
+import {
+  fetchAutomationScreenshots,
+  parseScreenshotUrls,
+} from "../../src/tools/automate-utils/fetch-screenshots";
+import { SessionType } from "../../src/lib/constants";
+import { apiClient } from "../../src/lib/apiClient";
+
+vi.mock("../../src/lib/apiClient", () => ({
+  apiClient: { get: vi.fn() },
+}));
+vi.mock("../../src/logger", () => ({
+  default: { error: vi.fn(), info: vi.fn(), debug: vi.fn(), warn: vi.fn() },
+}));
+
+const mockConfig = {
+  "browserstack-username": "fake-user",
+  "browserstack-access-key": "fake-key",
+};
+
+const S = "9eaca30c2d3fb8517ac11e469724ab2421bc63c3";
+const S3 = `https://automate.browserstack.com/s3-debug/browserstack-debug-screenshots-prod-use1/${S}`;
+
+const debugOnlyLogs = [
+  `2026-10-13 9:12:45:16 REQUEST [2026-10-13 9:12:45:16] POST /session/${S}/url {"url":"https://bstackdemo.com/signin"}`,
+  `2026-10-13 9:12:46:632 DEBUG ${S3}/screenshot-4.jpeg`,
+  `2026-10-13 9:12:46:633 RESPONSE {"text":{"response":{"guid":"response@23b4"}}}`,
+  `2026-10-13 9:12:48:839 REQUEST [2026-10-13 9:12:48:839] POST /session/${S}/click {"selector":"#username"}`,
+  `2026-10-13 9:12:49:127 DEBUG ${S3}/screenshot-6.jpeg`,
+  `2026-10-13 9:12:49:200 RESPONSE {"text":{}}`,
+].join("\n");
+
+const requestedOnlyLogs = [
+  `2026-10-13 9:18:20:871 REQUEST [2026-10-13 9:18:20:871] GET /session/${S}/screenshot {"text":{"fullPage":false}}`,
+  `2026-10-13 9:18:21:3 RESPONSE {"value":"${S3}/screenshot-playwright-85c8de7c55.png"}`,
+  `2026-10-13 9:18:23:992 REQUEST [2026-10-13 9:18:23:992] GET /session/${S}/screenshot {"text":{}}`,
+  `2026-10-13 9:18:24:81 RESPONSE {"value":"${S3}/screenshot-playwright-304ad8e401.png"}`,
+].join("\n");
+
+describe("parseScreenshotUrls", () => {
+  it("extracts DEBUG visual-log screenshots", () => {
+    expect(parseScreenshotUrls(debugOnlyLogs)).toEqual([
+      `${S3}/screenshot-4.jpeg`,
+      `${S3}/screenshot-6.jpeg`,
+    ]);
+  });
+
+  it("extracts screenshots requested via GET /screenshot", () => {
+    expect(parseScreenshotUrls(requestedOnlyLogs)).toEqual([
+      `${S3}/screenshot-playwright-85c8de7c55.png`,
+      `${S3}/screenshot-playwright-304ad8e401.png`,
+    ]);
+  });
+
+  it("merges both sources in log order", () => {
+    const logs = [
+      `2026-10-13 9:20:22:000 REQUEST [2026-10-13 9:20:22:000] POST /session/${S}/url {"url":"https://example.com"}`,
+      `2026-10-13 9:20:22:114 DEBUG ${S3}/screenshot-24ff228371.jpeg`,
+      `2026-10-13 9:20:22:200 RESPONSE {"value":null}`,
+      `2026-10-13 9:20:23:500 REQUEST [2026-10-13 9:20:23:500] GET /session/${S}/screenshot {}`,
+      `2026-10-13 9:20:23:740 RESPONSE {"value":"${S3}/screenshot-selenium-2d84facd1a.png"}`,
+    ].join("\n");
+
+    expect(parseScreenshotUrls(logs)).toEqual([
+      `${S3}/screenshot-24ff228371.jpeg`,
+      `${S3}/screenshot-selenium-2d84facd1a.png`,
+    ]);
+  });
+
+  it("accepts B2 and direct bucket URLs on DEBUG lines", () => {
+    const b2 = `https://automate.browserstack.com/b2-debug/bucket/s3.us-east-005.backblazeb2.com/${S}/screenshot-2.jpeg`;
+    const direct = `https://browserstack-debug-screenshots-prod-use1.s3.amazonaws.com/${S}/screenshot-3.jpeg`;
+    const logs = [
+      `2026-10-13 9:00:00:000 DEBUG ${b2}`,
+      `2026-10-13 9:00:01:000 DEBUG ${direct}`,
+    ].join("\r\n");
+
+    expect(parseScreenshotUrls(logs)).toEqual([b2, direct]);
+  });
+
+  it("skips intermediate lines between a screenshot request and its response", () => {
+    const logs = [
+      `2026-10-13 9:00:00:000 REQUEST [2026-10-13 9:00:00:000] GET /session/${S}/screenshot {}`,
+      `2026-10-13 9:00:00:050 PERFORMANCE {"lcp":120}`,
+      `2026-10-13 9:00:00:100 RESPONSE {"value":"${S3}/screenshot-selenium-1.png"}`,
+    ].join("\n");
+
+    expect(parseScreenshotUrls(logs)).toEqual([
+      `${S3}/screenshot-selenium-1.png`,
+    ]);
+  });
+
+  it("does not borrow a later request's response value", () => {
+    const logs = [
+      `2026-10-13 9:00:00:000 REQUEST [2026-10-13 9:00:00:000] GET /session/${S}/screenshot {}`,
+      `2026-10-13 9:00:01:000 REQUEST [2026-10-13 9:00:01:000] GET /session/${S}/title`,
+      `2026-10-13 9:00:01:100 RESPONSE {"value":"Example Domain"}`,
+    ].join("\n");
+
+    expect(parseScreenshotUrls(logs)).toEqual([]);
+  });
+
+  it("ignores DEBUG and response URLs echoed inside request payloads", () => {
+    const logs = [
+      `2026-10-13 9:00:00:000 REQUEST [2026-10-13 9:00:00:000] POST /session/${S}/value {"text":"x DEBUG https://evil.browserstack.com.attacker.io/a.png"}`,
+      `2026-10-13 9:00:00:100 RESPONSE {"value":null}`,
+      `2026-10-13 9:00:01:000 REQUEST [2026-10-13 9:00:01:000] POST /session/${S}/execute {"script":"GET /session/x/screenshot"}`,
+      `2026-10-13 9:00:01:100 RESPONSE {"value":"https://automate.browserstack.com/not-a-screenshot"}`,
+      `2026-10-13 9:00:02:000 REQUEST [2026-10-13 9:00:02:000] GET /session/${S}/screenshot {}`,
+      `2026-10-13 9:00:02:100 RESPONSE {"other":{"value":"${S3}/nested.png"}}`,
+    ].join("\n");
+
+    expect(parseScreenshotUrls(logs)).toEqual([]);
+  });
+
+  it("drops URLs outside BrowserStack, S3 and B2 hosts", () => {
+    const logs = [
+      `2026-10-13 9:00:00:000 DEBUG http://169.254.169.254/latest/meta-data/`,
+      `2026-10-13 9:00:01:000 DEBUG https://169.254.169.254/latest/meta-data/`,
+      `2026-10-13 9:00:02:000 DEBUG https://localhost/screenshot-1.jpeg`,
+      `2026-10-13 9:00:03:000 DEBUG https://browserstack.com.evil.io/screenshot-2.jpeg`,
+      `2026-10-13 9:00:04:000 DEBUG https://sts.amazonaws.com/screenshot-3.jpeg`,
+      `2026-10-13 9:00:05:000 DEBUG ${S3}/screenshot-4.jpeg`,
+    ].join("\n");
+
+    expect(parseScreenshotUrls(logs)).toEqual([`${S3}/screenshot-4.jpeg`]);
+  });
+
+  it("returns nothing when the session has no screenshots", () => {
+    const logs = [
+      `2026-10-13 9:21:1:278 REQUEST [2026-10-13 9:21:1:278] POST /session/${S}/url {"url":"https://example.com"}`,
+      `2026-10-13 9:21:3:848 RESPONSE {"value":null}`,
+    ].join("\n");
+
+    expect(parseScreenshotUrls(logs)).toEqual([]);
+  });
+});
+
+describe("fetchAutomationScreenshots", () => {
+  beforeEach(() => vi.clearAllMocks());
+
+  const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]);
+  const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a]);
+
+  it("returns the last five screenshots with detected mime types", async () => {
+    const urls = Array.from({ length: 6 }, (_, i) => `${S3}/screenshot-${i}.jpeg`);
+    const logs = urls
+      .map((u, i) => `2026-10-13 9:00:0${i}:000 DEBUG ${u}`)
+      .concat(requestedOnlyLogs.split("\n").slice(0, 2))
+      .join("\n");
+
+    (apiClient.get as Mock).mockImplementation(async ({ url }) => {
+      if (url.endsWith("/logs")) {
+        return { ok: true, status: 200, data: logs };
+      }
+      return { ok: true, status: 200, data: url.endsWith(".png") ? png : jpeg };
+    });
+
+    const result = await fetchAutomationScreenshots(
+      S,
+      SessionType.Automate,
+      mockConfig,
+    );
+
+    expect(result.map((s) => s.url)).toEqual([
+      ...urls.slice(2),
+      `${S3}/screenshot-playwright-85c8de7c55.png`,
+    ]);
+    expect(result.map((s) => s.mimeType)).toEqual([
+      "image/jpeg",
+      "image/jpeg",
+      "image/jpeg",
+      "image/jpeg",
+      "image/png",
+    ]);
+  });
+});