feat(web): OpenAI-compatible STT endpoint for self-hosted transcription (whisper.cpp, faster-whisper)

apps/web/__tests__/fixtures/whisper-cpp-verbose-response.ts

@@ -0,0 +1,189 @@
+export const whisperCppVerboseResponse = {
+	task: "transcribe",
+	language: "english",
+	duration: 6.91,
+	text: " Hello, this is a quick test of the local transcription\n server.\n Um, it should return Word timestamps.\n",
+	segments: [
+		{
+			id: 0,
+			text: " Hello, this is a quick test of the local transcription",
+			start: 0.1,
+			end: 3.1,
+			words: [
+				{
+					word: " Hello",
+					start: 0.03,
+					end: 0.37,
+					t_dtw: -1,
+					probability: 0.72,
+				},
+				{
+					word: ",",
+					start: 0.37,
+					end: 0.49,
+					t_dtw: -1,
+					probability: 0.83,
+				},
+				{
+					word: " this",
+					start: 0.65,
+					end: 0.82,
+					t_dtw: -1,
+					probability: 0.98,
+				},
+				{
+					word: " is",
+					start: 0.82,
+					end: 0.97,
+					t_dtw: -1,
+					probability: 1.0,
+				},
+				{
+					word: " a",
+					start: 0.97,
+					end: 1.02,
+					t_dtw: -1,
+					probability: 1.0,
+				},
+				{
+					word: " quick",
+					start: 1.09,
+					end: 1.27,
+					t_dtw: -1,
+					probability: 1.0,
+				},
+				{
+					word: " test",
+					start: 1.42,
+					end: 1.7,
+					t_dtw: -1,
+					probability: 1.0,
+				},
+				{
+					word: " of",
+					start: 1.75,
+					end: 1.83,
+					t_dtw: -1,
+					probability: 0.98,
+				},
+				{
+					word: " the",
+					start: 1.9,
+					end: 2.0,
+					t_dtw: -1,
+					probability: 1.0,
+				},
+				{
+					word: " local",
+					start: 2.0,
+					end: 2.27,
+					t_dtw: -1,
+					probability: 0.99,
+				},
+				{
+					word: " transcription",
+					start: 2.27,
+					end: 3.0,
+					t_dtw: -1,
+					probability: 0.99,
+				},
+			],
+		},
+		{
+			id: 1,
+			text: " server.",
+			start: 3.1,
+			end: 3.42,
+			words: [
+				{
+					word: " server",
+					start: 3.0,
+					end: 3.19,
+					t_dtw: -1,
+					probability: 0.99,
+				},
+				{
+					word: ".",
+					start: 3.22,
+					end: 3.32,
+					t_dtw: -1,
+					probability: 0.93,
+				},
+			],
+		},
+		{
+			id: 2,
+			text: " Um, it should return Word timestamps.",
+			start: 4.2,
+			end: 6.8,
+			words: [
+				{
+					word: " Um",
+					start: 3.72,
+					end: 3.72,
+					t_dtw: -1,
+					probability: 0.89,
+				},
+				{
+					word: ",",
+					start: 3.72,
+					end: 3.72,
+					t_dtw: -1,
+					probability: 1.0,
+				},
+				{
+					word: " it",
+					start: 3.75,
+					end: 3.82,
+					t_dtw: -1,
+					probability: 1.0,
+				},
+				{
+					word: " should",
+					start: 3.82,
+					end: 4.2,
+					t_dtw: -1,
+					probability: 1.0,
+				},
+				{
+					word: " return",
+					start: 4.37,
+					end: 4.82,
+					t_dtw: -1,
+					probability: 1.0,
+				},
+				{
+					word: " Word",
+					start: 4.88,
+					end: 5.18,
+					t_dtw: -1,
+					probability: 0.64,
+				},
+				{
+					word: " timest",
+					start: 5.18,
+					end: 5.69,
+					t_dtw: -1,
+					probability: 0.89,
+				},
+				{
+					word: "amps",
+					start: 5.69,
+					end: 6.0,
+					t_dtw: -1,
+					probability: 1.0,
+				},
+				{
+					word: ".",
+					start: 6.06,
+					end: 6.3,
+					t_dtw: -1,
+					probability: 0.99,
+				},
+			],
+		},
+	],
+	language_probabilities: {
+		en: 0.9997,
+	},
+};

apps/web/__tests__/integration/transcribe-early-segments.test.ts

@@ -5,6 +5,7 @@ import { join } from "node:path";
 import { Option } from "effect";
 import { beforeEach, describe, expect, it, vi } from "vitest";
 import { assemblyAIEditResponse } from "../fixtures/assemblyai-edit-response";
+import { whisperCppVerboseResponse } from "../fixtures/whisper-cpp-verbose-response";
 
 const mocks = vi.hoisted(() => ({
 	transcribe: vi.fn(),
@@ -50,12 +51,14 @@ const videoRow = vi.hoisted(() => ({
 const state = vi.hoisted(() => ({
 	editRows: [] as unknown[],
 	fakeAudioPath: "",
+	sttBaseUrl: undefined as string | undefined,
 }));
 
 vi.mock("@cap/env", () => ({
 	serverEnv: () => ({
-		ASSEMBLY_API_KEY: "test-assembly-api-key",
+		ASSEMBLY_API_KEY: state.sttBaseUrl ? undefined : "test-assembly-api-key",
 		NEXTAUTH_SECRET: "test-secret-with-enough-entropy",
+		STT_BASE_URL: state.sttBaseUrl,
 	}),
 }));
 
@@ -192,6 +195,7 @@ describe("transcribeVideoWorkflow earlyFromSegments", () => {
 	beforeEach(async () => {
 		mocks.updates.length = 0;
 		state.editRows = [];
+		state.sttBaseUrl = undefined;
 		videoRow.duration = null;
 		videoRow.source = { type: "desktopSegments" };
 		mocks.transcribe.mockReset();
@@ -290,6 +294,44 @@ describe("transcribeVideoWorkflow earlyFromSegments", () => {
 		);
 	});
 
+	it("sends early segment audio to the STT endpoint as m4a", async () => {
+		state.sttBaseUrl = "http://whisper:9000/v1";
+		const fetchMock = vi.fn(
+			async (input: string | URL | Request, _init?: RequestInit) => {
+				const url = String(input);
+				if (url.startsWith("http://whisper:9000")) {
+					return new Response(JSON.stringify(whisperCppVerboseResponse));
+				}
+				const key = url.replace("https://storage.test/", "");
+				const contentType = mocks.putObject.mock.calls.find(
+					(call) => call[0] === key,
+				)?.[2]?.contentType;
+				return new Response(new TextEncoder().encode("seg"), {
+					headers: contentType ? { "content-type": contentType } : {},
+				});
+			},
+		);
+		vi.stubGlobal("fetch", fetchMock);
+
+		const { transcribeVideoWorkflow } = await import("@/workflows/transcribe");
+		const result = await transcribeVideoWorkflow({
+			videoId: "video-123",
+			userId: "user-456",
+			aiGenerationEnabled: false,
+			earlyFromSegments: true,
+		});
+
+		expect(result.success).toBe(true);
+		expect(mocks.transcribe).not.toHaveBeenCalled();
+		const sttCall = fetchMock.mock.calls.find(([input]) =>
+			String(input).startsWith("http://whisper:9000"),
+		);
+		const file = (sttCall?.[1]?.body as FormData).get("file") as File;
+		expect(file.name).toBe("audio.m4a");
+		expect(file.type).toBe("audio/mp4");
+		expect(mocks.updates).toContainEqual({ transcriptionStatus: "COMPLETE" });
+	});
+
 	it("defers back to the post-mux queue when the manifest is missing", async () => {
 		mocks.getObject.mockImplementation(() => pipeValue(Option.none()));