fix: size AI summary chunks in tokens so local OpenAI-compatible models get the instructions

apps/web/__tests__/unit/generate-ai-fallback.test.ts

@@ -43,6 +43,8 @@ vi.mock("server-only", () => ({}));
 import { AiUnavailableError } from "@/lib/ai/run";
 import {
 	callAiApi,
+	chunkTranscriptForProviderChain,
+	estimateTokens,
 	parseAiResponse,
 	parseChunkAnalysis,
 	parseFinalSummary,
@@ -186,3 +188,100 @@ describe("map-reduce output parsers", () => {
 		expect(() => parseFinalSummary('{"title":"","summary":""}')).toThrow();
 	});
 });
+
+describe("transcript chunk budget", () => {
+	// A typical eight-word, 40-character VTT cue.
+	const CUE = "Then we checked the numbers for the week";
+	const cues = (count: number, text = CUE) =>
+		Array.from({ length: count }, (_, i) => ({ start: i * 3, text }));
+
+	describe("hosted chains keep the 24,000-character chunks", () => {
+		beforeEach(() => {
+			getAiProviderChainMock.mockReturnValue([
+				makeSelection("groq"),
+				makeSelection("openai"),
+			]);
+		});
+
+		it("fits 585 cues in one chunk and splits at 586", () => {
+			expect(chunkTranscriptForProviderChain(cues(585))).toHaveLength(1);
+
+			const chunks = chunkTranscriptForProviderChain(cues(586));
+
+			expect(chunks.map((chunk) => chunk.text.length)).toEqual([23984, 40]);
+			expect(chunks.map((chunk) => [chunk.startTime, chunk.endTime])).toEqual([
+				[0, 1752],
+				[1755, 1755],
+			]);
+		});
+
+		it("allows exactly 24,000 joined characters and splits at 24,001", () => {
+			const withLastCue = (length: number) =>
+				chunkTranscriptForProviderChain([
+					...cues(585),
+					{ start: 1755, text: "x".repeat(length) },
+				]).map((chunk) => chunk.text.length);
+
+			expect(withLastCue(15)).toEqual([24000]);
+			expect(withLastCue(16)).toEqual([23984, 16]);
+		});
+
+		it("keeps an oversized first segment whole", () => {
+			const chunks = chunkTranscriptForProviderChain([
+				{ start: 0, text: "a".repeat(30000) },
+				{ start: 5, text: "b".repeat(10) },
+			]);
+
+			expect(chunks.map((chunk) => chunk.text.length)).toEqual([30000, 10]);
+		});
+
+		it("counts non-ASCII text by string length, not estimated tokens", () => {
+			const japanese = chunkTranscriptForProviderChain(
+				cues(1600, "会議の要約です会議の要約です"),
+			);
+			// Each emoji is two UTF-16 units.
+			const emoji = chunkTranscriptForProviderChain([
+				{ start: 0, text: "😀".repeat(6001) },
+				{ start: 5, text: "b".repeat(11999) },
+			]);
+
+			expect(japanese.map((chunk) => chunk.text.length)).toEqual([23999]);
+			expect(emoji.map((chunk) => chunk.text.length)).toEqual([12002, 11999]);
+		});
+	});
+
+	it("uses the token budget when openai-compatible is only a fallback", () => {
+		getAiProviderChainMock.mockReturnValue([
+			makeSelection("groq"),
+			makeSelection("openai-compatible"),
+		]);
+
+		expect(chunkTranscriptForProviderChain(cues(585))).toHaveLength(6);
+	});
+
+	it("estimates one token per non-ASCII character", () => {
+		expect(estimateTokens("abcdefgh")).toBe(2);
+		expect(estimateTokens("会議の要約")).toBe(5);
+	});
+
+	it("splits a transcript so no chunk exceeds the token budget", () => {
+		getAiProviderChainMock.mockReturnValue([
+			makeSelection("openai-compatible"),
+		]);
+		const text = "We reviewed the search console numbers for the client today.";
+		const segments = Array.from({ length: 300 }, (_, i) => ({
+			start: i * 4,
+			text,
+		}));
+
+		const chunks = chunkTranscriptForProviderChain(segments);
+
+		expect(getAiProviderChainMock).toHaveBeenLastCalledWith("generation");
+		// 15 tokens of text plus 6 for the timestamp: 85 segments fit in 1,800.
+		expect(
+			chunks.map((chunk) => (chunk.text.length + 1) / (text.length + 1)),
+		).toEqual([85, 85, 85, 45]);
+		expect(chunks[0]?.startTime).toBe(0);
+		expect(chunks[chunks.length - 1]?.endTime).toBe(299 * 4);
+	});
+});

apps/web/content/docs/self-hosting.mdx

@@ -151,6 +151,18 @@ AI_MODEL=llama3.3
 AI_API_KEY=your-key          # If the endpoint requires one
 ```
 
+When `openai-compatible` is in the chain, Cap sends transcripts in smaller
+sections so each section fits a 4k-token context window, which is Ollama's
+default on machines with less than 24 GB of VRAM. Ollama's OpenAI-compatible
+endpoint ignores per-request context settings and silently cuts longer
+prompts from the start. The step that combines the sections can still pass
+4k tokens on very long recordings, so for those, raise the context on the
+Ollama server:
+
+```bash
+OLLAMA_CONTEXT_LENGTH=16384 ollama serve
+```
+
 Each provider has sensible default models. You can override them per role for
 the provider selected by `AI_PROVIDER`:
 

apps/web/workflows/generate-ai.ts

@@ -13,7 +13,7 @@ import { generateText } from "ai";
 import { and, eq, sql } from "drizzle-orm";
 import { Effect, Option } from "effect";
 import { FatalError } from "workflow";
-import { isAiConfigured } from "@/lib/ai/provider";
+import { getAiProviderChain, isAiConfigured } from "@/lib/ai/provider";
 import { AiUnavailableError, runWithAiProviders } from "@/lib/ai/run";
 import { setGeneratedAiContent } from "@/lib/ai-content-metadata";
 import { enqueueVideoStorageNameSync } from "@/lib/sync-video-storage-names";
@@ -58,6 +58,14 @@ const getAffectedRows = (result: unknown) => {
 };
 
 const MAX_CHARS_PER_CHUNK = 24000;
+// Ollama serves a 4k-token context on machines under 24 GB of VRAM and its
+// /v1 endpoint ignores num_ctx, so a longer prompt is cut from the front
+// (losing the instructions) with no error. Leave room for the ~800-token
+// instructions and the JSON reply inside that 4k window.
+const OPENAI_COMPATIBLE_TRANSCRIPT_CHUNK_TOKENS = 1800;
+// The single-chunk prompt prefixes every segment with "[m:ss] ", which
+// tokenizes to about six tokens; word-level transcripts have thousands.
+const SEGMENT_TIMESTAMP_TOKENS = 6;
 const LEGACY_AI_TITLE_FALLBACK = "Generated Title";
 const LEGACY_AI_SUMMARY_FALLBACK =
 	"The AI was unable to generate a proper summary for this content.";
@@ -251,7 +259,7 @@ async function generateWithAi(
 ): Promise<AiResult> {
 	"use step";
 
-	const chunks = chunkTranscriptWithTimestamps(transcript.segments);
+	const chunks = chunkTranscriptForProviderChain(transcript.segments);
 
 	const videoDuration = getVideoDuration(transcript.segments);
 	const languageInstruction = getAiLanguageInstruction(language);
@@ -462,6 +470,21 @@ function parseVttWithTimestamps(vttContent: string): VttSegment[] {
 	return segments;
 }
 
+// Every provider gets the same chunks, so a local model anywhere in the
+// chain means sizing for its 4k context.
+export function chunkTranscriptForProviderChain(
+	segments: VttSegment[],
+): { text: string; startTime: number; endTime: number }[] {
+	const chain = getAiProviderChain("generation");
+	if (chain.some((selection) => selection.provider === "openai-compatible")) {
+		return chunkTranscriptByTokens(
+			segments,
+			OPENAI_COMPATIBLE_TRANSCRIPT_CHUNK_TOKENS,
+		);
+	}
+	return chunkTranscriptWithTimestamps(segments);
+}
+
 function chunkTranscriptWithTimestamps(
 	segments: VttSegment[],
 ): { text: string; startTime: number; endTime: number }[] {
@@ -497,6 +520,56 @@ function chunkTranscriptWithTimestamps(
 	return chunks;
 }
 
+// Conservative without a tokenizer: ~4 ASCII characters per token, and one
+// token per non-ASCII character so CJK and other scripts don't overflow.
+export function estimateTokens(text: string): number {
+	let ascii = 0;
+	let other = 0;
+	for (const character of text) {
+		if (character.charCodeAt(0) < 128) ascii += 1;
+		else other += 1;
+	}
+	return Math.ceil(ascii / 4) + other;
+}
+
+function chunkTranscriptByTokens(
+	segments: VttSegment[],
+	maxTokensPerChunk: number,
+): { text: string; startTime: number; endTime: number }[] {
+	const chunks: { text: string; startTime: number; endTime: number }[] = [];
+	let currentChunk: VttSegment[] = [];
+	let currentTokens = 0;
+
+	for (const segment of segments) {
+		const segmentTokens =
+			estimateTokens(segment.text) + SEGMENT_TIMESTAMP_TOKENS;
+		if (
+			currentTokens + segmentTokens > maxTokensPerChunk &&
+			currentChunk.length > 0
+		) {
+			chunks.push({
+				text: currentChunk.map((s) => s.text).join(" "),
+				startTime: currentChunk[0]?.start ?? 0,
+				endTime: currentChunk[currentChunk.length - 1]?.start ?? 0,
+			});
+			currentChunk = [];
+			currentTokens = 0;
+		}
+		currentChunk.push(segment);
+		currentTokens += segmentTokens;
+	}
+
+	if (currentChunk.length > 0) {
+		chunks.push({
+			text: currentChunk.map((s) => s.text).join(" "),
+			startTime: currentChunk[0]?.start ?? 0,
+			endTime: currentChunk[currentChunk.length - 1]?.start ?? 0,
+		});
+	}
+
+	return chunks;
+}
+
 class InvalidAiOutputError extends Error {
 	constructor(message: string, options?: { cause?: unknown }) {
 		super(message, options);