Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions apps/web/src/components/chat/ChatComposer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -1850,19 +1850,22 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
};
const readComposerSnapshotStable = useStableCallback(readComposerSnapshot);

const dictationDisabled =
!canOperateThread ||
environmentUnavailable !== null ||
// The draft outlives its environment reconnecting, so that only blocks starting a recording.
const dictationStartDisabled = !canOperateThread || environmentUnavailable !== null;
// Another surface has replaced the editor, so a transcript would have nowhere to land.
const dictationDisplaced =
projectSelectionRequired ||
isComposerApprovalState ||
activePendingIsResponding ||
activePendingProgress?.activeQuestion?.allowCustomAnswer === false;
const dictationDisabled = dictationStartDisabled || dictationDisplaced;
const speechInput = useEnvironmentSpeechInput({
environmentId,
projectId: activeProjectId,
projectName,
ownerKey: questionAttachmentTarget ?? composerDraftTargetKey,
disabled: dictationDisabled,
disabled: dictationDisplaced,
startDisabled: dictationStartDisabled,
label: activeThread?.title ?? null,
presented: !hiddenThread,
// A question's answer lives only in this composer, so only the thread draft outlives it.
Expand Down
3 changes: 1 addition & 2 deletions apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,7 @@ import { ComposerSpeechRecordingPill } from "./ComposerSpeechButton";
*/
export function GlobalVoiceInputIndicator({ hidden = false }: { hidden?: boolean }) {
const gt = useGT();
const { prepared, grantId, cloudProvider, cloudModel } = useVoiceTranscriptionDestination();
const session = useRetainedVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel });
const session = useRetainedVoiceInputSession(useVoiceTranscriptionDestination());
const { state, target, recordingStartedAt } = useVoiceInputSessionSnapshot(session);
const presented = useVoiceOwnerPresented(target?.ownerKey ?? null);
const { level } = useVoiceInputSessionActivity(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ it.each([{ cpu: null, accelerated: 2.1 }, { cpu: 5, accelerated: null }, { accel
label: prepared.label,
grantId: "test",
prepared,
usable: true,
}),
async () => {
const previous = getClientSettings();
Expand Down Expand Up @@ -303,7 +304,13 @@ it("tests the selected model after download and preserves its transcript across
const transcript = "Keep this transcript.";
using _scope = vi.mocked(useEnvironmentScope).mockReturnValue(true);
await vi.mocked(useVoiceTranscriptionDestination).withImplementation(
() => ({ environmentId: target.environmentId, label: target.label, grantId: "test", prepared }),
() => ({
environmentId: target.environmentId,
label: target.label,
grantId: "test",
prepared,
usable: true,
}),
async () => {
await vi.mocked(useEnvironmentSpeechInput).withImplementation(
(input) => ({
Expand Down Expand Up @@ -486,6 +493,7 @@ it.each([
label: target.label,
grantId: "test",
prepared,
usable: true,
}));
await using screen = await render(
<SettingsScopeProvider search={{}} onChange={() => {}}>
Expand Down Expand Up @@ -562,7 +570,13 @@ it("recovers from a failed download progress poll without notifying, but reports
});
using notify = vi.spyOn(toastManager, "add").mockReturnValue("test");
await vi.mocked(useVoiceTranscriptionDestination).withImplementation(
() => ({ environmentId: target.environmentId, label: target.label, grantId: "test", prepared }),
() => ({
environmentId: target.environmentId,
label: target.label,
grantId: "test",
prepared,
usable: true,
}),
async () => {
const panel = (key: string) => (
<SettingsScopeProvider search={{}} onChange={() => {}}>
Expand Down
7 changes: 4 additions & 3 deletions apps/web/src/speech/browserPcmCapture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,9 @@ export function createBrowserPcmCapture(
input.onLevel(data);
return;
}
if (feed) feed(data);
else chunks.push(data);
// Kept while streaming too, so a stream that drops can still upload the recording.
chunks.push(data);
feed?.(data);
};
node.onprocessorerror = () => {
stopped?.reject(new Error(m(msg("Microphone processing failed."))));
Expand Down Expand Up @@ -156,7 +157,7 @@ export function createBrowserPcmCapture(
worklet.port.postMessage("stop");
try {
await pending;
if (!feed && chunks.length) {
if (chunks.length) {
const blob = new Blob(chunks, { type: "application/octet-stream" });
recordingUri = URL.createObjectURL(blob);
}
Expand Down
99 changes: 80 additions & 19 deletions apps/web/src/speech/browserVoiceInput.browser.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,29 @@ const status = {
benchmarkTier: "cpu",
} satisfies EnvironmentSpeechStatus;

function controllerFor(
platform: ReturnType<typeof createBrowserVoiceInputPlatform>,
commitDraft: (text: string) => void,
) {
return new VoiceInputController({
recorder: platform.recorder,
retryFailedRecording: true,
getTranscriber: () => platform.transcriber,
requestPermission: async () => ({ granted: true, canAskAgain: true }),
configureRecording: async () => {},
releaseRecording: async () => platform.cancelRecording(),
deleteRecording: platform.deleteRecording,
readDraft: () => ({
ownerKey: "draft",
text: "",
selection: { start: 0, end: 0 },
revision: 0,
}),
commitDraft,
onStateChange() {},
});
}

it("retains all PCM until stop, retries batch audio, and captures preferences before status resolves", async () => {
using microphone = capture();
let streaming = false;
Expand All @@ -125,7 +148,7 @@ it("retains all PCM until stop, retries batch audio, and captures preferences be
});
const onError = vi.fn();
const platform = createBrowserVoiceInputPlatform({
prepared,
getPrepared: () => prepared,
hasAudioConsent: () => true,
getStreamingEnabled: () => streaming,
getTranscriptionOptions: () => options,
Expand All @@ -136,23 +159,7 @@ it("retains all PCM until stop, retries batch audio, and captures preferences be
onError,
});
const commitDraft = vi.fn();
const controller = new VoiceInputController({
recorder: platform.recorder,
retryFailedRecording: true,
getTranscriber: () => platform.transcriber,
requestPermission: async () => ({ granted: true, canAskAgain: true }),
configureRecording: async () => {},
releaseRecording: async () => platform.cancelRecording(),
deleteRecording: platform.deleteRecording,
readDraft: () => ({
ownerKey: "draft",
text: "",
selection: { start: 0, end: 0 },
revision: 0,
}),
commitDraft,
onStateChange() {},
});
const controller = controllerFor(platform, commitDraft);
using _controller = { [Symbol.dispose]: () => controller.dispose() };
const start = controller.start();
await vi.waitFor(() => expect(_network).toHaveBeenCalled());
Expand Down Expand Up @@ -191,6 +198,60 @@ it("retains all PCM until stop, retries batch audio, and captures preferences be
expect(platform.recorder.uri).toBeNull();
});

it("keeps recording when the live stream drops and uploads the buffered audio on stop", async () => {
using microphone = capture();
const sockets: EventTarget[] = [];
using _socket = stubGlobal(
"WebSocket",
class extends EventTarget {
constructor() {
super();
sockets.push(this);
}
send() {}
close() {}
},
);
const uploads: number[][] = [];
const fetch = globalThis.fetch;
using _network = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => {
const request = new Request(input, init);
if (request.url.endsWith("/api/voice/status")) return Response.json(status);
if (request.url.includes("/api/auth/websocket-ticket/"))
return Response.json({ ticket: "stream-ticket", expiresAt: "2030-01-01T00:00:00.000Z" });
if (request.url.endsWith("/api/voice/transcribe")) {
const { pcm } = decodeSpeechPcmRequest(new Uint8Array(await request.arrayBuffer()));
uploads.push([...new Float32Array(pcm.slice().buffer)]);
return Response.json({ text: "bonjour" });
}
return fetch(input, init);
});
const onError = vi.fn();
const platform = createBrowserVoiceInputPlatform({
getPrepared: () => prepared,
hasAudioConsent: () => true,
getStreamingEnabled: () => true,
getTranscriptionOptions: () => DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
getMicrophoneId: () => "",
onLevel() {},
onDurationLimit() {},
onText() {},
onError,
});
const commitDraft = vi.fn();
const controller = controllerFor(platform, commitDraft);
using _controller = { [Symbol.dispose]: () => controller.dispose() };
await controller.start();
microphone.feed(new Float32Array([0.1]));
sockets[0]!.dispatchEvent(new Event("close"));
microphone.feed(new Float32Array([0.2]));
expect(controller.currentState.phase).toBe("recording");
expect(onError).not.toHaveBeenCalled();
await controller.stop();
expect(uploads).toEqual([[Math.fround(0.1), Math.fround(0.2)]]);
expect(commitDraft).toHaveBeenCalledWith("bonjour", { start: 7, end: 7 });
});

it.each([
{ native: true, enabled: true, buffered: true, live: true },
{ native: true, enabled: false, buffered: true, live: false },
Expand Down Expand Up @@ -226,7 +287,7 @@ it.each([
const abort = new AbortController();
using _abort = { [Symbol.dispose]: () => abort.abort() };
const platform = createBrowserVoiceInputPlatform({
prepared,
getPrepared: () => prepared,
hasAudioConsent: () => true,
getStreamingEnabled: () => enabled,
getTranscriptionOptions: () => ({
Expand Down
62 changes: 36 additions & 26 deletions apps/web/src/speech/browserVoiceInput.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ import { m } from "../i18n/messages";
const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError);

export function createBrowserVoiceInputPlatform(input: {
readonly prepared: PreparedConnection;
/** The destination's connection right now; null while it reconnects. */
readonly getPrepared: () => PreparedConnection | null;
readonly cloudProvider?: CloudSpeechProvider | undefined;
/** The exact OpenRouter model the grant names; required with that provider. */
readonly cloudModel?: CloudSpeechModelId | undefined;
Expand All @@ -37,7 +38,16 @@ export function createBrowserVoiceInputPlatform(input: {
readonly onText: (text: SpeechStreamText) => void;
readonly onError: (message: string, code?: VoiceInputErrorCode) => void;
}) {
const requirePrepared = () => {
const prepared = input.getPrepared();
if (prepared) return prepared;
throw new VoiceTranscriptionError({
code: "unavailable",
message: m(msg("Voice transcription failed. Check the destination in Settings → Voice.")),
});
};
return createBrowserPcmCapture(input, async (options) => {
const prepared = requirePrepared();
const recordingOptions = input.getTranscriptionOptions();
const streamingEnabled = input.getStreamingEnabled();
const transcribeRecording = async (
Expand All @@ -47,22 +57,19 @@ export function createBrowserVoiceInputPlatform(input: {
try {
const response = await fetch(uri, { signal: transcriptionSignal });
const pcm = new Uint8Array(await response.arrayBuffer());
// A recording can outlive the connection it started on, so it uploads on the current one.
const current = requirePrepared();
const result = await runEnvironmentSpeech(
input.cloudProvider
? transcribeCloudPcm(
input.prepared,
current,
input.cloudProvider,
pcm,
recordingOptions,
input.hasAudioConsent,
input.cloudModel,
)
: transcribeEnvironmentPcm(
input.prepared,
pcm,
recordingOptions,
input.hasAudioConsent,
),
: transcribeEnvironmentPcm(current, pcm, recordingOptions, input.hasAudioConsent),
{ signal: transcriptionSignal },
);
throwIfVoiceTranscriptionAborted(transcriptionSignal);
Expand All @@ -83,17 +90,14 @@ export function createBrowserVoiceInputPlatform(input: {
code: "unavailable",
message: m(msg("Choose an OpenRouter model in Settings → Voice.")),
});
await runEnvironmentSpeech(
requireCloudSpeechKey(input.prepared, input.cloudProvider),
options,
);
await runEnvironmentSpeech(requireCloudSpeechKey(prepared, input.cloudProvider), options);
throwIfVoiceTranscriptionAborted(options.signal);
return {
locale: effectiveSpeechLanguage(cloudSpeechLanguages, recordingOptions.speechLanguage),
transcribe: transcribeRecording,
};
}
const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options);
const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(prepared), options);
throwIfVoiceTranscriptionAborted(options.signal);
if (!status.supported)
throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
Expand All @@ -114,18 +118,24 @@ export function createBrowserVoiceInputPlatform(input: {
if (status.supportsStreaming ? !nativeStreaming : !status.supportsBufferedStreaming) {
return { locale, transcribe: transcribeRecording };
}
const url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(input.prepared), options);
return {
locale,
transcribe: transcribeRecording,
streaming: openSpeechStream({
url,
canSend: input.hasAudioConsent,
options: recordingOptions,
signal: options.signal,
onText: input.onText,
onError: (error) => input.onError(error.message),
}),
};
const url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(prepared), options);
const transcription: {
readonly locale: string;
readonly transcribe: typeof transcribeRecording;
streaming?: ReturnType<typeof openSpeechStream>;
} = { locale, transcribe: transcribeRecording };
transcription.streaming = openSpeechStream({
url,
canSend: input.hasAudioConsent,
options: recordingOptions,
signal: options.signal,
onText: input.onText,
onError: (error) => {
// A lost stream keeps recording; stopping then uploads the buffered audio instead.
if (input.hasAudioConsent()) delete transcription.streaming;
else input.onError(error.message);
},
});
return transcription;
});
}
Loading
Loading