diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 8a4eedd63..a3ba8ccfc 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -880,6 +880,7 @@ interface Window { success: boolean; microphoneEnabled: boolean; microphoneDeviceId?: string; + noiseSuppressionMode?: string; systemAudioEnabled: boolean; }>; getRecordingAudioLabConfig: () => Promise<{ @@ -889,6 +890,7 @@ interface Window { setRecordingPreferences: (prefs: { microphoneEnabled?: boolean; microphoneDeviceId?: string; + noiseSuppressionMode?: string; systemAudioEnabled?: boolean; }) => Promise<{ success: boolean; error?: string }>; /** Countdown timer before recording */ diff --git a/electron/ipc/register/settings.ts b/electron/ipc/register/settings.ts index e84f63171..af7a34eb0 100644 --- a/electron/ipc/register/settings.ts +++ b/electron/ipc/register/settings.ts @@ -153,6 +153,10 @@ export function registerSettingsHandlers() { typeof parsed.microphoneDeviceId === "string" ? parsed.microphoneDeviceId : undefined, + noiseSuppressionMode: + typeof parsed.noiseSuppressionMode === "string" + ? parsed.noiseSuppressionMode + : "rnnoise", systemAudioEnabled: parsed.systemAudioEnabled === true, }; } catch { @@ -160,6 +164,7 @@ export function registerSettingsHandlers() { success: true, microphoneEnabled: false, microphoneDeviceId: undefined, + noiseSuppressionMode: "rnnoise", systemAudioEnabled: false, }; } @@ -176,6 +181,7 @@ export function registerSettingsHandlers() { prefs: { microphoneEnabled?: boolean; microphoneDeviceId?: string; + noiseSuppressionMode?: string; systemAudioEnabled?: boolean; }, ) => { diff --git a/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor b/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor index e0e478f11..34a4da9e3 100755 Binary files a/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor and b/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor differ diff --git a/electron/native/bin/darwin-arm64/recordly-screencapturekit-helper b/electron/native/bin/darwin-arm64/recordly-screencapturekit-helper index 259f18848..e8a29cdf7 100755 Binary files a/electron/native/bin/darwin-arm64/recordly-screencapturekit-helper and b/electron/native/bin/darwin-arm64/recordly-screencapturekit-helper differ diff --git a/electron/native/bin/darwin-arm64/recordly-system-cursors b/electron/native/bin/darwin-arm64/recordly-system-cursors index f4b41ab66..034c201f1 100755 Binary files a/electron/native/bin/darwin-arm64/recordly-system-cursors and b/electron/native/bin/darwin-arm64/recordly-system-cursors differ diff --git a/electron/native/bin/darwin-arm64/recordly-window-list b/electron/native/bin/darwin-arm64/recordly-window-list index 76a7dab4a..e4170eadf 100755 Binary files a/electron/native/bin/darwin-arm64/recordly-window-list and b/electron/native/bin/darwin-arm64/recordly-window-list differ diff --git a/electron/native/bin/darwin-x64/recordly-native-cursor-monitor b/electron/native/bin/darwin-x64/recordly-native-cursor-monitor index d577b1a6f..3c0cfa872 100755 Binary files a/electron/native/bin/darwin-x64/recordly-native-cursor-monitor and b/electron/native/bin/darwin-x64/recordly-native-cursor-monitor differ diff --git a/electron/native/bin/darwin-x64/recordly-screencapturekit-helper b/electron/native/bin/darwin-x64/recordly-screencapturekit-helper index 09e0ed5d6..a42997665 100755 Binary files a/electron/native/bin/darwin-x64/recordly-screencapturekit-helper and b/electron/native/bin/darwin-x64/recordly-screencapturekit-helper differ diff --git a/electron/native/bin/darwin-x64/recordly-system-cursors b/electron/native/bin/darwin-x64/recordly-system-cursors index 545613624..0dc0ff974 100755 Binary files a/electron/native/bin/darwin-x64/recordly-system-cursors and b/electron/native/bin/darwin-x64/recordly-system-cursors differ diff --git a/electron/native/bin/darwin-x64/recordly-window-list b/electron/native/bin/darwin-x64/recordly-window-list index e165257ae..affc5b010 100755 Binary files a/electron/native/bin/darwin-x64/recordly-window-list and b/electron/native/bin/darwin-x64/recordly-window-list differ diff --git a/electron/preload.ts b/electron/preload.ts index e55d42cbd..f12d04835 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -971,6 +971,7 @@ contextBridge.exposeInMainWorld("electronAPI", { setRecordingPreferences: (prefs: { microphoneEnabled?: boolean; microphoneDeviceId?: string; + noiseSuppressionMode?: string; systemAudioEnabled?: boolean; }) => ipcRenderer.invoke("set-recording-preferences", prefs), getCountdownDelay: () => ipcRenderer.invoke("get-countdown-delay"), diff --git a/package-lock.json b/package-lock.json index 71432b2b1..b41ab200b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,8 @@ "hasInstallScript": true, "dependencies": { "@phosphor-icons/react": "^2.1.10", + "@sapphi-red/speex-preprocess-wasm": "^0.4.0", + "@shiguredo/rnnoise-wasm": "^2025.1.5", "capturekit": "^1.0.13", "electron-updater": "^6.8.3", "ffmpeg-static": "^5.3.0", @@ -3477,6 +3479,22 @@ "win32" ] }, + "node_modules/@sapphi-red/speex-preprocess-wasm": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@sapphi-red/speex-preprocess-wasm/-/speex-preprocess-wasm-0.4.0.tgz", + "integrity": "sha512-VAybOqfE1m/3q+3BzpDiHuP/F5O1q0r6BosAkb2FYrBbMTiP1kWs/grELZqcIeSYa1tF5uVQzHi0hQtZUYRJVQ==", + "license": "MIT" + }, + "node_modules/@shiguredo/rnnoise-wasm": { + "version": "2025.1.5", + "resolved": "https://registry.npmjs.org/@shiguredo/rnnoise-wasm/-/rnnoise-wasm-2025.1.5.tgz", + "integrity": "sha512-9YJxzzHftlW936E5z9aEKK2CrPfUgc3HgZ2keyQ5e2Pb5bx+6tykYQ2Gyudvxd7Sxmlnar7SemDMFfx0OuVuGA==", + "license": "Apache-2.0", + "engines": { + "node": ">=20", + "pnpm": ">=10" + } + }, "node_modules/@sindresorhus/is": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", diff --git a/package.json b/package.json index 6fd53cd35..6691b3a54 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,8 @@ }, "dependencies": { "@phosphor-icons/react": "^2.1.10", + "@sapphi-red/speex-preprocess-wasm": "^0.4.0", + "@shiguredo/rnnoise-wasm": "^2025.1.5", "capturekit": "^1.0.13", "electron-updater": "^6.8.3", "ffmpeg-static": "^5.3.0", diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index 09cca63ef..3872468a2 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -69,6 +69,8 @@ function LaunchWindowContent() { setMicrophoneEnabled, microphoneDeviceId, setMicrophoneDeviceId, + noiseSuppressionMode, + setNoiseSuppressionMode, systemAudioEnabled, setSystemAudioEnabled, webcamEnabled, @@ -262,6 +264,8 @@ function LaunchWindowContent() { devices={devices} microphoneDeviceId={microphoneDeviceId} selectedDeviceId={selectedDeviceId} + noiseSuppressionMode={noiseSuppressionMode} + onSelectNoiseSuppressionMode={setNoiseSuppressionMode} onSelectDevice={(deviceId) => { setMicrophoneEnabled(true); setSelectedDeviceId(deviceId); diff --git a/src/components/launch/popovers/MicPopover.tsx b/src/components/launch/popovers/MicPopover.tsx index 02247d662..fe4ed3fef 100644 --- a/src/components/launch/popovers/MicPopover.tsx +++ b/src/components/launch/popovers/MicPopover.tsx @@ -1,10 +1,17 @@ -import { MicrophoneSlashIcon, SpeakerHighIcon, SpeakerXIcon } from "@phosphor-icons/react"; +import { + CheckCircleIcon, + MicrophoneIcon, + MicrophoneSlashIcon, + SpeakerHighIcon, + SpeakerXIcon, +} from "@phosphor-icons/react"; +import type { ReactElement } from "react"; import { useScopedT } from "@/contexts/I18nContext"; -import { DropdownItem, HudPopover, MicDeviceRow } from "./PopoverScaffold"; +import type { NoiseSuppressionMode } from "@/lib/audio/noiseSuppression"; +import styles from "../LaunchWindow.module.css"; import { useLaunchPopoverCoordinator } from "./LaunchPopoverCoordinator"; import type { DeviceOption } from "./launchPopoverTypes"; -import type { ReactElement } from "react"; -import styles from "../LaunchWindow.module.css"; +import { DropdownItem, HudPopover, MicDeviceRow } from "./PopoverScaffold"; const POPOVER_ID = "mic"; @@ -19,6 +26,8 @@ export function MicPopover({ microphoneDeviceId, selectedDeviceId, onSelectDevice, + noiseSuppressionMode, + onSelectNoiseSuppressionMode, }: { trigger: ReactElement; disabled?: boolean; @@ -30,6 +39,8 @@ export function MicPopover({ microphoneDeviceId?: string; selectedDeviceId?: string; onSelectDevice: (deviceId: string) => void; + noiseSuppressionMode: NoiseSuppressionMode; + onSelectNoiseSuppressionMode: (mode: NoiseSuppressionMode) => void; }) { const t = useScopedT("launch"); const { isOpen, requestOpen, requestClose } = useLaunchPopoverCoordinator(); @@ -53,7 +64,9 @@ export function MicPopover({ >
{t("recording.microphone")}
: } + icon={ + systemAudioEnabled ? : + } selected={systemAudioEnabled} onClick={onToggleSystemAudio} > @@ -73,7 +86,9 @@ export function MicPopover({ )} {!microphoneEnabled && ( -
{t("recording.selectMicToEnable")}
+
+ {t("recording.selectMicToEnable")} +
)} {devices.map((device) => ( onSelectDevice(device.deviceId)} /> ))} {devices.length === 0 && ( -
{t("recording.noMicrophonesFound")}
+
+ {t("recording.noMicrophonesFound")} +
)} +
{t("recording.noiseSuppression")}
+ + + ); } + +function NoiseSuppressionOption({ + mode, + selectedMode, + label, + description, + onSelect, +}: { + mode: NoiseSuppressionMode; + selectedMode: NoiseSuppressionMode; + label: string; + description: string; + onSelect: (mode: NoiseSuppressionMode) => void; +}) { + const selected = mode === selectedMode; + return ( + + ); +} diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index c5cd70056..071c8cfb5 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -1,6 +1,16 @@ import { fixWebmDuration } from "@fix-webm-duration/fix"; import { useCallback, useEffect, useRef, useState } from "react"; import { toast } from "sonner"; +import { useI18n } from "@/contexts/I18nContext"; +import { + DEFAULT_NOISE_SUPPRESSION_MODE, + normalizeNoiseSuppressionMode, + type NoiseSuppressionMode, +} from "@/lib/audio/noiseSuppression"; +import { + createNoiseSuppressedMicrophoneStream, + type NoiseSuppressedMicrophoneStream, +} from "@/lib/audio/noiseSuppressedStream"; import { getEffectiveRecordingDurationMs } from "@/lib/mediaTiming"; import { getVideoExtensionForMimeType, @@ -39,6 +49,7 @@ const WEBCAM_FRAME_RATE = 30; const WEBCAM_SUFFIX = "-webcam"; const MICROPHONE_FALLBACK_ERROR_TOAST_ID = "recording-microphone-fallback-error"; const MICROPHONE_SIDECAR_ERROR_TOAST_ID = "recording-microphone-sidecar-error"; + export type BrowserMicrophoneProfile = | "processed" | "no-agc" @@ -46,6 +57,7 @@ export type BrowserMicrophoneProfile = | "no-noise-suppression" | "raw"; type BrowserCaptureCursorMode = "always" | "never"; + export type BrowserCaptureCursorPolicy = { streamCursor: BrowserCaptureCursorMode; hideOsCursorBeforeRecording: boolean; @@ -141,6 +153,8 @@ type UseScreenRecorderReturn = { setMicrophoneEnabled: (enabled: boolean) => void; microphoneDeviceId: string | undefined; setMicrophoneDeviceId: (deviceId: string | undefined) => void; + noiseSuppressionMode: NoiseSuppressionMode; + setNoiseSuppressionMode: (mode: NoiseSuppressionMode) => void; systemAudioEnabled: boolean; setSystemAudioEnabled: (enabled: boolean) => void; webcamEnabled: boolean; @@ -213,10 +227,7 @@ export function resolveBrowserCaptureCursorPolicy({ export function shouldUseNativeWindowsCaptureForSource( source: Pick | null | undefined, ): boolean { - return ( - source?.id?.startsWith("screen:") === true || - source?.id?.startsWith("window:") === true - ); + return source?.id?.startsWith("screen:") === true || source?.id?.startsWith("window:") === true; } export function createProcessedMicrophoneConstraints( @@ -320,6 +331,7 @@ async function createAudioInputDeviceSnapshot(): Promise< } export function useScreenRecorder(): UseScreenRecorderReturn { + const { t } = useI18n(); const [recording, setRecording] = useState(false); const [paused, setPaused] = useState(false); const [starting, setStarting] = useState(false); @@ -328,6 +340,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const [isMacOS, setIsMacOS] = useState(false); const [microphoneEnabled, setMicrophoneEnabled] = useState(false); const [microphoneDeviceId, setMicrophoneDeviceId] = useState(undefined); + const [noiseSuppressionMode, setNoiseSuppressionModeState] = useState( + DEFAULT_NOISE_SUPPRESSION_MODE, + ); const [systemAudioEnabled, setSystemAudioEnabled] = useState(false); const [webcamEnabled, setWebcamEnabled] = useState(false); const [webcamDeviceId, setWebcamDeviceId] = useState(undefined); @@ -337,6 +352,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const stream = useRef(null); const screenStream = useRef(null); const microphoneStream = useRef(null); + const noiseSuppressedMicrophoneStreams = useRef([]); const webcamStream = useRef(null); const mixingContext = useRef(null); const chunks = useRef([]); @@ -554,6 +570,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }; const cleanupCapturedMedia = useCallback(() => { + for (const processedStream of noiseSuppressedMicrophoneStreams.current) { + processedStream.destroy(); + } + noiseSuppressedMicrophoneStreams.current = []; + if (stream.current) { stream.current.getTracks().forEach((track) => track.stop()); stream.current = null; @@ -598,6 +619,35 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } }, [resetMicFallbackTimingDiagnostics]); + const prepareMicrophoneStream = useCallback( + async (rawStream: MediaStream) => { + try { + const processed = await createNoiseSuppressedMicrophoneStream({ + sourceStream: rawStream, + mode: noiseSuppressionMode, + onWarning: (warning) => { + const fallback = + warning.key === "recording.noiseSuppressionUnavailableWarning" + ? "Noise suppression is unavailable on this device. Recording will continue without it." + : "Noise suppression fell back to {{mode}}."; + const params = "params" in warning ? warning.params : undefined; + toast.warning(t(warning.key, fallback, params), { duration: 7000 }); + }, + }); + noiseSuppressedMicrophoneStreams.current.push(processed); + console.info("[NoiseSuppression] Selected microphone mode.", { + requestedMode: processed.requestedMode, + activeMode: processed.activeMode, + }); + return processed.stream; + } catch (error) { + rawStream.getTracks().forEach((track) => track.stop()); + throw error; + } + }, + [noiseSuppressionMode, t], + ); + const appendMicFallbackChunk = useCallback( (event: BlobEvent) => { if (event.data.size <= 0) { @@ -1266,6 +1316,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (result.microphoneDeviceId) { setMicrophoneDeviceId(result.microphoneDeviceId); } + setNoiseSuppressionModeState( + normalizeNoiseSuppressionMode(result.noiseSuppressionMode), + ); setSystemAudioEnabled(result.systemAudioEnabled); } })(); @@ -1281,6 +1334,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn { void window.electronAPI.setRecordingPreferences({ microphoneDeviceId: deviceId }); }, []); + const persistNoiseSuppressionMode = useCallback((mode: NoiseSuppressionMode) => { + const normalizedMode = normalizeNoiseSuppressionMode(mode); + setNoiseSuppressionModeState(normalizedMode); + void window.electronAPI.setRecordingPreferences({ + noiseSuppressionMode: normalizedMode, + }); + }, []); + const persistSystemAudioEnabled = useCallback((enabled: boolean) => { setSystemAudioEnabled(enabled); void window.electronAPI.setRecordingPreferences({ systemAudioEnabled: enabled }); @@ -1361,6 +1422,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } let hudSourceSelectionActive = false; + const setHudSourceSelectionActive = (active: boolean) => { if (hudSourceSelectionActive === active) { return; @@ -1457,7 +1519,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { selectedSource, { capturesSystemAudio: systemAudioEnabled, - capturesMicrophone: microphoneEnabled, + capturesMicrophone: + microphoneEnabled && noiseSuppressionMode === "disabled", microphoneDeviceId, microphoneLabel: micLabel, }, @@ -1504,7 +1567,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { // When native mic capture is unavailable or explicitly bypassed, // record mic via browser getUserMedia as a sidecar file. - if (nativeResult.microphoneFallbackRequired && microphoneEnabled) { + if ( + (nativeResult.microphoneFallbackRequired || + noiseSuppressionMode !== "disabled") && + microphoneEnabled + ) { void logNativeCaptureDiagnostics("start-browser-microphone-fallback"); console.info("Using browser microphone processing for this recording."); try { @@ -1513,10 +1580,12 @@ export function useScreenRecorder(): UseScreenRecorderReturn { browserMicrophoneProfile.current, ); micFallbackRequestedConstraints.current = microphoneConstraints; - const micStream = + const rawMicStream = await navigator.mediaDevices.getUserMedia(microphoneConstraints); + microphoneStream.current = rawMicStream; + const micStream = await prepareMicrophoneStream(rawMicStream); micFallbackTrackSettings.current = - createMicrophoneTrackSettingsSnapshot(micStream); + createMicrophoneTrackSettingsSnapshot(rawMicStream); micFallbackAudioInputDevices.current = await createAudioInputDeviceSnapshot().catch(() => null); console.info( @@ -1632,6 +1701,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (wantsAudioCapture) { let screenMediaStream: MediaStream; + const acquireLinuxPortalStream = (withAudio: boolean) => mediaDevices.getDisplayMedia({ audio: withAudio, @@ -1695,12 +1765,15 @@ export function useScreenRecorder(): UseScreenRecorderReturn { if (microphoneEnabled) { try { - microphoneStream.current = await navigator.mediaDevices.getUserMedia( + const rawMicrophoneStream = await navigator.mediaDevices.getUserMedia( createProcessedMicrophoneConstraints( microphoneDeviceId, browserMicrophoneProfile.current, ), ); + microphoneStream.current = rawMicrophoneStream; + microphoneStream.current = + await prepareMicrophoneStream(rawMicrophoneStream); } catch (audioError) { console.warn("Failed to get microphone access:", audioError); alert( @@ -2116,6 +2189,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { setMicrophoneEnabled: persistMicrophoneEnabled, microphoneDeviceId, setMicrophoneDeviceId: persistMicrophoneDeviceId, + noiseSuppressionMode, + setNoiseSuppressionMode: persistNoiseSuppressionMode, systemAudioEnabled, setSystemAudioEnabled: persistSystemAudioEnabled, webcamEnabled, diff --git a/src/i18n/locales/en/launch.json b/src/i18n/locales/en/launch.json index 9dfcbe038..633a50eb2 100644 --- a/src/i18n/locales/en/launch.json +++ b/src/i18n/locales/en/launch.json @@ -28,6 +28,15 @@ "turnOffMicrophone": "Turn Off Microphone", "selectMicToEnable": "Select a microphone to enable", "noMicrophonesFound": "No microphones found", + "noiseSuppression": "Noise suppression", + "noiseSuppressionRnnoise": "RNNoise - Better quality", + "noiseSuppressionRnnoiseDescription": "Better background-noise removal, higher CPU usage.", + "noiseSuppressionSpeex": "Speex - Lower CPU usage", + "noiseSuppressionSpeexDescription": "Lower CPU usage, but reduced voice quality.", + "noiseSuppressionDisabled": "Disabled", + "noiseSuppressionDisabledDescription": "Records the original microphone audio.", + "noiseSuppressionUnavailableWarning": "Noise suppression is unavailable on this device. Recording will continue without it.", + "noiseSuppressionFallbackWarning": "Noise suppression fell back to {{mode}}.", "webcam": "Webcam", "turnOffWebcam": "Turn Off Webcam", "hideFloatingWebcamPreview": "Hide Floating Preview", diff --git a/src/i18n/locales/es/launch.json b/src/i18n/locales/es/launch.json index a458844a2..773c51013 100644 --- a/src/i18n/locales/es/launch.json +++ b/src/i18n/locales/es/launch.json @@ -28,6 +28,15 @@ "turnOffMicrophone": "Desactivar micrófono", "selectMicToEnable": "Selecciona un micrófono para activar", "noMicrophonesFound": "No se encontraron micrófonos", + "noiseSuppression": "Supresión de ruido", + "noiseSuppressionRnnoise": "RNNoise - Mejor calidad", + "noiseSuppressionRnnoiseDescription": "Mejor eliminación del ruido de fondo, mayor uso de CPU.", + "noiseSuppressionSpeex": "Speex - Menor uso de CPU", + "noiseSuppressionSpeexDescription": "Menor uso de CPU, pero calidad de voz reducida.", + "noiseSuppressionDisabled": "Desactivado", + "noiseSuppressionDisabledDescription": "Graba el audio original del micrófono.", + "noiseSuppressionUnavailableWarning": "La supresión de ruido no está disponible en este dispositivo. La grabación continuará sin ella.", + "noiseSuppressionFallbackWarning": "La supresión de ruido cambió a {{mode}}.", "webcam": "Cámara", "turnOffWebcam": "Desactivar cámara", "hideFloatingWebcamPreview": "Ocultar vista flotante", diff --git a/src/i18n/locales/fr/launch.json b/src/i18n/locales/fr/launch.json index df2e43b2b..1a0699bd7 100644 --- a/src/i18n/locales/fr/launch.json +++ b/src/i18n/locales/fr/launch.json @@ -28,6 +28,15 @@ "turnOffMicrophone": "Désactiver le microphone", "selectMicToEnable": "Sélectionnez un microphone à activer", "noMicrophonesFound": "Aucun microphone trouvé", + "noiseSuppression": "Réduction du bruit", + "noiseSuppressionRnnoise": "RNNoise - Meilleure qualité", + "noiseSuppressionRnnoiseDescription": "Meilleure suppression du bruit de fond, utilisation du processeur plus élevée.", + "noiseSuppressionSpeex": "Speex - Utilisation du processeur réduite", + "noiseSuppressionSpeexDescription": "Utilisation du processeur réduite, mais qualité vocale moindre.", + "noiseSuppressionDisabled": "Désactivé", + "noiseSuppressionDisabledDescription": "Enregistre l’audio original du microphone.", + "noiseSuppressionUnavailableWarning": "La réduction du bruit n’est pas disponible sur cet appareil. L’enregistrement continuera sans elle.", + "noiseSuppressionFallbackWarning": "La réduction du bruit est passée à {{mode}}.", "webcam": "Webcam", "turnOffWebcam": "Désactiver la webcam", "hideFloatingWebcamPreview": "Masquer l’aperçu flottant", diff --git a/src/i18n/locales/it/launch.json b/src/i18n/locales/it/launch.json index c27abc3ad..78f0988bc 100644 --- a/src/i18n/locales/it/launch.json +++ b/src/i18n/locales/it/launch.json @@ -28,6 +28,15 @@ "turnOffMicrophone": "Spegni microfono", "selectMicToEnable": "Seleziona un microfono da abilitare", "noMicrophonesFound": "Nessun microfono trovato", + "noiseSuppression": "Riduzione del rumore", + "noiseSuppressionRnnoise": "RNNoise - Qualità migliore", + "noiseSuppressionRnnoiseDescription": "Rimozione migliore del rumore di fondo, maggiore uso della CPU.", + "noiseSuppressionSpeex": "Speex - Minore uso della CPU", + "noiseSuppressionSpeexDescription": "Minore uso della CPU, ma qualità vocale ridotta.", + "noiseSuppressionDisabled": "Disabilitata", + "noiseSuppressionDisabledDescription": "Registra l'audio originale del microfono.", + "noiseSuppressionUnavailableWarning": "La riduzione del rumore non è disponibile su questo dispositivo. La registrazione continuerà senza.", + "noiseSuppressionFallbackWarning": "La riduzione del rumore è passata a {{mode}}.", "webcam": "Webcam", "turnOffWebcam": "Spegni webcam", "hideFloatingWebcamPreview": "Nascondi anteprima fluttuante", diff --git a/src/i18n/locales/ko/launch.json b/src/i18n/locales/ko/launch.json index 345000399..2d348d8e1 100644 --- a/src/i18n/locales/ko/launch.json +++ b/src/i18n/locales/ko/launch.json @@ -28,6 +28,15 @@ "turnOffMicrophone": "마이크 끄기", "selectMicToEnable": "사용할 마이크를 선택하세요", "noMicrophonesFound": "마이크를 찾을 수 없습니다", + "noiseSuppression": "소음 억제", + "noiseSuppressionRnnoise": "RNNoise - 더 높은 품질", + "noiseSuppressionRnnoiseDescription": "배경 소음을 더 잘 제거하지만 CPU 사용량이 높습니다.", + "noiseSuppressionSpeex": "Speex - 더 낮은 CPU 사용량", + "noiseSuppressionSpeexDescription": "CPU 사용량은 낮지만 음성 품질이 떨어질 수 있습니다.", + "noiseSuppressionDisabled": "비활성화", + "noiseSuppressionDisabledDescription": "원본 마이크 오디오를 녹음합니다.", + "noiseSuppressionUnavailableWarning": "이 기기에서는 소음 억제를 사용할 수 없습니다. 녹음은 소음 억제 없이 계속됩니다.", + "noiseSuppressionFallbackWarning": "소음 억제가 {{mode}}(으)로 전환되었습니다.", "webcam": "웹캠", "turnOffWebcam": "웹캠 끄기", "hideFloatingWebcamPreview": "플로팅 미리보기 숨기기", diff --git a/src/i18n/locales/nl/launch.json b/src/i18n/locales/nl/launch.json index d3d5870c9..051ced057 100644 --- a/src/i18n/locales/nl/launch.json +++ b/src/i18n/locales/nl/launch.json @@ -28,6 +28,15 @@ "turnOffMicrophone": "Microfoon uitschakelen", "selectMicToEnable": "Selecteer een microfoon om in te schakelen", "noMicrophonesFound": "Geen microfoons gevonden", + "noiseSuppression": "Ruisonderdrukking", + "noiseSuppressionRnnoise": "RNNoise - Betere kwaliteit", + "noiseSuppressionRnnoiseDescription": "Betere verwijdering van achtergrondruis, hoger CPU-gebruik.", + "noiseSuppressionSpeex": "Speex - Lager CPU-gebruik", + "noiseSuppressionSpeexDescription": "Lager CPU-gebruik, maar lagere stemkwaliteit.", + "noiseSuppressionDisabled": "Uitgeschakeld", + "noiseSuppressionDisabledDescription": "Neemt de originele microfoonaudio op.", + "noiseSuppressionUnavailableWarning": "Ruisonderdrukking is niet beschikbaar op dit apparaat. De opname gaat zonder ruisonderdrukking door.", + "noiseSuppressionFallbackWarning": "Ruisonderdrukking is teruggevallen op {{mode}}.", "webcam": "Webcam", "turnOffWebcam": "Webcam uitschakelen", "hideFloatingWebcamPreview": "Zwevende voorbeeldweergave verbergen", diff --git a/src/i18n/locales/pt-BR/launch.json b/src/i18n/locales/pt-BR/launch.json index 8d19ac7db..1c6b55aa6 100644 --- a/src/i18n/locales/pt-BR/launch.json +++ b/src/i18n/locales/pt-BR/launch.json @@ -28,6 +28,15 @@ "turnOffMicrophone": "Desligar microfone", "selectMicToEnable": "Selecione um microfone para ativar", "noMicrophonesFound": "Nenhum microfone encontrado", + "noiseSuppression": "Supressão de ruído", + "noiseSuppressionRnnoise": "RNNoise - Melhor qualidade", + "noiseSuppressionRnnoiseDescription": "Melhor remoção de ruído de fundo, maior uso de CPU.", + "noiseSuppressionSpeex": "Speex - Menor uso de CPU", + "noiseSuppressionSpeexDescription": "Menor uso de CPU, mas qualidade de voz reduzida.", + "noiseSuppressionDisabled": "Desativado", + "noiseSuppressionDisabledDescription": "Grava o áudio original do microfone.", + "noiseSuppressionUnavailableWarning": "A supressão de ruído não está disponível neste dispositivo. A gravação continuará sem ela.", + "noiseSuppressionFallbackWarning": "A supressão de ruído alternou para {{mode}}.", "webcam": "Webcam", "turnOffWebcam": "Desligar webcam", "hideFloatingWebcamPreview": "Ocultar prévia flutuante", diff --git a/src/i18n/locales/ru/launch.json b/src/i18n/locales/ru/launch.json index 65bd57b8c..7e5230fe0 100644 --- a/src/i18n/locales/ru/launch.json +++ b/src/i18n/locales/ru/launch.json @@ -28,6 +28,15 @@ "turnOffMicrophone": "Выключить микрофон", "selectMicToEnable": "Выберите микрофон для записи звука", "noMicrophonesFound": "Нет доступных микрофонов", + "noiseSuppression": "Шумоподавление", + "noiseSuppressionRnnoise": "RNNoise - Лучшее качество", + "noiseSuppressionRnnoiseDescription": "Лучше удаляет фоновый шум, но сильнее нагружает процессор.", + "noiseSuppressionSpeex": "Speex - Меньше нагрузка на процессор", + "noiseSuppressionSpeexDescription": "Меньше нагрузка на процессор, но ниже качество голоса.", + "noiseSuppressionDisabled": "Отключено", + "noiseSuppressionDisabledDescription": "Записывает исходный звук с микрофона.", + "noiseSuppressionUnavailableWarning": "Шумоподавление недоступно на этом устройстве. Запись продолжится без него.", + "noiseSuppressionFallbackWarning": "Шумоподавление переключилось на {{mode}}.", "webcam": "Веб-камера", "turnOffWebcam": "Выключить веб-камеру", "hideFloatingWebcamPreview": "Скрыть предпросмотр", @@ -77,4 +86,4 @@ "failedToStart": "Не удалось начать запись: {{error}}", "failedToStartGeneric": "Не удалось начать запись" } -} \ No newline at end of file +} diff --git a/src/i18n/locales/zh-CN/launch.json b/src/i18n/locales/zh-CN/launch.json index 164c02afe..76c0ea89d 100644 --- a/src/i18n/locales/zh-CN/launch.json +++ b/src/i18n/locales/zh-CN/launch.json @@ -28,6 +28,15 @@ "turnOffMicrophone": "关闭麦克风", "selectMicToEnable": "选择一个麦克风以启用", "noMicrophonesFound": "未找到麦克风", + "noiseSuppression": "噪声抑制", + "noiseSuppressionRnnoise": "RNNoise - 更高质量", + "noiseSuppressionRnnoiseDescription": "更好地去除背景噪声，但 CPU 使用率更高。", + "noiseSuppressionSpeex": "Speex - 更低 CPU 使用率", + "noiseSuppressionSpeexDescription": "CPU 使用率更低，但语音质量会降低。", + "noiseSuppressionDisabled": "已禁用", + "noiseSuppressionDisabledDescription": "录制原始麦克风音频。", + "noiseSuppressionUnavailableWarning": "此设备不支持噪声抑制。录制将继续，但不会使用噪声抑制。", + "noiseSuppressionFallbackWarning": "噪声抑制已回退到 {{mode}}。", "webcam": "摄像头", "turnOffWebcam": "关闭摄像头", "hideFloatingWebcamPreview": "隐藏悬浮预览", diff --git a/src/i18n/locales/zh-TW/launch.json b/src/i18n/locales/zh-TW/launch.json index 12ec494d0..bf830d70e 100644 --- a/src/i18n/locales/zh-TW/launch.json +++ b/src/i18n/locales/zh-TW/launch.json @@ -28,6 +28,15 @@ "turnOffMicrophone": "關閉麥克風", "selectMicToEnable": "選擇要啟用的麥克風", "noMicrophonesFound": "找不到麥克風", + "noiseSuppression": "噪音抑制", + "noiseSuppressionRnnoise": "RNNoise - 較佳品質", + "noiseSuppressionRnnoiseDescription": "更有效移除背景噪音，但 CPU 使用率較高。", + "noiseSuppressionSpeex": "Speex - 較低 CPU 使用率", + "noiseSuppressionSpeexDescription": "CPU 使用率較低，但語音品質會下降。", + "noiseSuppressionDisabled": "已停用", + "noiseSuppressionDisabledDescription": "錄製原始麥克風音訊。", + "noiseSuppressionUnavailableWarning": "此裝置無法使用噪音抑制。錄製將繼續，但不會使用噪音抑制。", + "noiseSuppressionFallbackWarning": "噪音抑制已改用 {{mode}}。", "webcam": "網路攝影機", "turnOffWebcam": "關閉網路攝影機", "hideFloatingWebcamPreview": "隱藏浮動預覽", @@ -44,15 +53,15 @@ "cancel": "取消", "more": "更多", "update": { - "update": "更新", - "updated": "已更新", - "idleTitle": "檢查更新。", - "checkingTitle": "正在檢查更新...", - "downloadingTitle": "正在下載更新...", - "errorTitle": "檢查更新失敗，點一下重試。", - "upToDateTitle": "Recordly {{version}} 已是最新版本。", - "availableTitle": "Recordly {{version}} 已推出新版本。", - "availableGenericTitle": "有可用更新。" + "update": "更新", + "updated": "已更新", + "idleTitle": "檢查更新。", + "checkingTitle": "正在檢查更新...", + "downloadingTitle": "正在下載更新...", + "errorTitle": "檢查更新失敗，點一下重試。", + "upToDateTitle": "Recordly {{version}} 已是最新版本。", + "availableTitle": "Recordly {{version}} 已推出新版本。", + "availableGenericTitle": "有可用更新。" } }, "sourceSelector": { diff --git a/src/lib/audio/noiseSuppressedStream.test.ts b/src/lib/audio/noiseSuppressedStream.test.ts new file mode 100644 index 000000000..c48ef2aa4 --- /dev/null +++ b/src/lib/audio/noiseSuppressedStream.test.ts @@ -0,0 +1,134 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createNoiseSuppressedMicrophoneStream } from "./noiseSuppressedStream"; + +vi.mock("./noiseSuppressionWorklet.ts?worker&url", () => ({ + default: "mock-noise-suppression-worklet-url", +})); + +type FakeMediaStreamTrack = MediaStreamTrack & { + stop: ReturnType; +}; + +type FakeMediaStream = MediaStream & { + getTracks: ReturnType FakeMediaStreamTrack[]>>; +}; + +type FakeAudioNode = { + connect: ReturnType; + disconnect: ReturnType; +}; + +const createdContexts: FakeAudioContext[] = []; +const createdProcessors: FakeAudioWorkletNode[] = []; +let addModule = vi.fn(); + +class FakeAudioContext { + readonly audioWorklet = { + addModule: (...args: unknown[]) => addModule(...args), + }; + readonly close = vi.fn().mockResolvedValue(undefined); + readonly sampleRate = 48000; + readonly sourceNode = createFakeAudioNode(); + readonly destinationNode = { + ...createFakeAudioNode(), + stream: createFakeMediaStream().stream, + }; + + constructor() { + createdContexts.push(this); + } + + createMediaStreamSource() { + return this.sourceNode; + } + + createMediaStreamDestination() { + return this.destinationNode; + } +} + +class FakeAudioWorkletNode { + readonly connect = vi.fn(); + readonly disconnect = vi.fn(); + readonly port = { + onmessage: null as ((event: MessageEvent) => void) | null, + postMessage: vi.fn(), + }; + + constructor() { + createdProcessors.push(this); + } +} + +function createFakeAudioNode(): FakeAudioNode { + return { + connect: vi.fn(), + disconnect: vi.fn(), + }; +} + +function createFakeMediaStream(trackCount = 1) { + const tracks = Array.from({ length: trackCount }, () => ({ + stop: vi.fn(), + })) as FakeMediaStreamTrack[]; + const stream = { + getTracks: vi.fn(() => tracks), + } as FakeMediaStream; + + return { stream, tracks }; +} + +beforeEach(() => { + addModule = vi.fn().mockResolvedValue(undefined); + createdContexts.length = 0; + createdProcessors.length = 0; + vi.stubGlobal("AudioContext", FakeAudioContext); + vi.stubGlobal("AudioWorkletNode", FakeAudioWorkletNode); +}); + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + +describe("createNoiseSuppressedMicrophoneStream", () => { + it("stops source tracks and closes the AudioContext when loading the worklet fails", async () => { + const { stream, tracks } = createFakeMediaStream(2); + addModule.mockRejectedValue(new Error("worklet blocked")); + + await expect( + createNoiseSuppressedMicrophoneStream({ + sourceStream: stream, + mode: "rnnoise", + }), + ).rejects.toThrow("worklet blocked"); + + expect(tracks[0].stop).toHaveBeenCalledTimes(1); + expect(tracks[1].stop).toHaveBeenCalledTimes(1); + expect(createdContexts[0].sourceNode.disconnect).toHaveBeenCalledTimes(1); + expect(createdContexts[0].close).toHaveBeenCalledTimes(1); + }); + + it("times out a nonresponsive worklet and runs setup cleanup", async () => { + vi.useFakeTimers(); + const { stream, tracks } = createFakeMediaStream(); + + const setup = createNoiseSuppressedMicrophoneStream({ + sourceStream: stream, + mode: "rnnoise", + }); + const rejection = expect(setup).rejects.toThrow( + "Noise suppression worklet initialization timed out.", + ); + await vi.advanceTimersByTimeAsync(5000); + + await rejection; + expect(createdProcessors[0].port.postMessage).toHaveBeenCalledWith( + expect.objectContaining({ type: "init" }), + ); + expect(createdProcessors[0].port.postMessage).toHaveBeenCalledWith({ type: "destroy" }); + expect(tracks[0].stop).toHaveBeenCalledTimes(1); + expect(createdContexts[0].sourceNode.disconnect).toHaveBeenCalledTimes(1); + expect(createdContexts[0].close).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/lib/audio/noiseSuppressedStream.ts b/src/lib/audio/noiseSuppressedStream.ts new file mode 100644 index 000000000..6f454b394 --- /dev/null +++ b/src/lib/audio/noiseSuppressedStream.ts @@ -0,0 +1,246 @@ +import { + type NoiseSuppressionMode, + type NoiseSuppressionSelection, + normalizeNoiseSuppressionMode, +} from "./noiseSuppression"; +import noiseSuppressionWorkletUrl from "./noiseSuppressionWorklet.ts?worker&url"; + +const PROCESSOR_BUFFER_SIZE = 1024; +const DEFAULT_SAMPLE_RATE = 48000; +const FRAME_BUDGET_MS = 12; +const LONG_FRAME_LOG_INTERVAL_MS = 5000; +const WORKLET_READY_TIMEOUT_MS = 5000; + +export type NoiseSuppressedMicrophoneStream = { + stream: MediaStream; + activeMode: NoiseSuppressionMode; + requestedMode: NoiseSuppressionMode; + warnings: string[]; + destroy(): void; +}; + +export type NoiseSuppressionWarning = + | { + key: "recording.noiseSuppressionUnavailableWarning"; + } + | { + key: "recording.noiseSuppressionFallbackWarning"; + params: { mode: string }; + }; + +export async function createNoiseSuppressedMicrophoneStream({ + sourceStream, + mode, + onWarning, +}: { + sourceStream: MediaStream; + mode: NoiseSuppressionMode; + onWarning?: (warning: NoiseSuppressionWarning) => void; +}): Promise { + const requestedMode = normalizeNoiseSuppressionMode(mode); + if (requestedMode === "disabled") { + console.info("[NoiseSuppression] Microphone stream passthrough selected."); + return { + stream: sourceStream, + activeMode: "disabled", + requestedMode, + warnings: [], + destroy: () => { + sourceStream.getTracks().forEach((track) => track.stop()); + console.info("[NoiseSuppression] Passthrough stream cleanup complete."); + }, + }; + } + + const context = new AudioContext({ sampleRate: DEFAULT_SAMPLE_RATE }); + const source = context.createMediaStreamSource(sourceStream); + const destination = context.createMediaStreamDestination(); + const sampleRate = context.sampleRate || DEFAULT_SAMPLE_RATE; + let processor: AudioWorkletNode | null = null; + let selection: NoiseSuppressionWorkletReadySelection; + + try { + await context.audioWorklet.addModule(noiseSuppressionWorkletUrl); + processor = new AudioWorkletNode(context, "noise-suppression-processor", { + numberOfInputs: 1, + numberOfOutputs: 1, + outputChannelCount: [1], + }); + selection = await initializeNoiseSuppressionWorklet(processor, { + mode: requestedMode, + frameSize: PROCESSOR_BUFFER_SIZE, + sampleRate, + frameBudgetMs: FRAME_BUDGET_MS, + longFrameLogIntervalMs: LONG_FRAME_LOG_INTERVAL_MS, + }); + source.connect(processor); + processor.connect(destination); + } catch (error) { + cleanupFailedNoiseSuppressionSetup({ + context, + destinationStream: destination.stream, + processor, + source, + sourceStream, + }); + throw error; + } + + if (!processor) { + throw new Error("Noise suppression worklet processor was not created."); + } + const activeProcessor = processor; + + reportWarnings(selection, onWarning); + + console.info("[NoiseSuppression] Microphone stream processing active.", { + requestedMode, + activeMode: selection.activeMode, + frameSize: PROCESSOR_BUFFER_SIZE, + sampleRate, + }); + + return { + stream: destination.stream, + activeMode: selection.activeMode, + requestedMode, + warnings: selection.warnings, + destroy: () => { + activeProcessor.port.postMessage({ type: "destroy" }); + activeProcessor.disconnect(); + source.disconnect(); + sourceStream.getTracks().forEach((track) => track.stop()); + destination.stream.getTracks().forEach((track) => track.stop()); + context.close().catch(() => undefined); + console.info("[NoiseSuppression] Microphone stream cleanup complete.", { + requestedMode, + activeMode: selection.activeMode, + }); + }, + }; +} + +type NoiseSuppressionWorkletReadySelection = Omit; + +async function initializeNoiseSuppressionWorklet( + processor: AudioWorkletNode, + config: { + mode: NoiseSuppressionMode; + frameSize: number; + frameBudgetMs: number; + longFrameLogIntervalMs: number; + sampleRate: number; + }, +): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const readyTimeout = setTimeout(() => { + settled = true; + reject(new Error("Noise suppression worklet initialization timed out.")); + }, WORKLET_READY_TIMEOUT_MS); + + const settle = ( + callback: () => NoiseSuppressionWorkletReadySelection, + complete: (selection: NoiseSuppressionWorkletReadySelection) => void, + ) => { + if (settled) { + return; + } + settled = true; + clearTimeout(readyTimeout); + complete(callback()); + }; + + processor.port.onmessage = (event) => { + const message = event.data; + if (message.type === "ready") { + settle( + () => ({ + requestedMode: message.requestedMode, + activeMode: message.activeMode, + warnings: message.warnings, + }), + resolve, + ); + return; + } + if (message.type === "error") { + if (!settled) { + settled = true; + clearTimeout(readyTimeout); + reject(new Error(message.error)); + } + return; + } + if (message.type === "processing-error") { + console.warn("[NoiseSuppression] Processing error; passed through frame.", { + activeMode: message.activeMode, + droppedFrames: message.droppedFrames, + error: message.error, + }); + return; + } + if (message.type === "long-frame") { + console.warn("[NoiseSuppression] Processing exceeded real-time budget.", { + activeMode: message.activeMode, + elapsedMs: message.elapsedMs, + frameSize: message.frameSize, + sampleRate: message.sampleRate, + }); + } + }; + processor.port.postMessage({ type: "init", ...config }); + }); +} + +function cleanupFailedNoiseSuppressionSetup({ + context, + destinationStream, + processor, + source, + sourceStream, +}: { + context: AudioContext; + destinationStream: MediaStream; + processor: AudioWorkletNode | null; + source: MediaStreamAudioSourceNode; + sourceStream: MediaStream; +}) { + try { + processor?.port.postMessage({ type: "destroy" }); + } catch { + // Best effort: preserve the original setup error. + } + try { + processor?.disconnect(); + } catch { + // Best effort: preserve the original setup error. + } + try { + source.disconnect(); + } catch { + // Best effort: preserve the original setup error. + } + sourceStream.getTracks().forEach((track) => track.stop()); + destinationStream.getTracks().forEach((track) => track.stop()); + context.close().catch(() => undefined); +} + +function reportWarnings( + selection: Pick, + onWarning?: (warning: NoiseSuppressionWarning) => void, +): NoiseSuppressionWarning | null { + if (selection.warnings.length === 0) { + return null; + } + + const warning: NoiseSuppressionWarning = + selection.activeMode === "disabled" + ? { key: "recording.noiseSuppressionUnavailableWarning" } + : { + key: "recording.noiseSuppressionFallbackWarning", + params: { mode: selection.activeMode.toUpperCase() }, + }; + onWarning?.(warning); + return warning; +} diff --git a/src/lib/audio/noiseSuppression.test.ts b/src/lib/audio/noiseSuppression.test.ts new file mode 100644 index 000000000..f7d2a747d --- /dev/null +++ b/src/lib/audio/noiseSuppression.test.ts @@ -0,0 +1,185 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + createNoiseSuppressorWithFallback, + DEFAULT_NOISE_SUPPRESSION_MODE, + type NoiseSuppressor, + normalizeNoiseSuppressionMode, + RnnoiseNoiseSuppressor, +} from "./noiseSuppression"; + +const rnnoiseLoad = vi.hoisted(() => vi.fn()); + +vi.mock("@shiguredo/rnnoise-wasm", () => ({ + Rnnoise: { + load: rnnoiseLoad, + }, +})); + +function createMockSuppressor({ + initialize = vi.fn().mockResolvedValue(undefined), + processAudioFrame = vi.fn((frame: Float32Array) => frame), + destroy = vi.fn(), +}: Partial = {}): NoiseSuppressor { + return { + initialize, + processAudioFrame, + destroy, + }; +} + +beforeEach(() => { + rnnoiseLoad.mockReset(); +}); + +describe("normalizeNoiseSuppressionMode", () => { + it("defaults new and invalid values to RNNoise", () => { + expect(normalizeNoiseSuppressionMode()).toBe(DEFAULT_NOISE_SUPPRESSION_MODE); + expect(normalizeNoiseSuppressionMode("unknown")).toBe(DEFAULT_NOISE_SUPPRESSION_MODE); + }); + + it("normalizes supported persisted modes", () => { + expect(normalizeNoiseSuppressionMode(" RNNOISE ")).toBe("rnnoise"); + expect(normalizeNoiseSuppressionMode("speex")).toBe("speex"); + expect(normalizeNoiseSuppressionMode("disabled")).toBe("disabled"); + }); +}); + +describe("RnnoiseNoiseSuppressor", () => { + it("buffers incomplete RNNoise frames across calls without returning raw remainder samples", async () => { + const processFrame = vi.fn((frame: Float32Array) => { + for (let index = 0; index < frame.length; index += 1) { + frame[index] += 100; + } + }); + const destroy = vi.fn(); + rnnoiseLoad.mockResolvedValue({ + frameSize: 4, + createDenoiseState: () => ({ + processFrame, + destroy, + }), + }); + + const suppressor = new RnnoiseNoiseSuppressor(); + await suppressor.initialize(); + + const firstOutput = suppressor.processAudioFrame(new Float32Array([1, 2, 3, 4, 5, 6])); + expect(Array.from(firstOutput)).toEqual([101, 102, 103, 104, 0, 0]); + expect(processFrame).toHaveBeenCalledTimes(1); + + const secondOutput = suppressor.processAudioFrame(new Float32Array([7, 8, 9, 10, 11, 12])); + expect(Array.from(secondOutput)).toEqual([105, 106, 107, 108, 109, 110]); + expect(processFrame).toHaveBeenCalledTimes(3); + + const thirdOutput = suppressor.processAudioFrame(new Float32Array([13, 14])); + expect(Array.from(thirdOutput)).toEqual([111, 112]); + expect(processFrame).toHaveBeenCalledTimes(3); + + const fourthOutput = suppressor.processAudioFrame(new Float32Array([15, 16])); + expect(Array.from(fourthOutput)).toEqual([113, 114]); + expect(processFrame).toHaveBeenCalledTimes(4); + + suppressor.destroy(); + expect(destroy).toHaveBeenCalledTimes(1); + }); +}); + +describe("createNoiseSuppressorWithFallback", () => { + it("initializes RNNoise when requested", async () => { + const rnnoise = createMockSuppressor(); + const selection = await createNoiseSuppressorWithFallback({ + mode: "rnnoise", + frameSize: 1024, + sampleRate: 48000, + deps: { + createRnnoise: () => rnnoise, + }, + }); + + expect(selection.activeMode).toBe("rnnoise"); + expect(rnnoise.initialize).toHaveBeenCalledTimes(1); + }); + + it("initializes Speex when requested", async () => { + const speex = createMockSuppressor(); + const selection = await createNoiseSuppressorWithFallback({ + mode: "speex", + frameSize: 1024, + sampleRate: 48000, + deps: { + createSpeex: () => speex, + }, + }); + + expect(selection.activeMode).toBe("speex"); + expect(speex.initialize).toHaveBeenCalledTimes(1); + }); + + it("falls back from RNNoise to Speex", async () => { + const rnnoise = createMockSuppressor({ + initialize: vi.fn().mockRejectedValue(new Error("wasm unavailable")), + }); + const speex = createMockSuppressor(); + + const selection = await createNoiseSuppressorWithFallback({ + mode: "rnnoise", + frameSize: 1024, + sampleRate: 48000, + deps: { + createRnnoise: () => rnnoise, + createSpeex: () => speex, + }, + }); + + expect(selection.activeMode).toBe("speex"); + expect(selection.warnings).toHaveLength(1); + expect(rnnoise.destroy).toHaveBeenCalledTimes(1); + expect(speex.initialize).toHaveBeenCalledTimes(1); + }); + + it("falls back to disabled when both algorithms fail", async () => { + const disabled = createMockSuppressor(); + + const selection = await createNoiseSuppressorWithFallback({ + mode: "rnnoise", + frameSize: 1024, + sampleRate: 48000, + deps: { + createRnnoise: () => + createMockSuppressor({ + initialize: vi.fn().mockRejectedValue(new Error("rnnoise failed")), + }), + createSpeex: () => + createMockSuppressor({ + initialize: vi.fn().mockRejectedValue(new Error("speex failed")), + }), + createDisabled: () => disabled, + }, + }); + + expect(selection.activeMode).toBe("disabled"); + expect(selection.warnings).toHaveLength(2); + expect(disabled.initialize).toHaveBeenCalledTimes(1); + }); + + it("processes frames through the active suppressor and cleans up", async () => { + const frame = new Float32Array([0.1, -0.1]); + const processed = new Float32Array([0.05, -0.05]); + const speex = createMockSuppressor({ + processAudioFrame: vi.fn(() => processed), + }); + + const selection = await createNoiseSuppressorWithFallback({ + mode: "speex", + frameSize: 2, + sampleRate: 48000, + deps: { + createSpeex: () => speex, + }, + }); + + expect(selection.suppressor.processAudioFrame(frame)).toBe(processed); + selection.suppressor.destroy(); + expect(speex.destroy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/lib/audio/noiseSuppression.ts b/src/lib/audio/noiseSuppression.ts new file mode 100644 index 000000000..be69e0d52 --- /dev/null +++ b/src/lib/audio/noiseSuppression.ts @@ -0,0 +1,226 @@ +import { loadSpeexModule, SpeexPreprocessor } from "@sapphi-red/speex-preprocess-wasm"; +import { Rnnoise } from "@shiguredo/rnnoise-wasm"; + +export type NoiseSuppressionMode = "rnnoise" | "speex" | "disabled"; + +export interface NoiseSuppressor { + initialize(): Promise; + processAudioFrame(frame: Float32Array): Float32Array; + destroy(): void; +} + +export type NoiseSuppressionSelection = { + requestedMode: NoiseSuppressionMode; + activeMode: NoiseSuppressionMode; + suppressor: NoiseSuppressor; + warnings: string[]; +}; + +export const DEFAULT_NOISE_SUPPRESSION_MODE: NoiseSuppressionMode = "rnnoise"; + +export const NOISE_SUPPRESSION_MODES = new Set([ + "rnnoise", + "speex", + "disabled", +]); + +export function normalizeNoiseSuppressionMode(value?: string | null): NoiseSuppressionMode { + const normalized = value?.trim().toLowerCase(); + return normalized && NOISE_SUPPRESSION_MODES.has(normalized as NoiseSuppressionMode) + ? (normalized as NoiseSuppressionMode) + : DEFAULT_NOISE_SUPPRESSION_MODE; +} + +export class DisabledNoiseSuppressor implements NoiseSuppressor { + async initialize() { + console.info("[NoiseSuppression] Disabled."); + } + + processAudioFrame(frame: Float32Array) { + return frame; + } + + destroy() { + console.info("[NoiseSuppression] Disabled cleanup complete."); + } +} + +export class RnnoiseNoiseSuppressor implements NoiseSuppressor { + private rnnoise: Awaited> | null = null; + private state: ReturnType< + Awaited>["createDenoiseState"] + > | null = null; + private remainder: Float32Array = new Float32Array(0); + private queuedOutput: Float32Array = new Float32Array(0); + + async initialize() { + console.info("[NoiseSuppression] Initializing RNNoise."); + this.rnnoise = await Rnnoise.load(); + this.state = this.rnnoise.createDenoiseState(); + console.info("[NoiseSuppression] RNNoise initialized.", { + frameSize: this.rnnoise.frameSize, + }); + } + + processAudioFrame(frame: Float32Array) { + if (!this.rnnoise || !this.state) { + throw new Error("RNNoise suppressor has not been initialized."); + } + + const frameSize = this.rnnoise.frameSize; + const pendingInput = new Float32Array(this.remainder.length + frame.length); + pendingInput.set(this.remainder); + pendingInput.set(frame, this.remainder.length); + + const completeLength = pendingInput.length - (pendingInput.length % frameSize); + for (let offset = 0; offset < completeLength; offset += frameSize) { + this.state.processFrame(pendingInput.subarray(offset, offset + frameSize)); + } + + this.remainder = pendingInput.slice(completeLength); + this.queuedOutput = appendFloat32Arrays( + this.queuedOutput, + pendingInput.subarray(0, completeLength), + ); + + const output = new Float32Array(frame.length); + const outputLength = Math.min(output.length, this.queuedOutput.length); + if (outputLength > 0) { + output.set(this.queuedOutput.subarray(0, outputLength)); + this.queuedOutput = this.queuedOutput.slice(outputLength); + } + return output; + } + + destroy() { + this.state?.destroy(); + this.state = null; + this.rnnoise = null; + this.remainder = new Float32Array(0); + this.queuedOutput = new Float32Array(0); + console.info("[NoiseSuppression] RNNoise cleanup complete."); + } +} + +function appendFloat32Arrays(left: Float32Array, right: Float32Array) { + if (left.length === 0) { + return right.slice(); + } + if (right.length === 0) { + return left; + } + + const combined = new Float32Array(left.length + right.length); + combined.set(left); + combined.set(right, left.length); + return combined; +} + +export class SpeexNoiseSuppressor implements NoiseSuppressor { + private preprocessor: SpeexPreprocessor | null = null; + + constructor( + private readonly frameSize: number, + private readonly sampleRate: number, + ) {} + + async initialize() { + console.info("[NoiseSuppression] Initializing Speex.", { + frameSize: this.frameSize, + sampleRate: this.sampleRate, + }); + const module = await loadSpeexModule(); + this.preprocessor = new SpeexPreprocessor(module, this.frameSize, this.sampleRate); + this.preprocessor.denoise = true; + this.preprocessor.noiseSuppress = -24; + this.preprocessor.agc = false; + this.preprocessor.vad = false; + console.info("[NoiseSuppression] Speex initialized."); + } + + processAudioFrame(frame: Float32Array) { + if (!this.preprocessor) { + throw new Error("Speex suppressor has not been initialized."); + } + + if (frame.length === this.frameSize) { + this.preprocessor.process(frame); + return frame; + } + + for (let offset = 0; offset + this.frameSize <= frame.length; offset += this.frameSize) { + this.preprocessor.process(frame.subarray(offset, offset + this.frameSize)); + } + return frame; + } + + destroy() { + this.preprocessor?.destroy(); + this.preprocessor = null; + console.info("[NoiseSuppression] Speex cleanup complete."); + } +} + +export type NoiseSuppressorFactoryDeps = { + createRnnoise?: () => NoiseSuppressor; + createSpeex?: () => NoiseSuppressor; + createDisabled?: () => NoiseSuppressor; +}; + +export async function createNoiseSuppressorWithFallback({ + mode, + frameSize, + sampleRate, + deps = {}, +}: { + mode: NoiseSuppressionMode; + frameSize: number; + sampleRate: number; + deps?: NoiseSuppressorFactoryDeps; +}): Promise { + const warnings: string[] = []; + const createDisabled = deps.createDisabled ?? (() => new DisabledNoiseSuppressor()); + const candidates: NoiseSuppressionMode[] = + mode === "disabled" ? ["disabled"] : mode === "rnnoise" ? ["rnnoise", "speex"] : ["speex"]; + + for (const candidate of candidates) { + const suppressor = + candidate === "rnnoise" + ? (deps.createRnnoise?.() ?? new RnnoiseNoiseSuppressor()) + : candidate === "speex" + ? (deps.createSpeex?.() ?? new SpeexNoiseSuppressor(frameSize, sampleRate)) + : createDisabled(); + try { + await suppressor.initialize(); + if (candidate !== mode) { + console.warn("[NoiseSuppression] Falling back.", { + requestedMode: mode, + activeMode: candidate, + }); + } + return { + requestedMode: mode, + activeMode: candidate, + suppressor, + warnings, + }; + } catch (error) { + suppressor.destroy(); + const warning = `Failed to initialize ${candidate}: ${ + error instanceof Error ? error.message : String(error) + }`; + warnings.push(warning); + console.warn("[NoiseSuppression]", warning); + } + } + + const suppressor = createDisabled(); + await suppressor.initialize(); + console.warn("[NoiseSuppression] All algorithms failed; recording without noise suppression."); + return { + requestedMode: mode, + activeMode: "disabled", + suppressor, + warnings, + }; +} diff --git a/src/lib/audio/noiseSuppressionWorklet.ts b/src/lib/audio/noiseSuppressionWorklet.ts new file mode 100644 index 000000000..ea5e765a1 --- /dev/null +++ b/src/lib/audio/noiseSuppressionWorklet.ts @@ -0,0 +1,198 @@ +import { + createNoiseSuppressorWithFallback, + type NoiseSuppressionMode, + type NoiseSuppressionSelection, +} from "./noiseSuppression"; + +declare const AudioWorkletProcessor: { + new (): AudioWorkletProcessor; +}; +declare function registerProcessor(name: string, processorCtor: typeof AudioWorkletProcessor): void; +declare const currentTime: number; + +type AudioWorkletProcessor = { + readonly port: MessagePort; + process(inputs: Float32Array[][], outputs: Float32Array[][]): boolean; +}; + +type NoiseSuppressionWorkletMessage = + | { + type: "init"; + mode: NoiseSuppressionMode; + frameSize: number; + frameBudgetMs: number; + longFrameLogIntervalMs: number; + sampleRate: number; + } + | { type: "destroy" }; + +class NoiseSuppressionProcessor extends AudioWorkletProcessor { + private droppedFrames = 0; + private frameBudgetMs = 12; + private frameSize = 1024; + private inputBuffer = new Float32Array(1024); + private inputBufferOffset = 0; + private lastLongFrameLogAt = 0; + private longFrameLogIntervalMs = 5000; + private outputBuffer: Float32Array = new Float32Array(0); + private sampleRate = 48000; + private selection: NoiseSuppressionSelection | null = null; + private scratchFrame = new Float32Array(1024); + + constructor() { + super(); + this.port.onmessage = (event: MessageEvent) => { + if (event.data.type === "init") { + void this.initialize(event.data); + return; + } + this.destroy(); + }; + } + + private async initialize({ + mode, + frameSize, + frameBudgetMs, + longFrameLogIntervalMs, + sampleRate, + }: Extract) { + this.frameSize = frameSize; + this.frameBudgetMs = frameBudgetMs; + this.longFrameLogIntervalMs = longFrameLogIntervalMs; + this.sampleRate = sampleRate; + this.inputBuffer = new Float32Array(frameSize); + this.inputBufferOffset = 0; + this.outputBuffer = new Float32Array(0); + this.scratchFrame = new Float32Array(frameSize); + + try { + this.selection = await createNoiseSuppressorWithFallback({ + mode, + frameSize, + sampleRate, + }); + this.port.postMessage({ + type: "ready", + activeMode: this.selection.activeMode, + requestedMode: this.selection.requestedMode, + warnings: this.selection.warnings, + }); + } catch (error) { + this.port.postMessage({ + type: "error", + error: error instanceof Error ? error.message : String(error), + }); + } + } + + process(inputs: Float32Array[][], outputs: Float32Array[][]) { + const input = inputs[0]?.[0]; + const output = outputs[0]?.[0]; + if (!output) { + return true; + } + + if (!input || !this.selection) { + output.fill(0); + return true; + } + + let inputOffset = 0; + while (inputOffset < input.length) { + const writableLength = Math.min( + this.frameSize - this.inputBufferOffset, + input.length - inputOffset, + ); + this.inputBuffer.set( + input.subarray(inputOffset, inputOffset + writableLength), + this.inputBufferOffset, + ); + this.inputBufferOffset += writableLength; + inputOffset += writableLength; + + if (this.inputBufferOffset === this.frameSize) { + this.processBufferedFrame(); + this.inputBufferOffset = 0; + } + } + + const readableLength = Math.min(output.length, this.outputBuffer.length); + if (readableLength > 0) { + output.set(this.outputBuffer.subarray(0, readableLength)); + this.outputBuffer = this.outputBuffer.slice(readableLength); + } + if (readableLength < output.length) { + output.fill(0, readableLength); + } + + return true; + } + + private processBufferedFrame() { + if (!this.selection) { + return; + } + + const startedAt = getNowMs(); + try { + this.scratchFrame.set(this.inputBuffer); + this.outputBuffer = appendFloat32Arrays( + this.outputBuffer, + this.selection.suppressor.processAudioFrame(this.scratchFrame), + ); + } catch (error) { + this.droppedFrames += 1; + this.outputBuffer = appendFloat32Arrays(this.outputBuffer, this.inputBuffer); + this.port.postMessage({ + type: "processing-error", + activeMode: this.selection.activeMode, + droppedFrames: this.droppedFrames, + error: error instanceof Error ? error.message : String(error), + }); + } + + const now = getNowMs(); + const elapsedMs = now - startedAt; + if ( + elapsedMs > this.frameBudgetMs && + now - this.lastLongFrameLogAt > this.longFrameLogIntervalMs + ) { + this.lastLongFrameLogAt = now; + this.port.postMessage({ + type: "long-frame", + activeMode: this.selection.activeMode, + elapsedMs: Number(elapsedMs.toFixed(2)), + frameSize: this.frameSize, + sampleRate: this.sampleRate, + }); + } + } + + private destroy() { + this.selection?.suppressor.destroy(); + this.selection = null; + this.outputBuffer = new Float32Array(0); + this.inputBufferOffset = 0; + } +} + +function appendFloat32Arrays(left: Float32Array, right: Float32Array) { + if (left.length === 0) { + return right.slice(); + } + if (right.length === 0) { + return left; + } + + const combined = new Float32Array(left.length + right.length); + combined.set(left); + combined.set(right, left.length); + return combined; +} + +function getNowMs() { + return globalThis.performance?.now() ?? currentTime * 1000; +} + +registerProcessor("noise-suppression-processor", NoiseSuppressionProcessor);