diff --git a/CHANGELOG.md b/CHANGELOG.md index a5cce433..4cde1f8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to ### Added - ✨(back) add ODT parsing support +- ✨(feat) add audio prompting ### Changed diff --git a/src/backend/chat/exceptions.py b/src/backend/chat/exceptions.py new file mode 100644 index 00000000..f106ed0e --- /dev/null +++ b/src/backend/chat/exceptions.py @@ -0,0 +1,5 @@ +"""Exceptions related to chat.""" + + +class TranscriptionError(Exception): + """Raised when an error occurs during transcription.""" diff --git a/src/backend/chat/input/__init__.py b/src/backend/chat/input/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/chat/input/albert_audio_backend.py b/src/backend/chat/input/albert_audio_backend.py new file mode 100644 index 00000000..8ee62e10 --- /dev/null +++ b/src/backend/chat/input/albert_audio_backend.py @@ -0,0 +1,63 @@ +"""Albert API implementation for audio transcription.""" + +import logging +from typing import IO +from urllib.parse import urljoin + +from django.conf import settings + +import requests + +from chat.exceptions import TranscriptionError +from chat.input.base_audio_backend import BaseAudioBackend + +logger = logging.getLogger(__name__) + + +class AlbertAudioBackend(BaseAudioBackend): + """ + Audio transcription backend using the Albert API. + + Sends audio files to the Albert ASR endpoint and returns the transcribed text. + Requires ALBERT_API_URL and ALBERT_API_KEY to be configured in settings. + """ + + def __init__(self): + self._transcriptions_endpoint = urljoin(settings.ALBERT_API_URL, "/v1/audio/transcriptions") + self._headers = {"Authorization": f"Bearer {settings.ALBERT_API_KEY}"} + + def transcribe(self, file_name: str, file_content: IO[bytes], content_type: str) -> str: + """ + Transcribe the given audio file using the Albert ASR API. + + Args: + file_name (str): The original file name of the audio file. + file_content (IO[bytes]): A file-like object with the audio content. + content_type (str): The MIME type of the audio file. + + Returns: + str: The transcribed text. + + Raises: + TranscriptionError: If an error occurs during transcription. + """ + try: + response = requests.post( + self._transcriptions_endpoint, + headers=self._headers, + files={"file": (file_name, file_content, content_type)}, + # whisper on vLLM does not support language detection yet, + # so we need to pass the language parameter to Albert API. + # We should allow passing 'auto' when whisper once vLLM + # supports language detection, or when Albert API migrates + # to whisperx. + data={ + "model": settings.ALBERT_API_ASR_MODEL, + "language": settings.ALBERT_API_ASR_LANGUAGE, + }, + timeout=settings.ALBERT_API_TIMEOUT, + ) + response.raise_for_status() + return response.json().get("text", "").strip() + except requests.RequestException as e: + raise TranscriptionError(f"Error during transcription: {e}") from e diff --git a/src/backend/chat/input/base_audio_backend.py b/src/backend/chat/input/base_audio_backend.py new file mode 100644 index 00000000..0642de65 --- /dev/null +++ b/src/backend/chat/input/base_audio_backend.py @@ -0,0 +1,25 @@ +"""Base class for audio input backends.""" + +from abc import ABC, abstractmethod +from typing import IO + + +class BaseAudioBackend(ABC): + """Abstract base class for audio transcription backends.""" + + @abstractmethod + def transcribe(self, file_name: str, file_content: IO[bytes], content_type: str) -> str: + """ + Transcribe the given audio file to text. + + Args: + file_name (str): The original file name of the audio file. + file_content (IO[bytes]): A file-like object with the audio content. + content_type (str): The MIME type of the audio file. + + Returns: + str: The transcribed text. + + Raises: + TranscriptionError: If an error occurs during transcription. + """ diff --git a/src/backend/chat/tests/input/__init__.py b/src/backend/chat/tests/input/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backend/chat/tests/input/test_albert_audio_backend.py b/src/backend/chat/tests/input/test_albert_audio_backend.py new file mode 100644 index 00000000..530e16af --- /dev/null +++ b/src/backend/chat/tests/input/test_albert_audio_backend.py @@ -0,0 +1,63 @@ +"""Tests for the Albert audio transcription backend.""" + +import pytest +import responses + +from chat.exceptions import TranscriptionError +from chat.input.albert_audio_backend import AlbertAudioBackend + + +@pytest.fixture(name="backend") +def backend_fixture(settings): + """Fixture providing a configured AlbertAudioBackend instance.""" + settings.ALBERT_API_URL = "https://albert.example.com" + settings.ALBERT_API_KEY = "test-key" + settings.ALBERT_API_TIMEOUT = 10 + settings.ALBERT_API_ASR_MODEL = "openweight-audio" + return AlbertAudioBackend() + + +@responses.activate +def test_transcribe_strips_whitespace(backend): + """Transcription result has surrounding whitespace stripped.""" + responses.add( + responses.POST, + "https://albert.example.com/v1/audio/transcriptions", + json={"text": " bonjour monde "}, + status=200, + ) + + result = backend.transcribe("audio.webm", b"data", "audio/webm") + + assert result == "bonjour monde" + + +@responses.activate +def test_transcribe_raises_transcription_error_on_http_failure(backend): + """An HTTP error from the Albert API raises TranscriptionError.""" + responses.add( + responses.POST, + "https://albert.example.com/v1/audio/transcriptions", + status=503, + ) + + with pytest.raises(TranscriptionError): + backend.transcribe("audio.webm", b"data", "audio/webm") + + +@responses.activate +def test_transcribe_sends_correct_request(backend): + """The backend sends the file and required fields to the Albert API.""" + responses.add( + responses.POST, + "https://albert.example.com/v1/audio/transcriptions", + json={"text": "test"}, + status=200, + ) + + backend.transcribe("recording.webm", b"audio-content", "audio/webm") + + assert len(responses.calls) == 1 + request = responses.calls[0].request + assert "Bearer test-key" in request.headers["Authorization"] + assert b"audio-content" in request.body diff --git a/src/backend/chat/tests/views/test_audio_transcription.py b/src/backend/chat/tests/views/test_audio_transcription.py new file mode 100644 index 00000000..bcac918f --- /dev/null +++ b/src/backend/chat/tests/views/test_audio_transcription.py @@ -0,0 +1,109 @@ +"""Tests for the audio transcription view.""" + +from io import BytesIO + +import pytest +import responses + +from core.factories import UserFactory + +pytestmark = pytest.mark.django_db + +AUDIO_BYTES = b"RIFF....WAVEfmt " + + +@responses.activate +def test_audio_transcription_success(api_client, settings): + """Test successful audio transcription returns stripped text.""" + settings.AUDIO_TRANSCRIPTION_BACKEND = "chat.input.albert_audio_backend.AlbertAudioBackend" + settings.ALBERT_API_URL = "https://albert.example.com" + settings.ALBERT_API_KEY = "test-key" + settings.ALBERT_API_TIMEOUT = 10 + settings.ALBERT_API_ASR_MODEL = "openweight-audio" + + responses.add( + responses.POST, + "https://albert.example.com/v1/audio/transcriptions", + json={"text": " hello world"}, + status=200, + ) + + user = UserFactory() + api_client.force_login(user) + response = api_client.post( + "/api/v1.0/transcribe/", + {"audio": BytesIO(AUDIO_BYTES)}, + format="multipart", + ) + + assert response.status_code == 200 + assert response.json() == {"text": "hello world"} + + +def test_audio_transcription_anonymous(api_client, settings): + """Anonymous users cannot access the transcription endpoint.""" + settings.AUDIO_TRANSCRIPTION_BACKEND = "chat.input.albert_audio_backend.AlbertAudioBackend" + + response = api_client.post( + "/api/v1.0/transcribe/", + {"audio": BytesIO(AUDIO_BYTES)}, + format="multipart", + ) + + assert response.status_code == 401 + + +def test_audio_transcription_backend_disabled(api_client, settings): + """Returns 501 when no transcription backend is configured.""" + settings.AUDIO_TRANSCRIPTION_BACKEND = None + + user = UserFactory() + api_client.force_login(user) + response = api_client.post( + "/api/v1.0/transcribe/", + {"audio": BytesIO(AUDIO_BYTES)}, + format="multipart", + ) + + assert response.status_code == 501 + + +def test_audio_transcription_no_file(api_client, settings): + """Returns 400 when no audio file is provided.""" + settings.AUDIO_TRANSCRIPTION_BACKEND = "chat.input.albert_audio_backend.AlbertAudioBackend" + settings.ALBERT_API_URL = "https://albert.example.com" + settings.ALBERT_API_KEY = "test-key" + settings.ALBERT_API_TIMEOUT = 10 + settings.ALBERT_API_ASR_MODEL = "openweight-audio" + + user = UserFactory() + api_client.force_login(user) + response = api_client.post("/api/v1.0/transcribe/", {}, format="multipart") + + assert response.status_code == 400 + + +@responses.activate +def test_audio_transcription_backend_error(api_client, settings): + """Returns 502 when the transcription backend fails.""" + settings.AUDIO_TRANSCRIPTION_BACKEND = "chat.input.albert_audio_backend.AlbertAudioBackend" + settings.ALBERT_API_URL = "https://albert.example.com" + settings.ALBERT_API_KEY = "test-key" + settings.ALBERT_API_TIMEOUT = 10 + settings.ALBERT_API_ASR_MODEL = "openweight-audio" + + responses.add( + responses.POST, + "https://albert.example.com/v1/audio/transcriptions", + status=500, + ) + + user = UserFactory() + api_client.force_login(user) + response = api_client.post( + "/api/v1.0/transcribe/", + {"audio": BytesIO(AUDIO_BYTES)}, + format="multipart", + ) + + assert response.status_code == 502 diff --git a/src/backend/chat/views.py b/src/backend/chat/views.py index 17297423..2b5b28ec 100644 --- a/src/backend/chat/views.py +++ b/src/backend/chat/views.py @@ -10,6 +10,7 @@ from django.db.models import Prefetch from django.http import Http404, StreamingHttpResponse from django.utils.decorators import method_decorator +from django.utils.module_loading import import_string import langfuse import magic @@ -19,6 +20,7 @@ from lasuite.oidc_login.decorators import refresh_oidc_access_token from rest_framework import decorators, filters, mixins, permissions, status, viewsets from rest_framework.exceptions import MethodNotAllowed, PermissionDenied, ValidationError +from rest_framework.parsers import MultiPartParser from rest_framework.response import Response from rest_framework.throttling import ScopedRateThrottle from rest_framework.views import APIView @@ -33,6 +35,7 @@ from activation_codes.permissions import IsActivatedUser from chat import models, serializers from chat.clients.pydantic_ai import AIAgentService +from chat.exceptions import TranscriptionError from chat.keepalive import stream_with_keepalive_async, stream_with_keepalive_sync from chat.serializers import ChatConversationRequestSerializer @@ -741,3 +744,54 @@ def perform_destroy(self, instance): """ instance.conversations.all().delete() instance.delete() + + +class AudioTranscriptionView(APIView): + """Transcribe audio using a configurable ASR backend. + + Accepts a multipart/form-data POST with an ``audio`` file field and returns + ``{"text": ""}``. + + Requires ``AUDIO_TRANSCRIPTION_BACKEND`` to be set in settings. + """ + + permission_classes = [ + IsActivatedUser, + permissions.IsAuthenticated, + ] + parser_classes = [MultiPartParser] + throttle_classes = [ScopedRateThrottle] + throttle_scope = "transcribe" + + def post(self, request): + """Handle POST requests to transcribe audio.""" + if not settings.AUDIO_TRANSCRIPTION_BACKEND: + return Response( + {"error": "Audio transcription is not available"}, + status=status.HTTP_501_NOT_IMPLEMENTED, + ) + + audio_file = request.FILES.get("audio") + if not audio_file: + return Response({"error": "No audio file provided"}, status=status.HTTP_400_BAD_REQUEST) + + if audio_file.size > settings.AUDIO_MAX_SIZE: + return Response( + {"error": "Audio file too large"}, + status=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + ) + + backend = import_string(settings.AUDIO_TRANSCRIPTION_BACKEND)() + try: + text = backend.transcribe( + audio_file.name, + audio_file, + audio_file.content_type, + ) + except TranscriptionError: + return Response( + {"error": "Transcription failed"}, + status=status.HTTP_502_BAD_GATEWAY, + ) + + return Response({"text": text}, status=status.HTTP_200_OK) diff --git a/src/backend/conversations/settings.py b/src/backend/conversations/settings.py index 970ea1eb..3f5a8150 100755 --- a/src/backend/conversations/settings.py +++ b/src/backend/conversations/settings.py @@ -422,6 +422,11 @@ class Base(BraveSettings, Configuration): environ_name="API_FILE_STREAM_THROTTLE_RATE", environ_prefix=None, ), + "transcribe": values.Value( + default="30/minute", + environ_name="API_TRANSCRIBE_THROTTLE_RATE", + environ_prefix=None, + ), }, } @@ -750,6 +755,16 @@ class Base(BraveSettings, Configuration): environ_name="RAG_DOCUMENT_SEARCH_BACKEND", environ_prefix=None, ) + AUDIO_TRANSCRIPTION_BACKEND = values.Value( + None, + environ_name="AUDIO_TRANSCRIPTION_BACKEND", + environ_prefix=None, + ) + AUDIO_MAX_SIZE = values.PositiveIntegerValue( + 512 * 1024 * 1024, # 512 MB + environ_name="MAX_AUDIO_UPLOAD_BYTES", + environ_prefix=None, + ) RAG_DOCUMENT_PARSER = values.Value( "chat.agent_rag.document_converter.parser.AlbertParser", environ_name="RAG_DOCUMENT_PARSER", @@ -924,6 +939,16 @@ class Base(BraveSettings, Configuration): environ_name="ALBERT_API_TIMEOUT", environ_prefix=None, ) + ALBERT_API_ASR_MODEL = values.Value( + "openweight-audio", # Default ASR model for Albert API + environ_name="ALBERT_API_ASR_MODEL", + environ_prefix=None, + ) + ALBERT_API_ASR_LANGUAGE = values.Value( + "fr", # Default ASR language for Albert API + environ_name="ALBERT_API_ASR_LANGUAGE", + environ_prefix=None, + ) # Find FIND_API_KEY = values.Value( diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index 5d4b8f99..1bb92691 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -225,6 +225,8 @@ def get(self, request): dict_settings["chat_upload_accept"] = ",".join(settings.RAG_FILES_ACCEPTED_FORMATS) + dict_settings["audio_transcription_enabled"] = bool(settings.AUDIO_TRANSCRIPTION_BACKEND) + return drf.response.Response(dict_settings) def _load_theme_customization(self): diff --git a/src/backend/core/tests/test_api_config.py b/src/backend/core/tests/test_api_config.py index 0571e513..118650b1 100644 --- a/src/backend/core/tests/test_api_config.py +++ b/src/backend/core/tests/test_api_config.py @@ -64,6 +64,7 @@ def test_api_config(is_authenticated): "SENTRY_DSN": "https://sentry.test/123", "theme_customization": {}, "chat_upload_accept": "application/pdf,text/plain", + "audio_transcription_enabled": False, } @@ -207,4 +208,5 @@ async def test_api_config_async(is_authenticated): "SENTRY_DSN": "https://sentry.test/123", "theme_customization": {}, "chat_upload_accept": "application/pdf,text/plain", + "audio_transcription_enabled": False, } diff --git a/src/backend/core/urls.py b/src/backend/core/urls.py index acf88627..6b0a0ffa 100644 --- a/src/backend/core/urls.py +++ b/src/backend/core/urls.py @@ -11,6 +11,7 @@ from activation_codes import viewsets as activation_viewsets from chat.views import ( + AudioTranscriptionView, ChatConversationAttachmentViewSet, ChatProjectViewSet, ChatViewSet, @@ -40,6 +41,7 @@ path( "llm-configuration/", LLMConfigurationView.as_view(), name="llm-configuration" ), + path("transcribe/", AudioTranscriptionView.as_view(), name="audio-transcription"), path( "chats//", include(conversation_router.urls), diff --git a/src/frontend/apps/conversations/next-env.d.ts b/src/frontend/apps/conversations/next-env.d.ts index 7996d352..19709046 100644 --- a/src/frontend/apps/conversations/next-env.d.ts +++ b/src/frontend/apps/conversations/next-env.d.ts @@ -1,6 +1,6 @@ /// /// -import "./.next/dev/types/routes.d.ts"; +import "./.next/types/routes.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/pages/api-reference/config/typescript for more information. diff --git a/src/frontend/apps/conversations/src/core/config/api/useConfig.tsx b/src/frontend/apps/conversations/src/core/config/api/useConfig.tsx index 5e2c8015..b42545ed 100644 --- a/src/frontend/apps/conversations/src/core/config/api/useConfig.tsx +++ b/src/frontend/apps/conversations/src/core/config/api/useConfig.tsx @@ -37,6 +37,7 @@ export interface ConfigResponse { FILE_UPLOAD_MODE?: string; theme_customization?: ThemeCustomization; chat_upload_accept?: string; + audio_transcription_enabled?: boolean; } const LOCAL_STORAGE_KEY = 'conversations_config'; diff --git a/src/frontend/apps/conversations/src/features/chat/components/AudioRecordButton.tsx b/src/frontend/apps/conversations/src/features/chat/components/AudioRecordButton.tsx new file mode 100644 index 00000000..e42e9968 --- /dev/null +++ b/src/frontend/apps/conversations/src/features/chat/components/AudioRecordButton.tsx @@ -0,0 +1,177 @@ +import { Button } from '@gouvfr-lasuite/cunningham-react'; +import React, { useEffect, useRef, useState } from 'react'; +import { useTranslation } from 'react-i18next'; + +import { Box, Icon } from '@/components'; + +import { RecordingState } from '../hooks/useAudioRecording'; + +interface AudioRecordButtonProps { + disabled?: boolean; + recordingState: RecordingState; + onStartRecording: () => void; + onConfirm: () => void; + onCancel: () => void; + volume: number; +} + +const BAR_COUNT = 40; +const SAMPLE_INTERVAL_MS = 80; + +const SPINNER_CSS = ` + width: 20px; + height: 20px; + border: 2px solid var(--c--contextuals--border--surface--primary); + border-top-color: var(--c--contextuals--content--semantic--brand--primary); + border-radius: 50%; + animation: audio-spin 0.7s linear infinite; + + @keyframes audio-spin { + to { transform: rotate(360deg); } + } +`; + +const RECORDING_BAR_CSS = ` + padding: 0 1rem; + min-height: 14px; +`; + +const WAVEFORM_WRAPPER_STYLE: React.CSSProperties = { + flex: 1, + display: 'flex', + alignItems: 'center', + gap: '2px', + overflow: 'hidden', + height: '36px', + padding: '0 0.5rem', +}; + +export const AudioRecordButton = ({ + recordingState, + onStartRecording, + onConfirm, + onCancel, + volume, + disabled, +}: AudioRecordButtonProps) => { + const { t } = useTranslation(); + + if (recordingState === 'idle') { + return ( + - - {/* Web search toggle button */} - {onWebSearchToggle && ( - - + + {/* Web search toggle button */} + {onWebSearchToggle && ( + + - - )} - - - {/* Right side: Model selector + Send */} - - {onModelSelect && ( - - + + {t('Web')} + + + + )} + + + )} + + )} + + {/* Right side: Model selector + Send (hidden while recording) */} + {!isRecording && ( + + {onModelSelect && ( + + + + )} + + {/* Audio record button + Send, with extra gap between them */} + + {audioTranscriptionEnabled && ( + + )} + + - )} - - - + + )} ); }, diff --git a/src/frontend/apps/conversations/src/features/chat/components/__tests__/InputChatAction.test.tsx b/src/frontend/apps/conversations/src/features/chat/components/__tests__/InputChatAction.test.tsx deleted file mode 100644 index 90059079..00000000 --- a/src/frontend/apps/conversations/src/features/chat/components/__tests__/InputChatAction.test.tsx +++ /dev/null @@ -1,217 +0,0 @@ -import { render, screen } from '@testing-library/react'; -import userEvent from '@testing-library/user-event'; - -import '@/i18n/initI18n'; - -import { InputChatActions } from '../InputChatAction'; - -jest.mock('../ModelSelector', () => ({ - ModelSelector: ({ onModelSelect }: { onModelSelect: () => void }) => ( - - ), -})); - -jest.mock('../SendButton', () => ({ - SendButton: ({ - onClick, - disabled, - status, - }: { - onClick: () => void; - disabled: boolean; - status: string | null; - }) => ( - - ), -})); - -const defaultProps = { - fileUploadEnabled: true, - webSearchEnabled: true, - isUploadingFiles: false, - isMobile: false, - forceWebSearch: false, - onAttachClick: jest.fn(), - selectedModel: null, - status: null, - inputHasContent: true, -}; - -describe('InputChatActions', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('should render attach file button', () => { - render(); - - expect( - screen.getByRole('button', { name: 'Add attach file' }), - ).toBeInTheDocument(); - expect(screen.getByText('Attach file')).toBeInTheDocument(); - }); - - it('should call onAttachClick when attach button is clicked', async () => { - const user = userEvent.setup(); - const onAttachClick = jest.fn(); - render( - , - ); - - await user.click(screen.getByRole('button', { name: 'Add attach file' })); - - expect(onAttachClick).toHaveBeenCalledTimes(1); - }); - - it('should disable attach button when fileUploadEnabled is false', () => { - render(); - - expect( - screen.getByRole('button', { name: 'Add attach file' }), - ).toBeDisabled(); - }); - - it('should disable attach button when isUploadingFiles is true', () => { - render(); - - expect( - screen.getByRole('button', { name: 'Add attach file' }), - ).toBeDisabled(); - }); - - it('should not show attach text on mobile', () => { - render(); - - expect(screen.queryByText('Attach file')).not.toBeInTheDocument(); - }); - - it('should render web search button when onWebSearchToggle is provided', () => { - const onWebSearchToggle = jest.fn(); - render( - , - ); - - expect( - screen.getByRole('button', { name: 'Research on the web' }), - ).toBeInTheDocument(); - }); - - it('should not render web search button when onWebSearchToggle is undefined', () => { - render( - , - ); - - expect( - screen.queryByRole('button', { name: 'Research on the web' }), - ).not.toBeInTheDocument(); - }); - - it('should call onWebSearchToggle when web search button is clicked', async () => { - const user = userEvent.setup(); - const onWebSearchToggle = jest.fn(); - render( - , - ); - - await user.click( - screen.getByRole('button', { name: 'Research on the web' }), - ); - - expect(onWebSearchToggle).toHaveBeenCalledTimes(1); - }); - - it('should disable web search button when webSearchEnabled is false', () => { - render( - , - ); - - expect( - screen.getByRole('button', { name: 'Research on the web' }), - ).toBeDisabled(); - }); - - it('should render model selector when onModelSelect is provided', () => { - const onModelSelect = jest.fn(); - render( - , - ); - - expect(screen.getByTestId('model-selector')).toBeInTheDocument(); - }); - - it('should not render model selector when onModelSelect is undefined', () => { - render(); - - expect(screen.queryByTestId('model-selector')).not.toBeInTheDocument(); - }); - - it('should render send button', () => { - render(); - - expect(screen.getByTestId('send-button')).toBeInTheDocument(); - }); - - it('should pass streaming status to SendButton', () => { - render(); - - expect(screen.getByTestId('send-button')).toHaveAttribute( - 'data-status', - 'streaming', - ); - }); - - it('should pass submitted status to SendButton', () => { - render(); - - expect(screen.getByTestId('send-button')).toHaveAttribute( - 'data-status', - 'submitted', - ); - }); - - it('should show "Web" text on mobile when forceWebSearch is active', () => { - render( - , - ); - - expect(screen.getByText('Web')).toBeInTheDocument(); - }); - - it('should show "Research on the web" text on desktop when forceWebSearch is active', () => { - render( - , - ); - - expect(screen.getByText('Research on the web')).toBeInTheDocument(); - expect(screen.queryByText('Web')).not.toBeInTheDocument(); - }); -}); diff --git a/src/frontend/apps/conversations/src/features/chat/hooks/useAudioRecording.tsx b/src/frontend/apps/conversations/src/features/chat/hooks/useAudioRecording.tsx new file mode 100644 index 00000000..0678008c --- /dev/null +++ b/src/frontend/apps/conversations/src/features/chat/hooks/useAudioRecording.tsx @@ -0,0 +1,140 @@ +import { useCallback, useRef, useState } from 'react'; + +import { fetchAPI } from '@/api'; + +import { TranscribeAudioResponse } from '../types'; + +export type RecordingState = 'idle' | 'recording' | 'transcribing'; + +interface UseAudioRecordingOptions { + onTranscription: (text: string) => void; + onTranscriptionError?: () => void; +} + +export const useAudioRecording = ({ + onTranscription, + onTranscriptionError, +}: UseAudioRecordingOptions) => { + const [recordingState, setRecordingState] = useState('idle'); + const [volume, setVolume] = useState(0); + + const onTranscriptionRef = useRef(onTranscription); + onTranscriptionRef.current = onTranscription; + + const onTranscriptionErrorRef = useRef(onTranscriptionError); + onTranscriptionErrorRef.current = onTranscriptionError; + + const mediaRecorderRef = useRef(null); + const audioChunksRef = useRef([]); + const streamRef = useRef(null); + const animFrameRef = useRef(0); + const audioCtxRef = useRef(null); + + const stopVisualization = useCallback(() => { + cancelAnimationFrame(animFrameRef.current); + void audioCtxRef.current?.close(); + audioCtxRef.current = null; + setVolume(0); + }, []); + + const startVisualization = useCallback((stream: MediaStream) => { + const audioCtx = new AudioContext(); + audioCtxRef.current = audioCtx; + const analyser = audioCtx.createAnalyser(); + analyser.fftSize = 1024; + audioCtx.createMediaStreamSource(stream).connect(analyser); + const dataArray = new Uint8Array(analyser.fftSize); + + const tick = () => { + analyser.getByteTimeDomainData(dataArray); + // RMS amplitude: 0 = silence, 1 = full scale + const rms = + Math.sqrt( + dataArray.reduce((sum, v) => sum + (v - 128) ** 2, 0) / + dataArray.length, + ) / 128; + setVolume(rms); + animFrameRef.current = requestAnimationFrame(tick); + }; + tick(); + }, []); + + const stopStream = useCallback(() => { + streamRef.current?.getTracks().forEach((t) => t.stop()); + streamRef.current = null; + stopVisualization(); + }, [stopVisualization]); + + const startRecording = useCallback(async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + streamRef.current = stream; + const mediaRecorder = new MediaRecorder(stream); + audioChunksRef.current = []; + mediaRecorder.ondataavailable = (e) => { + if (e.data.size > 0) audioChunksRef.current.push(e.data); + }; + mediaRecorder.start(); + mediaRecorderRef.current = mediaRecorder; + startVisualization(stream); + setRecordingState('recording'); + } catch { + stopStream(); + setRecordingState('idle'); + onTranscriptionErrorRef.current?.(); + } + }, [startVisualization, stopStream]); + + const confirmRecording = useCallback(() => { + if (mediaRecorderRef.current) { + const recorder = mediaRecorderRef.current; + mediaRecorderRef.current = null; + + recorder.onstop = async () => { + stopStream(); + setRecordingState('transcribing'); + + const mimeType = recorder.mimeType || 'audio/webm'; + const ext = mimeType.includes('mp4') ? 'mp4' : 'webm'; + const blob = new Blob(audioChunksRef.current, { type: mimeType }); + const form = new FormData(); + form.append('audio', blob, `recording.${ext}`); + + try { + const res = await fetchAPI('transcribe/', { + method: 'POST', + body: form, + withoutContentType: true, + }); + const data = (await res.json()) as TranscribeAudioResponse; + if ('error' in data) onTranscriptionErrorRef.current?.(); + if ('text' in data) onTranscriptionRef.current(data.text); + } catch { + onTranscriptionErrorRef.current?.(); + } finally { + setRecordingState('idle'); + } + }; + + recorder.stop(); + } + }, [stopStream]); + + const cancelRecording = useCallback(() => { + if (mediaRecorderRef.current) { + mediaRecorderRef.current.onstop = null; + mediaRecorderRef.current.stop(); + mediaRecorderRef.current = null; + } + stopStream(); + setRecordingState('idle'); + }, [stopStream]); + + return { + recordingState, + volume, + startRecording, + confirmRecording, + cancelRecording, + }; +}; diff --git a/src/frontend/apps/conversations/src/features/chat/types.tsx b/src/frontend/apps/conversations/src/features/chat/types.tsx index 10eb862f..4dbb6ed0 100644 --- a/src/frontend/apps/conversations/src/features/chat/types.tsx +++ b/src/frontend/apps/conversations/src/features/chat/types.tsx @@ -31,3 +31,11 @@ export interface ChatProject { llm_instructions: string; conversations: ChatProjectConversation[]; } + +export type TranscribeAudioResponse = + | { + text: string; + } + | { + error: string; + }; diff --git a/src/frontend/apps/conversations/src/features/header/components/ButtonToggleLeftPanel.tsx b/src/frontend/apps/conversations/src/features/header/components/ButtonToggleLeftPanel.tsx index 7362d7df..e3bc0114 100644 --- a/src/frontend/apps/conversations/src/features/header/components/ButtonToggleLeftPanel.tsx +++ b/src/frontend/apps/conversations/src/features/header/components/ButtonToggleLeftPanel.tsx @@ -9,8 +9,8 @@ import LeftPanelIcon from '@/assets/icons/left-panel-bold.svg'; import { Box, Text } from '@/components'; import { useCunninghamTheme } from '@/cunningham'; import { - getConversation, KEY_CONVERSATION, + getConversation, } from '@/features/chat/api/useConversation'; import { useInfiniteProjects } from '@/features/chat/api/useProjects'; import { useChatPreferencesStore } from '@/features/chat/stores/useChatPreferencesStore'; diff --git a/src/frontend/apps/conversations/src/features/header/components/Header.tsx b/src/frontend/apps/conversations/src/features/header/components/Header.tsx index 51b7960d..6860110b 100644 --- a/src/frontend/apps/conversations/src/features/header/components/Header.tsx +++ b/src/frontend/apps/conversations/src/features/header/components/Header.tsx @@ -9,7 +9,6 @@ import NewChatIcon from '@/assets/icons/new-message-bold.svg'; import LogoAssistant from '@/assets/logo/logo-beta.svg'; import { Box } from '@/components/'; import { useCunninghamTheme } from '@/cunningham'; - import { useChatScroll } from '@/features/chat/hooks'; import { useChatPreferencesStore } from '@/features/chat/stores/useChatPreferencesStore'; import { useResponsiveStore } from '@/stores'; diff --git a/src/frontend/apps/conversations/src/features/language/components/LanguagePicker.tsx b/src/frontend/apps/conversations/src/features/language/components/LanguagePicker.tsx index a3d26b8d..27f13f5f 100644 --- a/src/frontend/apps/conversations/src/features/language/components/LanguagePicker.tsx +++ b/src/frontend/apps/conversations/src/features/language/components/LanguagePicker.tsx @@ -1,6 +1,7 @@ import { LanguagePicker as LanguagePickerUi } from '@gouvfr-lasuite/ui-kit'; import { useEffect, useMemo } from 'react'; import { useTranslation } from 'react-i18next'; + import { useConfig } from '@/core'; import { useAuth } from '@/features/auth/hooks'; diff --git a/src/frontend/apps/conversations/src/i18n/translations.json b/src/frontend/apps/conversations/src/i18n/translations.json index e75a9f7e..9d01f37e 100644 --- a/src/frontend/apps/conversations/src/i18n/translations.json +++ b/src/frontend/apps/conversations/src/i18n/translations.json @@ -1,6 +1,21 @@ { - "de": { "translation": { "ABC-1234-XY": "ABC-1234-XY" } }, - "en": { "translation": { "Login": "Login", "Logout": "Logout" } }, + "de": { + "translation": { + "ABC-1234-XY": "ABC-1234-XY", + "Cancel recording": "Aufnahme abbrechen", + "Confirm recording": "Aufnahme bestätigen", + "Record audio": "Audio aufnehmen" + } + }, + "en": { + "translation": { + "Cancel recording": "Cancel recording", + "Confirm recording": "Confirm recording", + "Login": "Login", + "Logout": "Logout", + "Record audio": "Record audio" + } + }, "fr": { "translation": { "30 sec to tell us what you think or report a bug": "Prenez 30 secondes pour partager votre avis ou signaler un bug", @@ -27,6 +42,9 @@ "Assistant is in development: your feedback matters! Choose how to share your ideas:": "L'assistant est en cours de développement : vos commentaires sont importants ! Choisissez comment partager vos avis :", "Assistant settings": "Paramètres de l'Assistant", "Attach file": "Joindre un fichier", + "Record audio": "Enregistrer un message vocal", + "Cancel recording": "Annuler l'enregistrement", + "Confirm recording": "Valider l'enregistrement", "Attachment summary not supported": "Résumé des pièces jointes non pris en charge", "Automatic web search": "Recherche web automatique", "Avatar and name": "Avatar et nom", @@ -65,6 +83,7 @@ "Failed to send feedback": "Échec de l’envoi du commentaire", "Failed to update settings": "Impossible de mettre à jour les paramètres", "Failed to upload file": "Impossible de téléverser le fichier", + "An error occurred during transcription. Please try again.": "Il semble y avoir eu une erreur dans la transcription. Merci de réessayer.", "Failed to upload files. Please try again.": "Le téléversement a échoué. Veuillez réessayer.", "Feedback Négatif": "Retour négatif", "Feedback positif": "Retour positif", @@ -195,11 +214,13 @@ "Attachment summary not supported": "Bijlageoverzicht niet ondersteund", "Automatic web search": "Automatisch webzoeken", "Cancel": "Annuleren", + "Cancel recording": "Opname annuleren", "Close model selector": "Sluit modelselector", "Close the left panel": "Sluit het linker venster", "Close the menu": "Sluit het menu", "Close the modal": "Sluit het venster", "Confirm deletion": "Verwijdering bevestigen", + "Confirm recording": "Opname bevestigen", "Content modal to delete conversation": "Inhoudsvenster om conversatie te verwijderen", "Content modal to rename a conversation": "Content modal om een gesprek te hernoemen", "Copied": "Gekopieerd", @@ -220,6 +241,7 @@ "Failed to send feedback": "Het is niet gelukt om feedback te verzenden", "Failed to update settings": "Het is niet gelukt om de instellingen bij te werken", "Failed to upload file": "Het uploaden van het bestand is mislukt", + "An error occurred during transcription. Please try again.": "Er is een fout opgetreden bij de transcriptie. Probeer het opnieuw.", "Failed to upload files. Please try again.": "Bestanden uploaden is mislukt. Probeer het opnieuw.", "Feedback Négatif": "Negatieve feedback", "Feedback positif": "Positieve feedback", @@ -255,6 +277,7 @@ "Please enter an activation code": "Voer een activeringscode in", "Proconnect Login": "Login", "Quick search input": "Snelle zoekinvoer", + "Record audio": "Audio opnemen", "Remove attachment": "Bijlage verwijderen", "Rename": "Hernoem", "Rename chat": "Chat hernoemen", @@ -337,12 +360,14 @@ "Automatic web search": "Автопоиск в Интернете", "Avatar and name": "Аватар и имя", "Cancel": "Отмена", + "Cancel recording": "Отменить запись", "Choose icon and color": "Выберите значок и цвет", "Close model selector": "Закрыть выбор модели", "Close the left panel": "Закрыть левую панель", "Close the menu": "Закрыть меню", "Close the modal": "Закрыть это окно", "Confirm deletion": "Подтвердите удаление", + "Confirm recording": "Подтвердить запись", "Content modal to create a project": "Создание проекта", "Content modal to delete conversation": "Подтверждение удаления беседы", "Content modal to delete project": "Подтверждение удаления проекта", @@ -371,6 +396,7 @@ "Failed to send feedback": "Не удалось отправить отзыв", "Failed to update settings": "Не удалось обновить настройки", "Failed to upload file": "Не удалось выгрузить файл", + "An error occurred during transcription. Please try again.": "При транскрибировании произошла ошибка. Пожалуйста, попробуйте ещё раз.", "Failed to upload files. Please try again.": "Не удалось выгрузить файлы. Повторите попытку.", "Feedback Négatif": "Отрицательный отзыв", "Feedback positif": "Положительный отзыв", @@ -413,6 +439,7 @@ "Projects": "Проекты", "Quick search input": "Быстрый поиск", "Reading project instructions...": "Чтение инструкций проекта...", + "Record audio": "Записать аудио", "Remove attachment": "Удалить вложение", "Rename": "Переименовать", "Rename chat": "Переименовать чат", @@ -507,12 +534,14 @@ "Automatic web search": "Автоматичний пошук в Інтернеті", "Avatar and name": "Аватар та ім'я", "Cancel": "Скасувати", + "Cancel recording": "Скасувати запис", "Choose icon and color": "Виберіть піктограму і колір", "Close model selector": "Закрити вікно вибору моделі", "Close the left panel": "Закрити ліву панель", "Close the menu": "Закрити меню", "Close the modal": "Закрити це вікно", "Confirm deletion": "Підтвердження видалення", + "Confirm recording": "Підтвердити запис", "Content modal to create a project": "Створення проекту", "Content modal to delete conversation": "Підтвердження видалення розмови", "Content modal to delete project": "Видалення проекту", @@ -541,6 +570,7 @@ "Failed to send feedback": "Не вдалося надіслати відгук", "Failed to update settings": "Не вдалося оновити налаштування", "Failed to upload file": "Не вдалося вивантажити файл", + "An error occurred during transcription. Please try again.": "Під час транскрибування сталася помилка. Будь ласка, спробуйте ще раз.", "Failed to upload files. Please try again.": "Не вдалося вивантажити файли. Будь ласка, спробуйте ще раз.", "Feedback Négatif": "Негативний відгук", "Feedback positif": "Позитивний відгук", @@ -583,6 +613,7 @@ "Projects": "Проекти", "Quick search input": "Швидкий пошук", "Reading project instructions...": "Читання інструкцій проекту...", + "Record audio": "Записати аудіо", "Remove attachment": "Видалити вкладення", "Rename": "Перейменувати", "Rename chat": "Перейменувати чат", diff --git a/src/frontend/apps/conversations/src/pages/globals.css b/src/frontend/apps/conversations/src/pages/globals.css index cbb3d91d..dc847db7 100644 --- a/src/frontend/apps/conversations/src/pages/globals.css +++ b/src/frontend/apps/conversations/src/pages/globals.css @@ -170,11 +170,16 @@ nextjs-portal { background-color: inherit !important; } -.c__button--send { +.c__button--send, +.c__button--mic { height: 24px !important; width: 24px !important; } +.c__button--send { + padding: 0 !important; +} + .feedbackButton { display: flex; text-decoration: none; diff --git a/yarn.lock b/yarn.lock index 27622b77..67d9dde9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,17 +2,17 @@ # yarn lockfile v1 -"@fontsource-variable/material-symbols-outlined@^5.2.39": - version "5.2.39" - resolved "https://registry.yarnpkg.com/@fontsource-variable/material-symbols-outlined/-/material-symbols-outlined-5.2.39.tgz#214efcceb7a350ee69da7b1b48a9717f9435120d" - integrity sha512-bv0K84v5xyRJORvP9cFje/uVX+vTMY5DaA9r1f5EJSMnDwPsWHs9qKaZfTlgLtRl7OgY4E/HfDeW+1U9mOnPxg== +"@fontsource-variable/material-symbols-outlined@^5.2.40": + version "5.2.40" + resolved "https://registry.npmjs.org/@fontsource-variable/material-symbols-outlined/-/material-symbols-outlined-5.2.40.tgz" + integrity sha512-F14vK8ygOAMKJ1hFORNZaUQvzxU0bcQcyy/NKKJPODBuZ3JuHQuWlJFvJFWa2lMJrzaCaAIgwwEQN+84uP6qTA== "@fontsource/material-icons-outlined@^5.2.6": version "5.2.6" - resolved "https://registry.yarnpkg.com/@fontsource/material-icons-outlined/-/material-icons-outlined-5.2.6.tgz#64e47a0da250e3b06cd1610983cd1a7d73ec789a" + resolved "https://registry.npmjs.org/@fontsource/material-icons-outlined/-/material-icons-outlined-5.2.6.tgz" integrity sha512-99XKAkwnCg0s0/ywax+o3m01HSNM5gGzSBw+WnlWG2+WY3wOjcN+wXMfm4zP37Yme7Yze2DvKxF78tHWOrlwFw== "@fontsource/material-icons@^5.2.7": version "5.2.7" - resolved "https://registry.yarnpkg.com/@fontsource/material-icons/-/material-icons-5.2.7.tgz#d408ee5677fca128c058c9ab77af0808a2179693" + resolved "https://registry.npmjs.org/@fontsource/material-icons/-/material-icons-5.2.7.tgz" integrity sha512-crPmK0L34lPGmS5GSGLasKpRGQzl95SxMsLM+QhBHPgR9uxSsyI5CUTb0cgoMpjtR+Bf1bC9QOe6pavoybbBwg==