✨(feat) POC for audio prompting with Albert API

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to
 ### Added
 
 - ✨(back) add ODT parsing support
+- ✨(feat) add audio prompting
 
 ### Changed
 

src/backend/chat/exceptions.py

@@ -0,0 +1,5 @@
+"""Exceptions related to chat."""
+
+
+class TranscriptionError(Exception):
+    """Raised when an error occurs during transcription."""

src/backend/chat/input/albert_audio_backend.py

@@ -0,0 +1,63 @@
+"""Albert API implementation for audio transcription."""
+
+import logging
+from typing import IO
+from urllib.parse import urljoin
+
+from django.conf import settings
+
+import requests
+
+from chat.exceptions import TranscriptionError
+from chat.input.base_audio_backend import BaseAudioBackend
+
+logger = logging.getLogger(__name__)
+
+
+class AlbertAudioBackend(BaseAudioBackend):
+    """
+    Audio transcription backend using the Albert API.
+
+    Sends audio files to the Albert ASR endpoint and returns the transcribed text.
+    Requires ALBERT_API_URL and ALBERT_API_KEY to be configured in settings.
+    """
+
+    def __init__(self):
+        self._transcriptions_endpoint = urljoin(settings.ALBERT_API_URL, "/v1/audio/transcriptions")
+        self._headers = {"Authorization": f"Bearer {settings.ALBERT_API_KEY}"}
+
+    def transcribe(self, file_name: str, file_content: IO[bytes], content_type: str) -> str:
+        """
+        Transcribe the given audio file using the Albert ASR API.
+
+        Args:
+            file_name (str): The original file name of the audio file.
+            file_content (IO[bytes]): A file-like object with the audio content.
+            content_type (str): The MIME type of the audio file.
+
+        Returns:
+            str: The transcribed text.
+
+        Raises:
+            TranscriptionError: If an error occurs during transcription.
+        """
+        try:
+            response = requests.post(
+                self._transcriptions_endpoint,
+                headers=self._headers,
+                files={"file": (file_name, file_content, content_type)},
+                # whisper on vLLM does not support language detection yet,
+                # so we need to pass the language parameter to Albert API.
+                # We should allow passing 'auto' when whisper once vLLM
+                # supports language detection, or when Albert API migrates
+                # to whisperx.
+                data={
+                    "model": settings.ALBERT_API_ASR_MODEL,
+                    "language": settings.ALBERT_API_ASR_LANGUAGE,
+                },
+                timeout=settings.ALBERT_API_TIMEOUT,
+            )
+            response.raise_for_status()
+            return response.json().get("text", "").strip()
+        except requests.RequestException as e:
+            raise TranscriptionError(f"Error during transcription: {e}") from e

src/backend/chat/input/base_audio_backend.py

@@ -0,0 +1,25 @@
+"""Base class for audio input backends."""
+
+from abc import ABC, abstractmethod
+from typing import IO
+
+
+class BaseAudioBackend(ABC):
+    """Abstract base class for audio transcription backends."""
+
+    @abstractmethod
+    def transcribe(self, file_name: str, file_content: IO[bytes], content_type: str) -> str:
+        """
+        Transcribe the given audio file to text.
+
+        Args:
+            file_name (str): The original file name of the audio file.
+            file_content (IO[bytes]): A file-like object with the audio content.
+            content_type (str): The MIME type of the audio file.
+
+        Returns:
+            str: The transcribed text.
+
+        Raises:
+            TranscriptionError: If an error occurs during transcription.
+        """

src/backend/chat/tests/input/test_albert_audio_backend.py

@@ -0,0 +1,63 @@
+"""Tests for the Albert audio transcription backend."""
+
+import pytest
+import responses
+
+from chat.exceptions import TranscriptionError
+from chat.input.albert_audio_backend import AlbertAudioBackend
+
+
+@pytest.fixture(name="backend")
+def backend_fixture(settings):
+    """Fixture providing a configured AlbertAudioBackend instance."""
+    settings.ALBERT_API_URL = "https://albert.example.com"
+    settings.ALBERT_API_KEY = "test-key"
+    settings.ALBERT_API_TIMEOUT = 10
+    settings.ALBERT_API_ASR_MODEL = "openweight-audio"
+    return AlbertAudioBackend()
+
+
+@responses.activate
+def test_transcribe_strips_whitespace(backend):
+    """Transcription result has surrounding whitespace stripped."""
+    responses.add(
+        responses.POST,
+        "https://albert.example.com/v1/audio/transcriptions",
+        json={"text": "  bonjour monde  "},
+        status=200,
+    )
+
+    result = backend.transcribe("audio.webm", b"data", "audio/webm")
+
+    assert result == "bonjour monde"
+
+
+@responses.activate
+def test_transcribe_raises_transcription_error_on_http_failure(backend):
+    """An HTTP error from the Albert API raises TranscriptionError."""
+    responses.add(
+        responses.POST,
+        "https://albert.example.com/v1/audio/transcriptions",
+        status=503,
+    )
+
+    with pytest.raises(TranscriptionError):
+        backend.transcribe("audio.webm", b"data", "audio/webm")
+
+
+@responses.activate
+def test_transcribe_sends_correct_request(backend):
+    """The backend sends the file and required fields to the Albert API."""
+    responses.add(
+        responses.POST,
+        "https://albert.example.com/v1/audio/transcriptions",
+        json={"text": "test"},
+        status=200,
+    )
+
+    backend.transcribe("recording.webm", b"audio-content", "audio/webm")
+
+    assert len(responses.calls) == 1
+    request = responses.calls[0].request
+    assert "Bearer test-key" in request.headers["Authorization"]
+    assert b"audio-content" in request.body

src/backend/chat/tests/views/test_audio_transcription.py

@@ -0,0 +1,109 @@
+"""Tests for the audio transcription view."""
+
+from io import BytesIO
+
+import pytest
+import responses
+
+from core.factories import UserFactory
+
+pytestmark = pytest.mark.django_db
+
+AUDIO_BYTES = b"RIFF....WAVEfmt "
+
+
+@responses.activate
+def test_audio_transcription_success(api_client, settings):
+    """Test successful audio transcription returns stripped text."""
+    settings.AUDIO_TRANSCRIPTION_BACKEND = "chat.input.albert_audio_backend.AlbertAudioBackend"
+    settings.ALBERT_API_URL = "https://albert.example.com"
+    settings.ALBERT_API_KEY = "test-key"
+    settings.ALBERT_API_TIMEOUT = 10
+    settings.ALBERT_API_ASR_MODEL = "openweight-audio"
+
+    responses.add(
+        responses.POST,
+        "https://albert.example.com/v1/audio/transcriptions",
+        json={"text": "  hello world"},
+        status=200,
+    )
+
+    user = UserFactory()
+    api_client.force_login(user)
+    response = api_client.post(
+        "/api/v1.0/transcribe/",
+        {"audio": BytesIO(AUDIO_BYTES)},
+        format="multipart",
+    )
+
+    assert response.status_code == 200
+    assert response.json() == {"text": "hello world"}
+
+
+def test_audio_transcription_anonymous(api_client, settings):
+    """Anonymous users cannot access the transcription endpoint."""
+    settings.AUDIO_TRANSCRIPTION_BACKEND = "chat.input.albert_audio_backend.AlbertAudioBackend"
+
+    response = api_client.post(
+        "/api/v1.0/transcribe/",
+        {"audio": BytesIO(AUDIO_BYTES)},
+        format="multipart",
+    )
+
+    assert response.status_code == 401
+
+
+def test_audio_transcription_backend_disabled(api_client, settings):
+    """Returns 501 when no transcription backend is configured."""
+    settings.AUDIO_TRANSCRIPTION_BACKEND = None
+
+    user = UserFactory()
+    api_client.force_login(user)
+    response = api_client.post(
+        "/api/v1.0/transcribe/",
+        {"audio": BytesIO(AUDIO_BYTES)},
+        format="multipart",
+    )
+
+    assert response.status_code == 501
+
+
+def test_audio_transcription_no_file(api_client, settings):
+    """Returns 400 when no audio file is provided."""
+    settings.AUDIO_TRANSCRIPTION_BACKEND = "chat.input.albert_audio_backend.AlbertAudioBackend"
+    settings.ALBERT_API_URL = "https://albert.example.com"
+    settings.ALBERT_API_KEY = "test-key"
+    settings.ALBERT_API_TIMEOUT = 10
+    settings.ALBERT_API_ASR_MODEL = "openweight-audio"
+
+    user = UserFactory()
+    api_client.force_login(user)
+    response = api_client.post("/api/v1.0/transcribe/", {}, format="multipart")
+
+    assert response.status_code == 400
+
+
+@responses.activate
+def test_audio_transcription_backend_error(api_client, settings):
+    """Returns 502 when the transcription backend fails."""
+    settings.AUDIO_TRANSCRIPTION_BACKEND = "chat.input.albert_audio_backend.AlbertAudioBackend"
+    settings.ALBERT_API_URL = "https://albert.example.com"
+    settings.ALBERT_API_KEY = "test-key"
+    settings.ALBERT_API_TIMEOUT = 10
+    settings.ALBERT_API_ASR_MODEL = "openweight-audio"
+
+    responses.add(
+        responses.POST,
+        "https://albert.example.com/v1/audio/transcriptions",
+        status=500,
+    )
+
+    user = UserFactory()
+    api_client.force_login(user)
+    response = api_client.post(
+        "/api/v1.0/transcribe/",
+        {"audio": BytesIO(AUDIO_BYTES)},
+        format="multipart",
+    )
+
+    assert response.status_code == 502

src/backend/chat/views.py

@@ -10,6 +10,7 @@
 from django.db.models import Prefetch
 from django.http import Http404, StreamingHttpResponse
 from django.utils.decorators import method_decorator
+from django.utils.module_loading import import_string
 
 import langfuse
 import magic
@@ -19,6 +20,7 @@
 from lasuite.oidc_login.decorators import refresh_oidc_access_token
 from rest_framework import decorators, filters, mixins, permissions, status, viewsets
 from rest_framework.exceptions import MethodNotAllowed, PermissionDenied, ValidationError
+from rest_framework.parsers import MultiPartParser
 from rest_framework.response import Response
 from rest_framework.throttling import ScopedRateThrottle
 from rest_framework.views import APIView
@@ -33,6 +35,7 @@
 from activation_codes.permissions import IsActivatedUser
 from chat import models, serializers
 from chat.clients.pydantic_ai import AIAgentService
+from chat.exceptions import TranscriptionError
 from chat.keepalive import stream_with_keepalive_async, stream_with_keepalive_sync
 from chat.serializers import ChatConversationRequestSerializer
 
@@ -741,3 +744,54 @@ def perform_destroy(self, instance):
         """
         instance.conversations.all().delete()
         instance.delete()
+
+
+class AudioTranscriptionView(APIView):
+    """Transcribe audio using a configurable ASR backend.
+
+    Accepts a multipart/form-data POST with an ``audio`` file field and returns
+    ``{"text": "<transcription>"}``.
+
+    Requires ``AUDIO_TRANSCRIPTION_BACKEND`` to be set in settings.
+    """
+
+    permission_classes = [
+        IsActivatedUser,
+        permissions.IsAuthenticated,
+    ]
+    parser_classes = [MultiPartParser]
+    throttle_classes = [ScopedRateThrottle]
+    throttle_scope = "transcribe"
+
+    def post(self, request):
+        """Handle POST requests to transcribe audio."""
+        if not settings.AUDIO_TRANSCRIPTION_BACKEND:
+            return Response(
+                {"error": "Audio transcription is not available"},
+                status=status.HTTP_501_NOT_IMPLEMENTED,
+            )
+
+        audio_file = request.FILES.get("audio")
+        if not audio_file:
+            return Response({"error": "No audio file provided"}, status=status.HTTP_400_BAD_REQUEST)
+
+        if audio_file.size > settings.AUDIO_MAX_SIZE:
+            return Response(
+                {"error": "Audio file too large"},
+                status=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
+            )
+
+        backend = import_string(settings.AUDIO_TRANSCRIPTION_BACKEND)()
+        try:
+            text = backend.transcribe(
+                audio_file.name,
+                audio_file,
+                audio_file.content_type,
+            )
+        except TranscriptionError:
+            return Response(
+                {"error": "Transcription failed"},
+                status=status.HTTP_502_BAD_GATEWAY,
+            )
+
+        return Response({"text": text}, status=status.HTTP_200_OK)