feat: add outliers_nb_target to approximate_distribution() for target-based min_similarity [stacks on #2519]

bertopic/representation/_mmr.py

@@ -1,9 +1,12 @@
 import warnings
+from collections.abc import Mapping
+from typing import List
+
 import numpy as np
 import pandas as pd
-from typing import List, Mapping, Tuple
 from scipy.sparse import csr_matrix
 from sklearn.metrics.pairwise import cosine_similarity
+
 from bertopic.representation._base import BaseRepresentation
 
 
@@ -45,9 +48,13 @@ def extract_topics(
         topic_model,
         documents: pd.DataFrame,
         c_tf_idf: csr_matrix,
-        topics: Mapping[str, List[Tuple[str, float]]],
-    ) -> Mapping[str, List[Tuple[str, float]]]:
-        """Extract topic representations.
+        topics: Mapping[str, list[tuple[str, float]]],
+    ) -> Mapping[str, list[tuple[str, float]]]:
+        """Extract topic representations using batched embedding extraction.
+
+        Instead of calling _extract_embeddings 2N times (once for words and once
+        for the concatenated sentence per topic), this collects all items and makes
+        a single embedding call.
 
         Arguments:
             topic_model: The BERTopic model
@@ -60,26 +67,48 @@ def extract_topics(
         """
         if topic_model.embedding_model is None:
             warnings.warn(
-                "MaximalMarginalRelevance can only be used BERTopic was instantiated"
+                "MaximalMarginalRelevance can only be used if BERTopic was instantiated "
                 "with the `embedding_model` parameter."
             )
             return topics
 
+        # ---- CHANGED: batch all embedding calls into one ----
+        # Collect all items to embed: individual words + joined sentence per topic
+        items_to_embed = []
+        words_index_ranges = {}  # topic -> (start, end) for word embeddings
+        sentence_indices = {}  # topic -> index for sentence embedding
+
+        for topic, topic_words in topics.items():
+            words = [word[0] for word in topic_words]
+
+            # Record word embedding indices
+            start = len(items_to_embed)
+            items_to_embed.extend(words)
+            words_index_ranges[topic] = (start, len(items_to_embed))
+
+            # Record sentence embedding index
+            sentence_indices[topic] = len(items_to_embed)
+            items_to_embed.append(" ".join(words))
+
+        # Single embedding call for all items across all topics
+        all_embeddings = topic_model._extract_embeddings(items_to_embed, method="word", verbose=False)
+        # ---- END CHANGED ----
+
         updated_topics = {}
         for topic, topic_words in topics.items():
             words = [word[0] for word in topic_words]
-            word_embeddings = topic_model._extract_embeddings(words, method="word", verbose=False)
-            topic_embedding = topic_model._extract_embeddings(" ".join(words), method="word", verbose=False).reshape(
-                1, -1
-            )
-            topic_words = mmr(
+            w_start, w_end = words_index_ranges[topic]
+            word_embeddings = all_embeddings[w_start:w_end]
+            topic_embedding = all_embeddings[sentence_indices[topic]].reshape(1, -1)
+
+            topic_words_selected = mmr(
                 topic_embedding,
                 word_embeddings,
                 words,
                 self.diversity,
                 self.top_n_words,
             )
-            updated_topics[topic] = [(word, value) for word, value in topics[topic] if word in topic_words]
+            updated_topics[topic] = [(word, value) for word, value in topics[topic] if word in topic_words_selected]
         return updated_topics
 
 
@@ -111,15 +140,15 @@ def mmr(
     keywords_idx = [np.argmax(word_doc_similarity)]
     candidates_idx = [i for i in range(len(words)) if i != keywords_idx[0]]
 
-    for _ in range(top_n - 1):
+    for _ in range(min(top_n - 1, len(candidates_idx))):
         # Extract similarities within candidates and
         # between candidates and selected keywords/phrases
         candidate_similarities = word_doc_similarity[candidates_idx, :]
         target_similarities = np.max(word_similarity[candidates_idx][:, keywords_idx], axis=1)
 
         # Calculate MMR
-        mmr = (1 - diversity) * candidate_similarities - diversity * target_similarities.reshape(-1, 1)
-        mmr_idx = candidates_idx[np.argmax(mmr)]
+        mmr_score = (1 - diversity) * candidate_similarities - diversity * target_similarities.reshape(-1, 1)
+        mmr_idx = candidates_idx[np.argmax(mmr_score)]
 
         # Update keywords & candidates
         keywords_idx.append(mmr_idx)

docs/getting_started/distribution/distribution.md

@@ -105,3 +105,29 @@ As a default, we compare the c-TF-IDF calculations between the token sets and al
 ```python
 topic_distr, _ = topic_model.approximate_distribution(docs, use_embedding_model=True)
 ```
+
+
+## **Automatic min_similarity selection**
+
+Instead of manually choosing `min_similarity`, you can specify how many documents
+should have zero topic distributions and let BERTopic find the optimal value:
+
+```python
+# Target 5 documents with zero distributions
+topic_distr, _ = topic_model.approximate_distribution(docs, outliers_nb_target=5)
+
+# Remove all zero-distribution documents
+topic_distr, _ = topic_model.approximate_distribution(docs, outliers_nb_target=0)
+```
+
+You can constrain the search range with `min_similarity_min_threshold`:
+
+```python
+topic_distr, _ = topic_model.approximate_distribution(
+    docs, outliers_nb_target=5, min_similarity_min_threshold=0.05
+)
+```
+
+!!! note
+    `min_similarity` and `outliers_nb_target` are mutually exclusive.
+    Setting both will raise a `ValueError`.

docs/getting_started/outlier_reduction/outlier_reduction.md

@@ -119,6 +119,50 @@ new_topics = topic_model.reduce_outliers(docs, new_topics, strategy="distributio
 ```
 
 
+## **Automatic threshold selection**
+
+Instead of manually tuning the `threshold` parameter, you can specify a target outlier
+percentage and let BERTopic find the optimal threshold automatically:
+
+```python
+# Target roughly 5% outlier documents
+new_topics = topic_model.reduce_outliers(docs, topics, outliers_percentage_target=0.05)
+```
+
+This works with all four strategies:
+
+```python
+# Auto-threshold with embeddings strategy
+new_topics = topic_model.reduce_outliers(
+    docs, topics,
+    strategy="embeddings",
+    embeddings=embeddings,
+    outliers_percentage_target=0.05,
+)
+
+# Auto-threshold with c-TF-IDF strategy
+new_topics = topic_model.reduce_outliers(
+    docs, topics,
+    strategy="c-tf-idf",
+    outliers_percentage_target=0.0,  # remove all outliers
+)
+```
+
+You can also constrain the search range with `min_threshold` to prevent over-assignment:
+
+```python
+new_topics = topic_model.reduce_outliers(
+    docs, topics,
+    outliers_percentage_target=0.05,
+    min_threshold=0.2,  # never use a threshold below 0.2
+)
+```
+
+!!! note
+    `threshold` and `outliers_percentage_target` are mutually exclusive.
+    Setting both will raise a `ValueError`.
+
+
 ## **Update Topics**
 
 After generating our updated topics, we can feed them back into BERTopic in one of two ways. We can either update the topic representations themselves based on the documents that now belong to new topics or we can only update the topic frequency without updating the topic representations themselves.

docs/getting_started/tips_and_tricks/tips_and_tricks.md

@@ -76,6 +76,25 @@ topic_model = BERTopic(embedding_model=sentence_model, representation_model=repr
 ```
 
 
+## **Configuring representative documents**
+
+By default, BERTopic selects 3 representative documents per topic from a pool of 500
+sampled candidates. You can configure both values at construction time or recalculate
+them after training:
+
+```python
+# Configure at construction time
+topic_model = BERTopic(nr_repr_docs=10, nr_repr_docs_nr_samples=1000)
+topic_model.fit(docs)
+
+# Or recalculate after training with different settings
+topic_model.recalculate_representative_docs(docs, nr_repr_docs=10)
+
+# Access them
+topic_model.get_representative_docs(topic=0)
+```
+
+
 ## **Topic-term matrix**
 Although BERTopic focuses on clustering our documents, the end result does contain a topic-term matrix.
 This topic-term matrix is calculated using c-TF-IDF, a TF-IDF procedure optimized for class-based analyses.

tests/test_auto_min_similarity.py

@@ -0,0 +1,106 @@
+"""Tests for PR7: auto min_similarity for approximate_distribution.
+
+Run from BERTopic repo root:
+    pytest tests/test_auto_min_similarity.py -v
+"""
+
+import copy
+
+import numpy as np
+import pytest
+
+
+class TestAutoMinSimilarity:
+    """Verify auto min_similarity for approximate_distribution."""
+
+    def test_basic_auto_min_similarity(self, base_topic_model, documents):
+        """Auto min_similarity should produce valid topic distributions."""
+        model = copy.deepcopy(base_topic_model)
+        topic_distr, _ = model.approximate_distribution(
+            documents[:50],
+            min_similarity=None,
+            outliers_nb_target=5,
+        )
+        assert topic_distr.shape[0] == 50
+        assert topic_distr.shape[1] > 0
+
+    @pytest.mark.parametrize("target", [0, 5, 10, 25])
+    def test_outliers_near_target(self, base_topic_model, documents, target):
+        """Number of zero-distribution docs should be near the target."""
+        model = copy.deepcopy(base_topic_model)
+        n_docs = min(50, len(documents))
+        topic_distr, _ = model.approximate_distribution(
+            documents[:n_docs],
+            min_similarity=None,
+            outliers_nb_target=target,
+        )
+
+        if topic_distr is not None:
+            nb_outliers = int(np.sum(np.sum(topic_distr, axis=1) == 0))
+            # Should be >= target (or 0 if impossible)
+            assert nb_outliers >= target or nb_outliers == 0
+
+    def test_mutually_exclusive_params(self, base_topic_model, documents):
+        """Cannot set both min_similarity and outliers_nb_target."""
+        model = copy.deepcopy(base_topic_model)
+        with pytest.raises(ValueError, match="either"):
+            model.approximate_distribution(
+                documents[:10],
+                min_similarity=0.1,
+                outliers_nb_target=5,
+            )
+
+    def test_default_uses_min_similarity(self, base_topic_model, documents):
+        """Default call (no outliers_nb_target) should use min_similarity=0.1 as before."""
+        model = copy.deepcopy(base_topic_model)
+        # Calling with default args should work unchanged
+        topic_distr, _ = model.approximate_distribution(documents[:10])
+        assert topic_distr.shape[0] == 10
+
+    def test_min_similarity_min_threshold(self, base_topic_model, documents):
+        """min_similarity_min_threshold should constrain the search."""
+        model = copy.deepcopy(base_topic_model)
+        n_docs = 30
+
+        # Low threshold: more outliers removed
+        result_low, _ = model.approximate_distribution(
+            documents[:n_docs],
+            min_similarity=None,
+            outliers_nb_target=0,
+            min_similarity_min_threshold=0.0,
+        )
+
+        # High threshold: fewer outliers removed
+        result_high, _ = model.approximate_distribution(
+            documents[:n_docs],
+            min_similarity=None,
+            outliers_nb_target=0,
+            min_similarity_min_threshold=0.5,
+        )
+
+        if result_low is not None and result_high is not None:
+            outliers_low = int(np.sum(np.sum(result_low, axis=1) == 0))
+            outliers_high = int(np.sum(np.sum(result_high, axis=1) == 0))
+            assert outliers_high >= outliers_low
+
+    def test_backward_compatible_min_similarity(self, base_topic_model, documents):
+        """Existing min_similarity parameter should still work."""
+        model = copy.deepcopy(base_topic_model)
+        topic_distr, _ = model.approximate_distribution(
+            documents[:20],
+            min_similarity=0.1,
+        )
+        assert topic_distr.shape[0] == 20
+
+    def test_with_calculate_tokens(self, base_topic_model, documents):
+        """Auto min_similarity should work with calculate_tokens=True."""
+        model = copy.deepcopy(base_topic_model)
+        topic_distr, token_distr = model.approximate_distribution(
+            documents[:10],
+            min_similarity=None,
+            outliers_nb_target=2,
+            calculate_tokens=True,
+        )
+        assert topic_distr.shape[0] == 10
+        if token_distr is not None:
+            assert len(token_distr) == 10