Support for clustering detections

ami/main/admin.py

@@ -221,7 +221,29 @@ def update_calculated_fields(self, request: HttpRequest, queryset: QuerySet[Even
         self.message_user(request, f"Updated {queryset.count()} events.")
 
     list_filter = ("deployment", "project", "start")
-    actions = [update_calculated_fields]
+
+    @admin.action()
+    def cluster_detections(self, request: HttpRequest, queryset: QuerySet[SourceImageCollection]) -> None:
+        for collection in queryset:
+            from ami.jobs.models import DetectionClusteringJob, Job
+
+            job = Job.objects.create(
+                name=f"Cluster detections for collection {collection.pk}",
+                project=collection.project,
+                source_image_collection=collection,
+                job_type_key=DetectionClusteringJob.key,
+                params={
+                    "ood_threshold": 0.3,
+                    "algorithm": "agglomerative",
+                    "algorithm_kwargs": {"distance_threshold": 80},
+                    "pca": {"n_components": 384},
+                },
+            )
+            job.enqueue()
+
+        self.message_user(request, f"Clustered {queryset.count()} collection(s).")
+
+    actions = [update_calculated_fields, cluster_detections]
 
 
 @admin.register(SourceImage)

ami/main/api/views.py

@@ -757,10 +757,11 @@ def cluster_detections(self, request, pk=None):
             source_image_collection=collection,
             job_type_key=DetectionClusteringJob.key,
             params={
-                "ood_threshold": request.data.get("ood_threshold", 1),
+                "ood_threshold": request.data.get("ood_threshold", 0.3),
+                "feature_extraction_algorithm": request.data.get("feature_extraction_algorithm", None),
                 "algorithm": request.data.get("algorithm", "agglomerative"),
                 "algorithm_kwargs": request.data.get("algorithm_kwargs", {"distance_threshold": 0.5}),
-                "pca_dim": request.data.get("pca", {}).get("n_components", 384),
+                "pca": request.data.get("pca", {"n_components": 384}),
             },
         )
         job.enqueue()

ami/ml/clustering_algorithms/agglomerative.py

@@ -33,7 +33,7 @@ def __init__(self, config: dict):
         self.data_dict = None
         # Access from dictionary instead of attribute
         self.distance_threshold = config.get("algorithm_kwargs", {}).get("distance_threshold", 0.5)
-        self.n_components = config.get("pca", {}).get("n_components", 0)
+        self.n_components = config.get("pca", {}).get("n_components", 384)
 
     def setup(self, data_dict):
         # estimate the distance threshold
@@ -71,7 +71,8 @@ def setup(self, data_dict):
 
     def cluster(self, features):
         logger.info(f"distance threshold: {self.distance_threshold}")
-
+        logger.info("features shape: %s", features.shape)
+        logger.info(f"self.n_components: {self.n_components}")
         # Get n_components and linkage from dictionary
         if self.n_components <= min(features.shape[0], features.shape[1]):
             features = dimension_reduction(standardize(features), self.n_components)
@@ -81,6 +82,7 @@ def cluster(self, features):
 
         # Get linkage parameter from config
         linkage = self.config.get("algorithm_kwargs", {}).get("linkage", "ward")
+        logger.info(f" features shape after PCA: {features.shape}")
 
         clusters = AgglomerativeClustering(
             n_clusters=None, distance_threshold=self.distance_threshold, linkage=linkage

ami/ml/clustering_algorithms/cluster_detections.py

@@ -1,6 +1,7 @@
 import logging
 
 import numpy as np
+from django.db.models import Count
 from django.utils.timezone import now
 
 from ami.ml.clustering_algorithms.utils import get_clusterer
@@ -26,13 +27,34 @@ def cluster_detections(collection, params: dict, task_logger: logging.Logger = l
     from ami.ml.models.pipeline import create_and_update_occurrences_for_detections
 
     ood_threshold = params.get("ood_threshold", 1)
-    algorithm = params.get("algorithm", "agglomerative")
+    feature_extraction_algorithm = params.get("feature_extraction_algorithm", None)
+    algorithm = params.get("clustering_algorithm", "agglomerative")
     task_logger.info(f"Clustering Parameters: {params}")
     job_save(job)
+    if feature_extraction_algorithm:
+        task_logger.info(f"Feature Extraction Algorithm: {feature_extraction_algorithm}")
+        # Check if the feature extraction algorithm is valid
+        if not Algorithm.objects.filter(key=feature_extraction_algorithm).exists():
+            raise ValueError(f"Invalid feature extraction algorithm key: {feature_extraction_algorithm}")
+    else:
+        # Fallback to the most used feature extraction algorithm in this collection
+        feature_extraction_algorithm_id = (
+            Classification.objects.filter(features_2048__isnull=False, detection__source_image__collections=collection)
+            .values("algorithm")
+            .annotate(count=Count("id"))
+            .order_by("-count")
+            .values_list("algorithm", flat=True)
+            .first()
+        )
+        if feature_extraction_algorithm_id:
+            feature_extraction_algorithm = Algorithm.objects.get(pk=feature_extraction_algorithm_id)
+            task_logger.info(f"Using fallback feature extraction algorithm: {feature_extraction_algorithm.name}")
+
     detections = Detection.objects.filter(
         classifications__features_2048__isnull=False,
+        classifications__algorithm=feature_extraction_algorithm,
         source_image__collections=collection,
-        occurrence__determination_ood_score__gte=ood_threshold,
+        occurrence__determination_ood_score__gt=ood_threshold,
     )
 
     task_logger.info(f"Found {detections.count()} detections to process for clustering")
@@ -59,7 +81,8 @@ def cluster_detections(collection, params: dict, task_logger: logging.Logger = l
         raise ValueError("No feature vectors found")
 
     features_np = np.array(features)
-
+    task_logger.info(f"Feature vectors shape: {features_np.shape}")
+    logger.info(f"First feature vector: {features_np[0]}, shape: {features_np[0].shape}")
     update_job_progress(job, stage_key="clustering", status=JobState.STARTED, progress=0.0)
     # Clustering Detections
     ClusteringAlgorithm = get_clusterer(algorithm)

ami/ml/tests.py

@@ -7,7 +7,8 @@
 from rest_framework.test import APIRequestFactory, APITestCase
 
 from ami.base.serializers import reverse_with_params
-from ami.main.models import Classification, Detection, Project, SourceImage, SourceImageCollection
+from ami.main.models import Classification, Deployment, Detection, Project, SourceImage, SourceImageCollection, Taxon
+from ami.ml.clustering_algorithms.cluster_detections import cluster_detections
 from ami.ml.models import Algorithm, Pipeline, ProcessingService
 from ami.ml.models.pipeline import collect_images, get_or_create_algorithm_and_category_map, save_results
 from ami.ml.schemas import (
@@ -19,7 +20,13 @@
     PipelineResultsResponse,
     SourceImageResponse,
 )
-from ami.tests.fixtures.main import create_captures_from_files, create_processing_service, setup_test_project
+from ami.tests.fixtures.main import (
+    create_captures,
+    create_captures_from_files,
+    create_processing_service,
+    group_images_into_events,
+    setup_test_project,
+)
 from ami.tests.fixtures.ml import ALGORITHM_CHOICES
 from ami.users.models import User
 
@@ -685,3 +692,104 @@ def test_l2_distance(self):
 
         most_similar = qs.first()
         self.assertEqual(most_similar.pk, ref_cls.pk, "Most similar classification should be itself")
+
+
+class TestClustering(TestCase):
+    def setUp(self):
+        self.project = Project.objects.create(name="Test Clustering Project")
+        self.deployment = Deployment.objects.create(name="Test Deployment", project=self.project)
+
+        create_captures(deployment=self.deployment, num_nights=2, images_per_night=10, interval_minutes=1)
+        group_images_into_events(deployment=self.deployment)
+        sample_size = 10
+        self.collection = SourceImageCollection.objects.create(
+            name="Test Random Source Image Collection",
+            project=self.project,
+            method="random",
+            kwargs={"size": sample_size},
+        )
+        self.collection.save()
+        self.collection.populate_sample()
+        assert self.collection.images.count() == sample_size
+        self.populate_collection_with_detections()
+        self.collection.save()
+        self.detections = Detection.objects.filter(source_image__collections=self.collection)
+        self.assertGreater(len(self.detections), 0, "No detections found in the collection")
+        self._populate_detection_features()
+        for detection in self.detections:
+            assert detection.classifications.last().features_2048 is not None, "No features found in the detection"
+
+    def populate_collection_with_detections(self):
+        """Populate the collection with random detections."""
+        for image in self.collection.images.all():
+            # Create a random detection for each image
+            Detection.objects.create(
+                source_image=image,
+                detection_algorithm=Algorithm.objects.get(key="random-detector"),
+                bbox=[0.0, 0.0, 1.0, 1.0],
+                timestamp=datetime.datetime.now(),
+            )
+
+    def _populate_detection_features(self):
+        """Populate detection features with random values."""
+        for detection in self.detections:
+            # Create a random feature vector
+            feature_vector = np.random.rand(2048).tolist()
+            # Assign the feature vector to the detection
+            detection.classifications.create(
+                algorithm=Algorithm.objects.get(key="random-species-classifier"),
+                taxon=None,
+                score=1,
+                features_2048=feature_vector,
+                timestamp=datetime.datetime.now(),
+            )
+            detection.save()
+
+    def test_agglomerative_clustering(self):
+        """Test agglomerative clustering with real implementation."""
+        # Call with agglomerative clustering parameters
+        params = {
+            "algorithm": "agglomerative",
+            "ood_threshold": 1,
+            "agglomerative": {"distance_threshold": 0.5, "linkage": "ward"},
+            "pca": {"n_components": 5},  # Use fewer components for test performance
+        }
+        # Execute the clustering function
+        clusters = cluster_detections(self.collection, params)
+
+        # The exact number could vary based on the random features and threshold
+        self.assertGreaterEqual(len(clusters), 1, "Should create at least 1 cluster")
+        self.assertLessEqual(len(clusters), 5, "Should not create more than 5 clusters")
+
+        # Verify all detections are assigned to clusters
+        total_detections = sum(len(detections) for detections in clusters.values())
+        self.assertEqual(
+            total_detections,
+            len(self.detections),
+            f"All {len(self.detections)} detections should be assigned to clusters",
+        )
+
+        # Check if detections with similar features are in the same cluster
+        # Create a map of detection to cluster_id
+        detection_to_cluster = {}
+        for cluster_id, detections_list in clusters.items():
+            for detection in detections_list:
+                detection_to_cluster[detection.id] = cluster_id
+
+            # Get all classifications
+        all_classifications = Classification.objects.filter(detection__in=self.detections, terminal=True)
+
+        # Verify that each detection has a new classification linking it to a taxon
+        self.assertEqual(
+            all_classifications.count(), len(self.detections), "Each detection should have a new classification"
+        )
+
+        # Verify that each cluster has a corresponding taxon
+        taxa = Taxon.objects.filter(unknown_species=True)
+        self.assertEqual(taxa.count(), len(clusters), f"Should create {len(clusters)} taxa for the clusters")
+
+        # Verify that each taxon is associated with the project
+        for taxon in taxa:
+            self.assertIn(
+                self.project, taxon.projects.all(), f"Taxon {taxon.name} should be associated with the project"
+            )

ami/tests/fixtures/main.py

@@ -202,7 +202,9 @@ def create_taxa(project: Project) -> TaxaList:
     taxa_list.projects.add(project)
     root, _created = Taxon.objects.get_or_create(name="Lepidoptera", rank=TaxonRank.ORDER.name)
     root.projects.add(project)
-    family_taxon, _ = Taxon.objects.get_or_create(name="Nymphalidae", parent=root, rank=TaxonRank.FAMILY.name)
+    family_taxon, _ = Taxon.objects.get_or_create(
+        name="Nymphalidae", defaults={"parent": root, "rank": TaxonRank.FAMILY.name}
+    )
     family_taxon.projects.add(project)
     genus_taxon, _ = Taxon.objects.get_or_create(name="Vanessa", parent=family_taxon, rank=TaxonRank.GENUS.name)
     genus_taxon.projects.add(project)