diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 3e949f215..e44ee1b60 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -39,7 +39,6 @@ from ami.utils.storages import ConnectionTestResult from ..models import ( - NULL_DETECTIONS_FILTER, Classification, Deployment, Detection, @@ -704,9 +703,7 @@ def filter_by_has_detections(self, queryset: QuerySet) -> QuerySet: if has_detections is not None: has_detections = BooleanField(required=False).clean(has_detections) queryset = queryset.annotate( - has_detections=models.Exists( - Detection.objects.filter(source_image=models.OuterRef("pk")).exclude(NULL_DETECTIONS_FILTER) - ), + has_detections=models.Exists(Detection.objects.valid().filter(source_image=models.OuterRef("pk"))), ).filter(has_detections=has_detections) return queryset @@ -756,7 +753,7 @@ def prefetch_detections(self, queryset: QuerySet, project: Project | None = None score = get_default_classification_threshold(project, self.request) prefetch_queryset = ( - Detection.objects.exclude(NULL_DETECTIONS_FILTER) + Detection.objects.valid() .annotate( determination_score=models.Max("occurrence__detections__classifications__score"), # Store whether this occurrence should be included based on default filters @@ -1096,7 +1093,7 @@ class DetectionViewSet(DefaultViewSet, ProjectMixin): """ require_project_for_list = True # Unfiltered list scans are too expensive on this table - queryset = Detection.objects.exclude(NULL_DETECTIONS_FILTER).select_related("source_image", "detection_algorithm") + queryset = Detection.objects.valid().select_related("source_image", "detection_algorithm") serializer_class = DetectionSerializer filterset_fields = ["source_image", "detection_algorithm", "source_image__project"] ordering_fields = ["created_at", "updated_at", "detection_score", "timestamp"] diff --git a/ami/main/management/commands/cleanup_null_only_occurrences.py b/ami/main/management/commands/cleanup_null_only_occurrences.py new file mode 100644 index 000000000..d31c5233f --- /dev/null +++ b/ami/main/management/commands/cleanup_null_only_occurrences.py @@ -0,0 +1,95 @@ +""" +Delete phantom Occurrences and dangling null-marker Detections left by the Issue #1310 +field bug, on a per-project basis. + +The bug created two categories of rows that should never have been persisted: +- Occurrence rows with no real detections (their only detections are null-marker + sentinels, or they have none at all), surfaced as ghost rows in the API. +- Detection rows that mark a SourceImage as "processed" while no real detections + exist for it — these prevent filter_processed_images from re-yielding the image + on the next ML run. + +After cleanup, the source images become eligible for re-processing. + +Dry-run by default. Pass --commit to delete. +""" + +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction +from django.db.models import Exists, OuterRef + +from ami.main.models import Detection, Occurrence, Project + + +class Command(BaseCommand): + help = "Delete phantom Occurrences and dangling null-marker Detections (Issue #1310)." + + def add_arguments(self, parser): + parser.add_argument( + "--project", + type=int, + required=True, + help="Project ID to clean up.", + ) + parser.add_argument( + "--commit", + action="store_true", + help="Actually delete. Defaults to dry-run.", + ) + + def handle(self, *args, **options): + project_id: int = options["project"] + commit: bool = options["commit"] + + try: + project = Project.objects.get(pk=project_id) + except Project.DoesNotExist as err: + raise CommandError(f"Project {project_id} does not exist") from err + + all_occs = Occurrence.objects.filter(project=project) + # Phantom = an occurrence with NO real (valid) detection backing it: its only detections + # are null-marker sentinels, or it has none at all. This is the Issue #1310 debris. + # + # Deliberately narrower than Occurrence.valid(): valid() ALSO excludes occurrences whose + # determination is null, but an occurrence that has a real detection and merely a missing + # determination is a different (partial-write) shape, not #1310 debris. Deleting it would + # SET_NULL the real detection's occurrence FK (Detection.occurrence is on_delete=SET_NULL), + # stranding a classified detection on an image that filter_processed_images then skips + # forever. Those are left for a separate, targeted repair. + has_valid_detection = Exists(Detection.objects.valid().filter(occurrence_id=OuterRef("pk"))) + phantom_occs = all_occs.exclude(has_valid_detection) + + has_valid_detection = Detection.objects.valid().filter(source_image_id=OuterRef("source_image_id")) + dangling_null_markers = ( + Detection.objects.filter(source_image__project=project) + .null_markers() + .annotate(_has_valid=Exists(has_valid_detection)) + .filter(_has_valid=False) + ) + + phantom_count = phantom_occs.count() + null_count = dangling_null_markers.count() + + self.stdout.write(f"Project #{project.pk} ({project.name}):") + self.stdout.write(f" Phantom occurrences (no real detection backing them): {phantom_count}") + self.stdout.write(f" Dangling null-marker detections on images with no real detections: {null_count}") + + if phantom_count == 0 and null_count == 0: + self.stdout.write(self.style.SUCCESS("Nothing to clean up.")) + return + + if not commit: + self.stdout.write(self.style.WARNING("Dry run — pass --commit to delete.")) + return + + with transaction.atomic(): + dangling_null_markers.delete() + phantom_occs.delete() + + # Report the pre-calculated counts of the rows we targeted directly. The tuple from + # .delete() also counts cascade-deleted related rows (e.g. classifications under a + # phantom occurrence's detections), which would inflate the numbers and confuse the + # operator about what the command actually targeted. + self.stdout.write( + self.style.SUCCESS(f"Deleted {phantom_count} phantom occurrences and {null_count} dangling null markers.") + ) diff --git a/ami/main/models.py b/ami/main/models.py index 6ddabfa3e..2dba65525 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -97,12 +97,28 @@ class TaxonRank(OrderedEnum): ] ) -NULL_DETECTIONS_FILTER = Q(bbox__isnull=True) | Q(bbox=[]) - def bbox_is_null(bbox) -> bool: - """In-memory equivalent of NULL_DETECTIONS_FILTER for already-fetched bbox values.""" - return bbox is None or bbox == [] + """In-memory equivalent of null_detections_q() for an already-fetched bbox value.""" + return bbox is None + + +def null_detections_q(prefix: str = "") -> Q: + """ + Return a Q expression matching null-marker Detection rows, optionally prefixed + for use across relations (e.g. null_detections_q("images__detections__") for an + aggregate filter on a parent table). For Detection queries directly, prefer + Detection.objects.null_markers() / .valid() instead. + + Null markers are stored as SQL NULL (bbox IS NULL); that is the only sentinel form. + """ + return Q(**{f"{prefix}bbox__isnull": True}) + + +# Single source of truth for "this Detection is a null marker", shared by +# DetectionQuerySet.valid() / .null_markers(). Defined via null_detections_q() so the +# constant and the helper cannot drift apart. +NULL_DETECTIONS_FILTER = null_detections_q() def get_media_url(path: str) -> str: @@ -836,7 +852,7 @@ def get_detections_count(self) -> int | None: was processed and no detections were found) to stay consistent with ``SourceImage.get_detections_count`` and ``Event.get_detections_count``. """ - qs = Detection.objects.filter(source_image__deployment=self).exclude(NULL_DETECTIONS_FILTER) + qs = Detection.objects.filter(source_image__deployment=self).valid() filter_q = build_occurrence_default_filters_q( project=self.project, request=None, @@ -1271,7 +1287,7 @@ def get_detections_count(self) -> int | None: Excludes null-bbox placeholder detections to stay consistent with ``SourceImage.get_detections_count`` and ``Deployment.get_detections_count``. """ - qs = Detection.objects.filter(source_image__event=self).exclude(NULL_DETECTIONS_FILTER) + qs = Detection.objects.filter(source_image__event=self).valid() filter_q = build_occurrence_default_filters_q( project=self.project, request=None, @@ -2238,7 +2254,7 @@ def get_detections_count(self) -> int: Excludes detections without bounding boxes — those are placeholder records indicating the image was successfully processed and no detections were found. """ - qs = self.detections.exclude(NULL_DETECTIONS_FILTER) + qs = self.detections.all().valid() project = self.project if not project: return qs.distinct().count() @@ -2518,7 +2534,7 @@ def update_detection_counts( if null_only: qs = qs.filter(detections_count__isnull=True) - detection_qs = Detection.objects.filter(source_image_id=models.OuterRef("pk")).exclude(NULL_DETECTIONS_FILTER) + detection_qs = Detection.objects.filter(source_image_id=models.OuterRef("pk")).valid() if project is not None: filter_q = build_occurrence_default_filters_q( project=project, @@ -3024,7 +3040,23 @@ def save(self, *args, **kwargs): class DetectionQuerySet(BaseQuerySet): - def null_detections(self): + def valid(self): + """ + Detections suitable for consumer queries — excludes null-marker sentinels. + + Null markers are rows that record "an algorithm ran against this image and + found nothing." Consumers asking "give me detections" should always go + through .valid(). Future predicates to fold in here: soft-delete tombstones, + detections missing an algorithm reference, detections missing classifications. + """ + return self.exclude(NULL_DETECTIONS_FILTER) + + def null_markers(self): + """ + Sentinel rows that record "this algorithm ran against this image and found + nothing." Only relevant for SourceImage-level "has this been processed?" + questions. Detection consumers should use .valid() instead. + """ return self.filter(NULL_DETECTIONS_FILTER) @@ -3102,6 +3134,25 @@ class Detection(BaseModel): objects = DetectionManager() + NULL_BBOX = None + """Canonical bbox value for null markers (rows that record 'an algorithm ran but + found nothing'). Null markers are stored as SQL NULL; use Detection.build_null_marker() + to construct them.""" + + @property + def is_null_marker(self) -> bool: + """True for sentinel rows representing 'no detections found by this algorithm.'""" + return self.bbox is None + + @classmethod + def build_null_marker(cls, source_image, detection_algorithm) -> "Detection": + """Construct (without saving) a null-marker Detection for the given image+algorithm.""" + return cls( + source_image=source_image, + bbox=cls.NULL_BBOX, + detection_algorithm=detection_algorithm, + ) + def get_bbox(self): if self.bbox: return BoundingBox( @@ -3222,7 +3273,20 @@ def __str__(self) -> str: class OccurrenceQuerySet(BaseQuerySet): def valid(self): - return self.exclude(detections__isnull=True) + """ + Occurrences fit to surface in API responses: at least one real detection AND + a determination set. + + Excludes: + - Occurrences with no detections at all (empty occurrences) + - Occurrences whose only detections are null-marker sentinels (Issue #1310: + field bug created phantom occurrences with no real bounding box backing + them) + - Occurrences with determination__isnull=True (no taxonomic identification, + same field bug shape) + """ + has_valid_detection = Exists(Detection.objects.valid().filter(occurrence_id=OuterRef("pk"))) + return self.filter(has_valid_detection).exclude(determination__isnull=True) def with_detections_count(self): return self.annotate(detections_count=models.Count("detections", distinct=True)) @@ -4621,7 +4685,7 @@ def with_source_images_with_detections_count(self): return self.annotate( source_images_with_detections_count=models.Count( "images", - filter=(~models.Q(images__detections__bbox__isnull=True) & ~models.Q(images__detections__bbox=[])), + filter=~null_detections_q("images__detections__"), distinct=True, ) ) @@ -5013,10 +5077,11 @@ def sample_greatest_file_size_from_each_event(self, num_each: int = 1): return captures def sample_detections_only(self): - """Sample all source images with detections""" + """Sample all source images with at least one real (non-null-marker) detection.""" qs = self.get_queryset() - return qs.filter(detections__isnull=False).distinct() + valid_detection_image_ids = Detection.objects.valid().values("source_image_id") + return qs.filter(pk__in=valid_detection_image_ids).distinct() def sample_full( self, diff --git a/ami/main/tests.py b/ami/main/tests.py index 0dca7b836..4c8b4b219 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -6009,3 +6009,294 @@ def test_verified_count_not_inflated_by_collection_join(self): rows = self._list_by_name(f"{self.list_url}&collection={collection.pk}") # 2 verified cardui occurrences, not 3 — the duplicate detection must not double-count. self.assertEqual(rows["Vanessa cardui"]["verified_count"], 2) + + +class TestDetectionNullMarker(TestCase): + """ + Covers the null-marker abstraction added for Issue #1310 follow-up: + Detection.is_null_marker, Detection.build_null_marker, and + DetectionQuerySet.valid() / .null_markers(). + """ + + def setUp(self): + from ami.ml.models.algorithm import Algorithm + + self.project = Project.objects.create(name="Null Marker Test Project") + self.deployment = Deployment.objects.create(project=self.project, name="dep") + self.source_image = SourceImage.objects.create( + deployment=self.deployment, + project=self.project, + path="nullmarker-test.jpg", + ) + self.algorithm = Algorithm.objects.create(name="test-detector", key="test-detector", task_type="detection") + + def test_is_null_marker_for_bbox_none(self): + det = Detection.objects.create( + source_image=self.source_image, + bbox=None, + detection_algorithm=self.algorithm, + ) + self.assertTrue(det.is_null_marker) + + def test_is_null_marker_false_for_real_detection(self): + det = Detection.objects.create( + source_image=self.source_image, + bbox=[0.0, 0.0, 1.0, 1.0], + detection_algorithm=self.algorithm, + ) + self.assertFalse(det.is_null_marker) + + def test_build_null_marker_sets_canonical_fields(self): + det = Detection.build_null_marker(self.source_image, self.algorithm) + self.assertIsNone(det.bbox) + self.assertEqual(det.source_image, self.source_image) + self.assertEqual(det.detection_algorithm, self.algorithm) + # timestamp is intentionally not set on the builder — Detection.save() backfills + # it from the source image's capture time, so the marker sorts by capture time + # rather than processing time. + self.assertIsNone(det.timestamp) + self.assertTrue(det.is_null_marker) + + def test_valid_and_null_markers_are_disjoint_and_complete(self): + real = Detection.objects.create( + source_image=self.source_image, + bbox=[0.0, 0.0, 1.0, 1.0], + detection_algorithm=self.algorithm, + ) + null_marker = Detection.objects.create( + source_image=self.source_image, + bbox=None, + detection_algorithm=self.algorithm, + ) + scoped = Detection.objects.filter(source_image=self.source_image) + valid_pks = set(scoped.valid().values_list("pk", flat=True)) + null_pks = set(scoped.null_markers().values_list("pk", flat=True)) + self.assertEqual(valid_pks, {real.pk}) + self.assertEqual(null_pks, {null_marker.pk}) + self.assertEqual(valid_pks & null_pks, set()) + + +class TestOccurrenceValidQuerySet(TestCase): + """ + Covers OccurrenceQuerySet.valid() tightening for Issue #1310: + excludes occurrences with no real detections and occurrences missing a + determination, in addition to the existing detections__isnull=True + exclusion. + """ + + def setUp(self): + from ami.main.models import Taxon + from ami.ml.models.algorithm import Algorithm + + self.project = Project.objects.create(name="Occurrence Valid Test Project") + self.deployment = Deployment.objects.create(project=self.project, name="dep") + self.event = Event.objects.create( + project=self.project, + deployment=self.deployment, + group_by="2024-01-01", + start=datetime.datetime(2024, 1, 1, 0, 0), + ) + self.source_image = SourceImage.objects.create( + deployment=self.deployment, + project=self.project, + event=self.event, + path="occvalid-test.jpg", + ) + self.algorithm = Algorithm.objects.create(name="valid-detector", key="valid-detector", task_type="detection") + self.taxon = Taxon.objects.create(name="Occurrence Valid Test Taxon") + + def _make_occurrence_with_real_detection(self) -> Occurrence: + occ = Occurrence.objects.create( + project=self.project, + event=self.event, + deployment=self.deployment, + determination=self.taxon, + ) + Detection.objects.create( + source_image=self.source_image, + bbox=[0.0, 0.0, 1.0, 1.0], + detection_algorithm=self.algorithm, + occurrence=occ, + ) + return occ + + def _make_occurrence_with_only_null_detection(self) -> Occurrence: + occ = Occurrence.objects.create( + project=self.project, + event=self.event, + deployment=self.deployment, + determination=self.taxon, + ) + Detection.objects.create( + source_image=self.source_image, + bbox=None, + detection_algorithm=self.algorithm, + occurrence=occ, + ) + return occ + + def _make_occurrence_with_null_determination(self) -> Occurrence: + occ = Occurrence.objects.create( + project=self.project, + event=self.event, + deployment=self.deployment, + determination=None, + ) + Detection.objects.create( + source_image=self.source_image, + bbox=[0.0, 0.0, 1.0, 1.0], + detection_algorithm=self.algorithm, + occurrence=occ, + ) + return occ + + def test_valid_returns_only_real_with_determination(self): + real = self._make_occurrence_with_real_detection() + null_only = self._make_occurrence_with_only_null_detection() + no_determination = self._make_occurrence_with_null_determination() + + valid_qs = Occurrence.objects.filter(project=self.project).valid() + valid_pks = set(valid_qs.values_list("pk", flat=True)) + + self.assertIn(real.pk, valid_pks) + self.assertNotIn(null_only.pk, valid_pks) + self.assertNotIn(no_determination.pk, valid_pks) + + +class TestCleanupNullOnlyOccurrencesCommand(TestCase): + """ + Covers ami/main/management/commands/cleanup_null_only_occurrences.py. + Verifies dry-run reports counts without deleting and --commit deletes + phantom occurrences and dangling null markers, leaving valid rows alone. + """ + + def setUp(self): + from ami.main.models import Taxon + from ami.ml.models.algorithm import Algorithm + + self.project = Project.objects.create(name="Cleanup Cmd Test Project") + self.deployment = Deployment.objects.create(project=self.project, name="dep") + self.event = Event.objects.create( + project=self.project, + deployment=self.deployment, + group_by="2024-01-01", + start=datetime.datetime(2024, 1, 1, 0, 0), + ) + self.algorithm = Algorithm.objects.create( + name="cleanup-detector", key="cleanup-detector", task_type="detection" + ) + self.taxon = Taxon.objects.create(name="Cleanup Test Taxon") + + self.img_with_real = SourceImage.objects.create( + deployment=self.deployment, + project=self.project, + event=self.event, + path="with-real.jpg", + ) + self.img_only_null = SourceImage.objects.create( + deployment=self.deployment, + project=self.project, + event=self.event, + path="only-null.jpg", + ) + + self.valid_occurrence = Occurrence.objects.create( + project=self.project, + event=self.event, + deployment=self.deployment, + determination=self.taxon, + ) + self.real_detection = Detection.objects.create( + source_image=self.img_with_real, + bbox=[0.0, 0.0, 1.0, 1.0], + detection_algorithm=self.algorithm, + occurrence=self.valid_occurrence, + ) + self.null_on_processed_image = Detection.objects.create( + source_image=self.img_with_real, + bbox=None, + detection_algorithm=self.algorithm, + ) + + self.phantom_occurrence = Occurrence.objects.create( + project=self.project, + event=self.event, + deployment=self.deployment, + determination=self.taxon, + ) + self.phantom_null = Detection.objects.create( + source_image=self.img_only_null, + bbox=None, + detection_algorithm=self.algorithm, + occurrence=self.phantom_occurrence, + ) + + # A different (partial-write) shape: an occurrence with a real detection but a + # missing determination. Occurrence.valid() excludes it (determination IS NULL), + # but the cleanup command must NOT delete it — doing so would SET_NULL the real + # detection's occurrence FK and strand a classified detection on an image that + # filter_processed_images then skips forever. The command's phantom predicate is + # deliberately narrower than valid() to spare exactly this case. + self.img_real_no_determination = SourceImage.objects.create( + deployment=self.deployment, + project=self.project, + event=self.event, + path="real-no-determination.jpg", + ) + self.occ_real_no_determination = Occurrence.objects.create( + project=self.project, + event=self.event, + deployment=self.deployment, + determination=None, + ) + self.real_detection_no_determination = Detection.objects.create( + source_image=self.img_real_no_determination, + bbox=[0.0, 0.0, 1.0, 1.0], + detection_algorithm=self.algorithm, + occurrence=self.occ_real_no_determination, + ) + + def _call_command(self, *args): + from io import StringIO + + from django.core.management import call_command + + out = StringIO() + call_command("cleanup_null_only_occurrences", *args, stdout=out) + return out.getvalue() + + def test_dry_run_reports_counts_without_deleting(self): + output = self._call_command(f"--project={self.project.pk}") + self.assertIn("Phantom occurrences", output) + self.assertIn("Dangling null-marker detections", output) + self.assertIn("Dry run", output) + self.assertTrue(Occurrence.objects.filter(pk=self.phantom_occurrence.pk).exists()) + self.assertTrue(Detection.objects.filter(pk=self.phantom_null.pk).exists()) + + def test_commit_deletes_phantoms_and_dangling_null_markers(self): + self._call_command(f"--project={self.project.pk}", "--commit") + self.assertFalse(Occurrence.objects.filter(pk=self.phantom_occurrence.pk).exists()) + self.assertFalse(Detection.objects.filter(pk=self.phantom_null.pk).exists()) + self.assertTrue(Occurrence.objects.filter(pk=self.valid_occurrence.pk).exists()) + self.assertTrue(Detection.objects.filter(pk=self.real_detection.pk).exists()) + self.assertTrue( + Detection.objects.filter(pk=self.null_on_processed_image.pk).exists(), + "Null markers on images with at least one real detection must be kept", + ) + self.assertTrue( + Occurrence.objects.filter(pk=self.occ_real_no_determination.pk).exists(), + "Occurrence with a real detection but a missing determination is a partial-write " + "shape, not Issue #1310 debris — it must be preserved, not deleted", + ) + self.real_detection_no_determination.refresh_from_db() + self.assertEqual( + self.real_detection_no_determination.occurrence_id, + self.occ_real_no_determination.pk, + "The real detection must keep its occurrence FK — deleting the occurrence would " + "SET_NULL it and strand the detection", + ) + + def test_commit_is_idempotent(self): + self._call_command(f"--project={self.project.pk}", "--commit") + second_run = self._call_command(f"--project={self.project.pk}", "--commit") + self.assertIn("Nothing to clean up", second_run) diff --git a/ami/ml/models/pipeline.py b/ami/ml/models/pipeline.py index 7cb54d14f..7940207ac 100644 --- a/ami/ml/models/pipeline.py +++ b/ami/ml/models/pipeline.py @@ -545,9 +545,11 @@ def get_or_create_detection( ), f"Detection belongs to a different source image: {detection_repr}" if serialized_bbox is None: - # Null detection: algorithm-specific lookup so different pipelines don't share sentinels. - # Use bbox__isnull=True because JSONField filter(bbox=None) matches JSON null literal, - # not SQL NULL which is what Detection(bbox=None) stores. + # existing_detection := the null-marker sentinel already recorded for this image+algorithm + # (or None if there is none yet). The lookup is algorithm-specific so different pipelines + # don't share sentinels, and narrowed to .null_markers() rather than a bare filter: a null + # response has no bbox to match on, so without .null_markers() the (image, algorithm) filter + # would also return real detections, and .first() could reuse one as if it were the sentinel. assert detection_resp.algorithm, f"No detection algorithm was specified for detection {detection_repr}" try: detection_algo = algorithms_known[detection_resp.algorithm.key] @@ -557,13 +559,19 @@ def get_or_create_detection( "The processing service must declare it in the /info endpoint. " f"Known algorithms: {list(algorithms_known.keys())}" ) from err - existing_detection = Detection.objects.filter( - source_image=source_image, - bbox__isnull=True, - detection_algorithm=detection_algo, - ).first() + existing_detection = ( + Detection.objects.filter( + source_image=source_image, + detection_algorithm=detection_algo, + ) + .null_markers() + .first() + ) else: - # Real detection: algorithm-agnostic — same bbox = same physical detection + # existing_detection := the detection with this exact bbox on this image (or None). The + # match is algorithm-agnostic — the same bounding box on the same image is the same physical + # detection regardless of which algorithm found it, so a bbox match is a duplicate to reuse. + # A specific bbox can't match a null marker (bbox IS NULL), so sentinels are excluded here. existing_detection = Detection.objects.filter( source_image=source_image, bbox=serialized_bbox, @@ -1058,15 +1066,6 @@ def save_results( "Algorithms and category maps must be registered before processing, using /info endpoint." ) - # Ensure all images have detections - # if not, add a NULL detection (empty bbox) to the results - null_detections = create_null_detections_for_undetected_images( - results=results, - detection_algorithm=detection_algorithm, - logger=job_logger, - ) - results.detections = results.detections + null_detections - detections = create_detections( detections=results.detections, algorithms_known=algorithms_known, @@ -1105,6 +1104,22 @@ def save_results( for deployment in Deployment.objects.filter(pk__in=deployment_ids): deployment.update_calculated_fields(save=True) + # Mark images with no real detections as processed by creating null-bbox sentinels. + # Issue #1310: MUST be the final write. Persisting a null marker is the signal that + # filter_processed_images uses to skip an image on future runs, so it can only be + # written after every step that might fail has succeeded. Any raise earlier in the + # pipeline leaves the image unmarked so it gets retried on the next run. + null_detection_responses = create_null_detections_for_undetected_images( + results=results, + detection_algorithm=detection_algorithm, + logger=job_logger, + ) + create_detections( + detections=null_detection_responses, + algorithms_known=algorithms_known, + logger=job_logger, + ) + total_time = time.time() - start_time job_logger.info(f"Saved results from pipeline {pipeline} in {total_time:.2f} seconds") diff --git a/ami/ml/tests.py b/ami/ml/tests.py index ea375135e..898a69d11 100644 --- a/ami/ml/tests.py +++ b/ami/ml/tests.py @@ -13,6 +13,7 @@ Deployment, Detection, Event, + Occurrence, Project, SourceImage, SourceImageCollection, @@ -1245,6 +1246,117 @@ def test_null_detection_deduplication_same_pipeline(self): null_detections = image.detections.filter(bbox__isnull=True) self.assertEqual(null_detections.count(), 1, "Same pipeline should not create duplicate null detections") + def test_null_detection_does_not_create_phantom_occurrence(self): + """ + Issue #1310: a null detection (empty-bbox sentinel marking "image processed, + nothing found") must NOT spawn an Occurrence. Occurrences with no + determination and no real detections leak to the API as ghost rows. + """ + image = self.test_images[0] + results = self.fake_pipeline_results([image], self.pipeline) + results.detections = [] # pipeline found nothing + + save_results(results) + + null_dets = image.detections.filter(bbox__isnull=True) + self.assertEqual(null_dets.count(), 1, "Null marker should still be created") + self.assertIsNone( + null_dets.first().occurrence, + "Null detection must NOT be associated with an Occurrence", + ) + # No phantom Occurrence in DB tied to this image at all + phantom_occs = Occurrence.objects.filter(detections__source_image=image, determination__isnull=True) + self.assertEqual( + phantom_occs.count(), + 0, + "No Occurrence with NULL determination should exist for an image that had no detections", + ) + + def test_captures_not_marked_processed_after_failure(self): + """ + Issue #1310: null markers should only flag images as processed AFTER all + downstream save steps (classifications, occurrences) succeed. If any + downstream step raises, the image must remain unmarked so the next run + re-processes it. + + Reproduces the field bug where 400 images ended up with null markers but + no real detections — created when null-creation ran ahead of a later step + that failed. + """ + from unittest.mock import patch + + from ami.ml.models.pipeline import filter_processed_images + + # Mix: image_with_real has a detection in the response, image_without_real does not. + # The without-real image is the one that would get a null marker. + image_with_real, image_without_real = self.test_images + results = self.fake_pipeline_results(self.test_images, self.pipeline) + # Trim detections to only the first image so the second qualifies for null-marker creation + results.detections = [d for d in results.detections if str(d.source_image_id) == str(image_with_real.pk)] + + # Inject failure in a step that runs AFTER detection bulk_create + with patch( + "ami.ml.models.pipeline.create_classifications", + side_effect=RuntimeError("simulated classification failure"), + ): + with self.assertRaises(RuntimeError): + save_results(results) + + # The image with no real detection must NOT have a null marker — + # the run failed, so it should be re-tried. + null_dets = image_without_real.detections.filter(bbox__isnull=True) + self.assertEqual( + null_dets.count(), + 0, + "Image without real detections must not be marked processed when downstream step fails", + ) + # filter_processed_images should still yield it for the next run + retry_yield = list(filter_processed_images([image_without_real], self.pipeline)) + self.assertEqual( + retry_yield, + [image_without_real], + "Image with failed run must be re-yielded for processing", + ) + + def test_null_marker_not_persisted_when_broker_dispatch_fails(self): + """ + Issue #1310 (takeaway-review follow-up): null markers must be the FINAL + write in save_results. Failures in any of the trailing steps — + create_detection_images.delay (broker outage), update_calculated_fields_for_events + (DB error), Deployment.update_calculated_fields (DB error) — must leave the + image unmarked. + + This test patches the celery dispatch to raise, simulating a broker + outage between the real-detection save and the null-marker save. + """ + from unittest.mock import patch + + from ami.ml.models.pipeline import filter_processed_images + + image_with_real, image_without_real = self.test_images + results = self.fake_pipeline_results(self.test_images, self.pipeline) + results.detections = [d for d in results.detections if str(d.source_image_id) == str(image_with_real.pk)] + + with patch( + "ami.ml.models.pipeline.create_detection_images.delay", + side_effect=RuntimeError("simulated broker outage"), + ): + with self.assertRaises(RuntimeError): + save_results(results) + + null_dets = image_without_real.detections.filter(bbox__isnull=True) + self.assertEqual( + null_dets.count(), + 0, + "Null marker must not be persisted when create_detection_images.delay fails", + ) + retry_yield = list(filter_processed_images([image_without_real], self.pipeline)) + self.assertEqual( + retry_yield, + [image_without_real], + "Image with failed broker dispatch must be re-yielded for processing", + ) + class TestAlgorithmCategoryMaps(TestCase): def setUp(self):