Generate a project taxa list from a region, and track which species the models can predict

ami/main/admin.py

@@ -1,7 +1,7 @@
 import datetime
 from typing import Any
 
-from django.contrib import admin
+from django.contrib import admin, messages
 from django.db import models
 from django.db.models.query import QuerySet
 from django.http.request import HttpRequest
@@ -12,6 +12,7 @@
 import ami.utils
 from ami import tasks
 from ami.jobs.models import Job
+from ami.main.tasks import generate_regional_taxa_list_task
 from ami.ml.models.project_pipeline_config import ProjectPipelineConfig
 from ami.ml.post_processing.admin.actions import make_post_processing_action
 from ami.ml.post_processing.admin.class_masking_form import ClassMaskingActionForm
@@ -108,6 +109,7 @@ def save_related(self, request, form, formsets, change):
 
     inlines = [ProjectPipelineConfigInline]
     autocomplete_fields = ("owner", "default_filters_include_taxa", "default_filters_exclude_taxa")
+    raw_id_fields = ("default_taxa_list",)
 
     def get_queryset(self, request: HttpRequest) -> QuerySet[Any]:
         return super().get_queryset(request).select_related("owner")
@@ -138,6 +140,20 @@ def get_queryset(self, request: HttpRequest) -> QuerySet[Any]:
                 ),
             },
         ),
+        (
+            "Region & Taxa List",
+            {
+                "fields": (
+                    "region_source",
+                    "region_code",
+                    "default_taxa_list",
+                ),
+                "description": (
+                    "The region a taxa list can be generated from, and the list used as this "
+                    "project's fallback when a site has none configured."
+                ),
+            },
+        ),
         (
             "Ownership & Access",
             {
@@ -155,7 +171,33 @@ def _remove_duplicate_classifications(self, request: HttpRequest, queryset: Quer
             task_ids.append(task.id)
         self.message_user(request, f"Started {len(task_ids)} tasks to delete classification: {task_ids}")
 
-    actions = [_remove_duplicate_classifications]
+    @admin.action(description="Generate a regional taxa list from the configured region")
+    def generate_regional_taxa_list_action(self, request: HttpRequest, queryset: QuerySet[Project]) -> None:
+        """Enqueue regional taxa-list generation for each selected project that has a
+        region configured; the list is attached to the project's default_taxa_list.
+        Runs in the background because the external fetch is slow."""
+        enqueued = 0
+        skipped = []
+        for project in queryset:
+            if not project.region_source or not project.region_code:
+                skipped.append(project.name)
+                continue
+            generate_regional_taxa_list_task.delay(
+                project_id=project.pk,
+                region_source=project.region_source,
+                region_code=project.region_code,
+            )
+            enqueued += 1
+        if enqueued:
+            self.message_user(request, f"Queued regional taxa-list generation for {enqueued} project(s).")
+        if skipped:
+            self.message_user(
+                request,
+                f"Skipped (no region configured): {', '.join(skipped)}",
+                level=messages.WARNING,
+            )
+
+    actions = [_remove_duplicate_classifications, generate_regional_taxa_list_action]
 
 
 @admin.register(Deployment)
@@ -763,7 +805,40 @@ class DeviceAdmin(admin.ModelAdmin[Device]):
 
 @admin.register(Site)
 class SiteAdmin(admin.ModelAdmin[Site]):
-    """Admin panel example for ``Site`` model."""
+    """Admin panel for ``Site`` (Research Site) model."""
+
+    list_display = ("name", "project", "region_source", "region_code", "taxa_list")
+    list_filter = ("region_source",)
+    search_fields = ("name",)
+    raw_id_fields = ("project", "taxa_list")
+
+    @admin.action(description="Generate a regional taxa list from the configured region")
+    def generate_regional_taxa_list_action(self, request: HttpRequest, queryset: QuerySet[Site]) -> None:
+        """Enqueue regional taxa-list generation for each selected site that has a
+        project and a region configured; the list is attached to the site's taxa_list."""
+        enqueued = 0
+        skipped = []
+        for site in queryset:
+            if not site.project_id or not site.region_source or not site.region_code:
+                skipped.append(site.name)
+                continue
+            generate_regional_taxa_list_task.delay(
+                project_id=site.project_id,
+                region_source=site.region_source,
+                region_code=site.region_code,
+                site_id=site.pk,
+            )
+            enqueued += 1
+        if enqueued:
+            self.message_user(request, f"Queued regional taxa-list generation for {enqueued} site(s).")
+        if skipped:
+            self.message_user(
+                request,
+                f"Skipped (missing project or region): {', '.join(skipped)}",
+                level=messages.WARNING,
+            )
+
+    actions = [generate_regional_taxa_list_action]
 
 
 @admin.register(S3StorageSource)

ami/main/api/views.py

@@ -35,6 +35,8 @@
 from ami.main.api.schemas import limit_doc_param, project_id_doc_param
 from ami.main.api.serializers import TagSerializer
 from ami.main.models_future.occurrence import model_agreement_for_project, top_identifiers_for_project
+from ami.main.services import regional_taxa
+from ami.main.tasks import generate_regional_taxa_list_task
 from ami.utils.requests import get_default_classification_threshold
 from ami.utils.storages import ConnectionTestResult
 
@@ -49,6 +51,7 @@
     Page,
     Project,
     ProjectQuerySet,
+    RegionSource,
     S3StorageSource,
     Site,
     SourceImage,
@@ -155,6 +158,15 @@ def get_count(self, queryset):
         return super().get_count(queryset.order_by().values("pk"))
 
 
+class RegionalTaxaListRequestSerializer(serializers.Serializer):
+    """Body for POST /projects/{id}/generate-regional-taxa-list/ (see #1364)."""
+
+    region_source = serializers.ChoiceField(choices=RegionSource.choices, default=RegionSource.GBIF_GADM.value)
+    region_code = serializers.CharField(required=False, allow_blank=True)
+    classifier_id = serializers.IntegerField(required=False)
+    include_uncovered = serializers.BooleanField(required=False, default=False)
+
+
 class ProjectViewSet(DefaultViewSet, ProjectMixin):
     """
     API endpoint that allows projects to be viewed or edited.
@@ -256,6 +268,59 @@ def perform_create(self, serializer):
         # Add current user as project owner
         serializer.save(owner=self.request.user)
 
+    @action(
+        detail=True,
+        methods=["post"],
+        name="generate-regional-taxa-list",
+        url_path="generate-regional-taxa-list",
+    )
+    def generate_regional_taxa_list(self, request, pk=None) -> Response:
+        """Queue generation of a taxa list for this project from a geographic region.
+
+        The external biodiversity-database fetch is slow, so this enqueues a background
+        task and returns 202; on success the generated list becomes the project's
+        default_taxa_list. When region_code is omitted it is derived from the project's
+        deployments. Requires update permission on the project. See issue #1364.
+        """
+        project = get_object_or_404(self.get_queryset(), pk=pk)
+        if not request.user.has_perm("update_project", project):
+            raise PermissionDenied("You do not have permission to modify this project.")
+
+        params = RegionalTaxaListRequestSerializer(data=request.data)
+        params.is_valid(raise_exception=True)
+        data = params.validated_data
+
+        region_source = data["region_source"]
+        region_code = (data.get("region_code") or "").strip()
+        if not region_code:
+            derived = regional_taxa.derive_region_for_project(project, region_source=region_source)
+            if derived is None:
+                raise api_exceptions.ValidationError(
+                    {
+                        "region_code": (
+                            "No region_code was provided and none could be derived from the " "project's deployments."
+                        )
+                    }
+                )
+            _source, region_code = derived
+
+        generate_regional_taxa_list_task.delay(
+            project_id=project.pk,
+            region_source=region_source,
+            region_code=region_code,
+            classifier_id=data.get("classifier_id"),
+            include_uncovered=data.get("include_uncovered", False),
+        )
+        return Response(
+            {
+                "project_id": project.pk,
+                "region_source": region_source,
+                "region_code": region_code,
+                "status": "queued",
+            },
+            status=status.HTTP_202_ACCEPTED,
+        )
+
     @action(detail=True, methods=["get"], name="charts")
     def charts(self, request, pk=None):
         """

ami/main/management/commands/generate_regional_taxa_list.py

@@ -0,0 +1,119 @@
+"""Generate a project taxa list from a geographic region (issue #1364).
+
+A thin wrapper over `ami.main.services.regional_taxa.generate_regional_taxa_list`. Run
+it for one project with an explicit region, or use `--all-projects` to backfill every
+project, deriving each one's region from a representative deployment's coordinates
+(GBIF reverse-geocode). The heavy lifting lives in the service so the same behavior is
+reachable from the admin, the API, and tests; this command is the operator/backfill
+entry point.
+"""
+
+from __future__ import annotations
+
+import logging
+
+from django.core.management.base import BaseCommand, CommandError
+
+from ami.main.models import Project, RegionSource
+from ami.main.services import regional_taxa
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "Generate a project taxa list from a geographic region (GBIF/GADM)."
+
+    def add_arguments(self, parser):
+        parser.add_argument("--project", type=int, help="Project id to attach the list to.")
+        parser.add_argument(
+            "--all-projects",
+            action="store_true",
+            help="Backfill every project, deriving each region from its deployments.",
+        )
+        parser.add_argument(
+            "--region-source",
+            default=RegionSource.GBIF_GADM.value,
+            choices=[choice.value for choice in RegionSource],
+        )
+        parser.add_argument(
+            "--region-code",
+            help="Region id (a GADM gid such as USA.46_1). Omit with --all-projects; it is derived.",
+        )
+        parser.add_argument("--classifier", type=int, help="Algorithm id for a reporting-only coverage overlay.")
+        parser.add_argument("--name", help="TaxaList name (defaults to the region code).")
+        parser.add_argument(
+            "--include-uncovered",
+            action="store_true",
+            help="Also keep regional species no model can predict (each flagged as uncovered).",
+        )
+        parser.add_argument(
+            "--no-create-missing",
+            action="store_true",
+            help="Do not create Taxon rows for regional species absent from the database.",
+        )
+        parser.add_argument("--dry-run", action="store_true", help="Report the counts without writing anything.")
+
+    def handle(self, *args, **options):
+        classifier = self._resolve_classifier(options.get("classifier"))
+        common = dict(
+            region_source=options["region_source"],
+            classifier=classifier,
+            include_uncovered=options["include_uncovered"],
+            create_missing=not options["no_create_missing"],
+            name=options["name"],
+            dry_run=options["dry_run"],
+        )
+
+        if options["all_projects"]:
+            if options["region_code"]:
+                raise CommandError("--region-code is derived per project with --all-projects; do not pass it.")
+            self._run_all_projects(common)
+            return
+
+        if not options["region_code"]:
+            raise CommandError("--region-code is required unless --all-projects is used.")
+        project = self._resolve_project(options.get("project"))
+        result = regional_taxa.generate_regional_taxa_list(
+            project=project, region_code=options["region_code"], **common
+        )
+        self._report(project, result)
+
+    def _run_all_projects(self, common: dict) -> None:
+        for project in Project.objects.all().order_by("pk"):
+            derived = regional_taxa.derive_region_for_project(project, region_source=common["region_source"])
+            if derived is None:
+                self.stdout.write(f"[skip] project {project.pk} {project.name!r}: no region could be derived")
+                continue
+            _source, region_code = derived
+            result = regional_taxa.generate_regional_taxa_list(project=project, region_code=region_code, **common)
+            self._report(project, result)
+
+    def _resolve_project(self, project_id: int | None) -> Project | None:
+        if not project_id:
+            return None
+        try:
+            return Project.objects.get(pk=project_id)
+        except Project.DoesNotExist:
+            raise CommandError(f"Project {project_id} does not exist.")
+
+    def _resolve_classifier(self, classifier_id: int | None):
+        if not classifier_id:
+            return None
+        from ami.ml.models.algorithm import Algorithm
+
+        try:
+            return Algorithm.objects.get(pk=classifier_id)
+        except Algorithm.DoesNotExist:
+            raise CommandError(f"Algorithm {classifier_id} does not exist.")
+
+    def _report(self, project: Project | None, result) -> None:
+        scope = f"project {project.pk} {project.name!r}" if project else "global"
+        suffix = " [dry-run]" if result.dry_run else ""
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"[{scope}] region={result.region_code} saved={result.saved_list_size} "
+                f"(covered={result.model_covered}, uncovered={result.regional_no_model_coverage}, "
+                f"created={result.created_taxa}, in_db={result.already_in_db}, "
+                f"regional_total={result.regional_total}){suffix}"
+            )
+        )