diff --git a/backend/api/projects/osm_features.py b/backend/api/projects/osm_features.py new file mode 100644 index 0000000000..0ae7b03ff7 --- /dev/null +++ b/backend/api/projects/osm_features.py @@ -0,0 +1,74 @@ +import json + +from databases import Database +from fastapi import APIRouter, Depends +from fastapi.responses import JSONResponse + +from backend.db import get_db +from backend.exceptions import BadRequest, NotFound +from backend.services.overpass_service import OverpassService, OverpassServiceError + +router = APIRouter( + prefix="/projects", + tags=["projects"], + responses={404: {"description": "Not found"}}, +) + + +@router.get("/{project_id}/tasks/{task_id}/osm-features/") +async def get_osm_features_for_task( + project_id: int, + task_id: int, + db: Database = Depends(get_db), +): + """ + Fetch existing OSM features from the real OpenStreetMap within a task's boundary. + This endpoint is only available for sandbox projects and is intended for + data conflation in the sandbox environment. + + Returns a GeoJSON FeatureCollection of all tagged OSM features (nodes, ways) + within the task boundary. + """ + project = await db.fetch_one( + "SELECT id, sandbox FROM projects WHERE id = :project_id", + values={"project_id": project_id}, + ) + if not project: + raise NotFound( + sub_code="PROJECT_NOT_FOUND", + message=f"Project {project_id} not found.", + ) + if not project["sandbox"]: + raise BadRequest( + sub_code="NOT_SANDBOX_PROJECT", + message="This endpoint is only available for sandbox projects.", + ) + + task = await db.fetch_one( + """ + SELECT id, project_id, ST_AsGeoJSON(geometry) AS geojson + FROM tasks + WHERE id = :task_id + AND project_id = :project_id + """, + values={"task_id": task_id, "project_id": project_id}, + ) + if not task: + raise NotFound( + sub_code="TASK_NOT_FOUND", + message=f"Task {task_id} not found in project {project_id}.", + ) + + geometry_geojson = json.loads(task["geojson"]) + + try: + feature_collection = await OverpassService.fetch_osm_features_for_boundary( + geometry_geojson + ) + except OverpassServiceError as e: + raise BadRequest( + sub_code="OVERPASS_API_ERROR", + message=str(e.message), + ) + + return JSONResponse(content=feature_collection) diff --git a/backend/routes.py b/backend/routes.py index 72c8199084..e6198075fc 100644 --- a/backend/routes.py +++ b/backend/routes.py @@ -21,6 +21,7 @@ from backend.api.projects import campaigns as project_campaigns from backend.api.projects import contributions as project_contributions from backend.api.projects import favorites as project_favorites +from backend.api.projects import osm_features as project_osm_features from backend.api.projects import partnerships as project_partnerships from backend.api.projects import resources as project_resources from backend.api.projects import statistics as project_statistics @@ -55,6 +56,7 @@ def add_api_end_points(api): v2.include_router(project_actions.router) v2.include_router(project_favorites.router) v2.include_router(project_partnerships.router) + v2.include_router(project_osm_features.router) # Comments REST endpoint v2.include_router(comment_resources.router) diff --git a/backend/services/overpass_service.py b/backend/services/overpass_service.py new file mode 100644 index 0000000000..a4a5b21a10 --- /dev/null +++ b/backend/services/overpass_service.py @@ -0,0 +1,167 @@ +import geojson +import httpx +from loguru import logger +from shapely.geometry import shape + +from backend.config import settings + +OVERPASS_API_URL = "https://overpass-api.de/api/interpreter" + + +class OverpassServiceError(Exception): + def __init__(self, message): + self.message = message + logger.error(f"OverpassServiceError: {message}") + super().__init__(message) + + +class OverpassService: + @staticmethod + def _build_query(geometry_geojson: dict, timeout: int = 60) -> str: + """ + Build an Overpass QL query that fetches all OSM features (nodes, ways, + relations) within the given polygon boundary. + """ + geom = shape(geometry_geojson) + + if geom.geom_type == "MultiPolygon": + polygon = max(geom.geoms, key=lambda g: g.area) + else: + polygon = geom + + simplified = polygon.simplify(0.0001, preserve_topology=True) + coords = list(simplified.exterior.coords) + + poly_str = " ".join(f"{lat} {lon}" for lon, lat in coords) + + query = f""" +[out:json][timeout:{timeout}]; +( + nwr(poly:"{poly_str}"); +); +out body; +>; +out skel qt; +""" + return query + + @staticmethod + def _overpass_elements_to_geojson(data: dict) -> geojson.FeatureCollection: + """ + Convert Overpass JSON response to a GeoJSON FeatureCollection. + Handles nodes, ways, and basic relations. + """ + elements = data.get("elements", []) + + nodes = {} + ways = {} + features = [] + + for elem in elements: + if elem["type"] == "node": + nodes[elem["id"]] = elem + + for elem in elements: + if elem["type"] == "way": + ways[elem["id"]] = elem + + for elem in elements: + feature = None + if elem["type"] == "node" and "tags" in elem: + feature = geojson.Feature( + geometry=geojson.Point([elem["lon"], elem["lat"]]), + properties={ + "osm_id": elem["id"], + "osm_type": "node", + **(elem.get("tags", {})), + }, + ) + elif elem["type"] == "way" and "tags" in elem: + coords = [] + for nd_ref in elem.get("nodes", []): + nd = nodes.get(nd_ref) + if nd: + coords.append([nd["lon"], nd["lat"]]) + + if len(coords) < 2: + continue + + if len(coords) >= 4 and coords[0] == coords[-1]: + feature = geojson.Feature( + geometry=geojson.Polygon([coords]), + properties={ + "osm_id": elem["id"], + "osm_type": "way", + **(elem.get("tags", {})), + }, + ) + else: + feature = geojson.Feature( + geometry=geojson.LineString(coords), + properties={ + "osm_id": elem["id"], + "osm_type": "way", + **(elem.get("tags", {})), + }, + ) + elif elem["type"] == "relation" and "tags" in elem: + feature = geojson.Feature( + geometry=None, + properties={ + "osm_id": elem["id"], + "osm_type": "relation", + **(elem.get("tags", {})), + }, + ) + + if feature: + features.append(feature) + + return geojson.FeatureCollection(features) + + @staticmethod + async def fetch_osm_features_for_boundary( + geometry_geojson: dict, + timeout: int = 60, + ) -> geojson.FeatureCollection: + """ + Fetch all OSM features within the given boundary polygon from the + Overpass API and return as a GeoJSON FeatureCollection. + + :param geometry_geojson: A GeoJSON geometry (Polygon or MultiPolygon) + :param timeout: Overpass query timeout in seconds + :returns: GeoJSON FeatureCollection of OSM features + :raises OverpassServiceError: If the Overpass API request fails + """ + query = OverpassService._build_query(geometry_geojson, timeout=timeout) + + async with httpx.AsyncClient(timeout=timeout + 10) as client: + response = await client.post( + OVERPASS_API_URL, + data={"data": query}, + headers={ + "Accept": "application/json", + "Content-Type": "application/x-www-form-urlencoded", + "User-Agent": settings.OSM_USER_AGENT, + }, + ) + + if response.status_code == 429: + raise OverpassServiceError( + "Overpass API rate limit exceeded. Please try again later." + ) + if response.status_code == 504: + raise OverpassServiceError( + "Overpass API query timed out. The task boundary may be too large." + ) + if response.status_code != 200: + raise OverpassServiceError( + f"Overpass API returned status {response.status_code}: {response.text[:200]}" + ) + + try: + data = response.json() + except Exception: + raise OverpassServiceError("Failed to parse Overpass API response as JSON.") + + return OverpassService._overpass_elements_to_geojson(data) diff --git a/frontend/src/api/projects.js b/frontend/src/api/projects.js index 3fcb00c3cf..7dcf883acc 100644 --- a/frontend/src/api/projects.js +++ b/frontend/src/api/projects.js @@ -238,6 +238,23 @@ export const useAllPartnersQuery = (token, userId) => { }); }; +export const useOsmFeaturesQuery = (projectId, taskId, enabled = true) => { + const token = useSelector((state) => state.auth.token); + + const fetchOsmFeatures = ({ signal }) => { + return api(token).get(`projects/${projectId}/tasks/${taskId}/osm-features/`, { + signal, + }); + }; + + return useQuery({ + queryKey: ['osm-features', projectId, taskId], + queryFn: fetchOsmFeatures, + select: (data) => data.data, + enabled: !!(enabled && projectId && taskId), + }); +}; + const backendToQueryConversion = { difficulty: 'difficulty', campaign: 'campaign', diff --git a/frontend/src/components/sandboxEditor.js b/frontend/src/components/sandboxEditor.js index ce6d268168..a5ba753532 100644 --- a/frontend/src/components/sandboxEditor.js +++ b/frontend/src/components/sandboxEditor.js @@ -1,9 +1,12 @@ import { useEffect, useState } from 'react'; import { useNavigate } from 'react-router-dom'; import { useSelector, useDispatch } from 'react-redux'; +import { useIntl } from 'react-intl'; +import { gpx } from '@tmcw/togeojson'; import * as iD from '@osm-sandbox/sandbox-id'; import '@osm-sandbox/sandbox-id/dist/iD.css'; +import messages from './messages'; import { getSandboxAuthToken, setSandboxAuthError, @@ -11,6 +14,7 @@ import { } from '../store/actions/auth'; import { useSandboxOAuthCallback } from '../hooks/UseSandboxOAuthCallback'; import { getValidTokenOrInitiateAuth, fetchSandboxLicense } from '../utils/sandboxUtils'; +import { useOsmFeaturesQuery } from '../api/projects'; export default function SandboxEditor({ setDisable, @@ -19,9 +23,14 @@ export default function SandboxEditor({ imagery, sandboxId, gpxUrl, + projectId, + taskId, + showOsmFeatures, + osmLayerOpacity, }) { const dispatch = useDispatch(); const navigate = useNavigate(); + const intl = useIntl(); const session = useSelector((state) => state.auth.session); const sandboxTokens = useSelector((state) => state.auth.sandboxTokens); const sandboxAuthError = useSelector((state) => state.auth.sandboxAuthError); @@ -30,6 +39,13 @@ export default function SandboxEditor({ const locale = useSelector((state) => state.preferences.locale); const [customImageryIsSet, setCustomImageryIsSet] = useState(false); const [isInitialized, setIsInitialized] = useState(false); + const [gpxGeojson, setGpxGeojson] = useState(null); + + const { data: osmFeatures } = useOsmFeaturesQuery( + projectId, + taskId, + !!sandboxId && showOsmFeatures, + ); useSandboxOAuthCallback(sandboxId); @@ -122,10 +138,6 @@ export default function SandboxEditor({ iDContext.init(); } - if (gpxUrl) { - iDContext.layers().layer('data').url(gpxUrl, '.gpx'); - } - iDContext.connection().switch({ url: tokenData.sandbox_api_url, access_token: tokenData.access_token, @@ -163,6 +175,89 @@ export default function SandboxEditor({ sandboxAuthStatus, ]); + useEffect(() => { + if (gpxUrl) { + fetch(gpxUrl) + .then((response) => response.text()) + .then((data) => { + let gpxData = new DOMParser().parseFromString(data, 'text/xml'); + let trkNode = gpxData.getElementsByTagName('trk')[0]; + if (trkNode) { + let nameNode = trkNode.childNodes[0]; + let id = nameNode.textContent.match(/\d+/g); + nameNode.textContent = intl.formatMessage(messages.gpxNameAttribute, { + projectId: id ? id[0] : projectId, + }); + } + setGpxGeojson(gpx(gpxData)); + }) + .catch((error) => { + console.error('Error loading GPX data'); + }); + } + }, [gpxUrl, intl, projectId]); + + useEffect(() => { + if (isInitialized && iDContext && (osmFeatures || gpxGeojson)) { + // Assign stable IDs to ensure iD can maintain hover/select states + const features = [ + ...(gpxGeojson?.features || []).map((f, i) => ({ + ...f, + id: f.id || `gpx-${i}`, + __layerID__: 'gpx-features', + })), + ...(showOsmFeatures && osmFeatures?.features ? osmFeatures.features : []).map((f, i) => ({ + ...f, + id: + f.id || + (f.properties?.osm_id ? `osm-${f.properties.osm_id}` : `osm-${i}`), + __layerID__: 'osm-features', + })), + ]; + + if (features.length > 0 || (gpxGeojson && !showOsmFeatures)) { + iDContext.layers().layer('data').geojson({ + type: 'FeatureCollection', + features: features, + }); + } + } + }, [isInitialized, iDContext, osmFeatures, gpxGeojson, showOsmFeatures]); + + useEffect(() => { + if (!isInitialized || !iDContext) return; + const container = document.getElementById('id-container'); + if (container) { + const dataLayer = container.querySelector('.layer-data'); + if (dataLayer) { + dataLayer.style.opacity = ''; + } + } + + let styleEl = document.getElementById('osm-layer-opacity-style'); + if (!styleEl) { + styleEl = document.createElement('style'); + styleEl.id = 'osm-layer-opacity-style'; + document.head.appendChild(styleEl); + } + + styleEl.textContent = ` + #id-container .layer-data .osm-features { + opacity: ${osmLayerOpacity ?? 1} !important; + } + #id-container .layer-data .gpx-features { + opacity: 1 !important; + } + `; + + return () => { + const el = document.getElementById('osm-layer-opacity-style'); + if (el) { + el.remove(); + } + }; + }, [isInitialized, iDContext, osmLayerOpacity]); + useEffect(() => { return () => { dispatch(setSandboxAuthStatus(sandboxId, 'idle')); @@ -201,5 +296,9 @@ export default function SandboxEditor({ ); } - return
; + return ( +
+
+
+ ); } diff --git a/frontend/src/components/taskSelection/OsmDataControls.js b/frontend/src/components/taskSelection/OsmDataControls.js new file mode 100644 index 0000000000..156e259562 --- /dev/null +++ b/frontend/src/components/taskSelection/OsmDataControls.js @@ -0,0 +1,125 @@ +import { FormattedMessage, useIntl } from 'react-intl'; +import messages from './messages'; +import { EyeIcon, LoadingIcon } from '../svgIcons'; + +export function OsmDataControls({ + showOsmFeatures, + setShowOsmFeatures, + osmLayerOpacity, + setOsmLayerOpacity, + isFetching, + isSuccess, + isError, +}) { + const intl = useIntl(); + + // 1. Fetching State (only show full screen loader when we don't have success cached data yet) + if (showOsmFeatures && isFetching && !isSuccess) { + return ( +
+ + + + +
+ ); + } + + // 2. Error State + if (showOsmFeatures && isError) { + return ( +
+
+

+ +

+

+ +

+
+ +
+ ); + } + + // 3. Loaded/Fetched State (Active) + if (showOsmFeatures && isSuccess) { + return ( +
+
+ + + + +
+
+

+ +

+
+ + + + setOsmLayerOpacity(e.target.value / 100)} + className="flex-auto" + style={{ cursor: 'pointer' }} + /> + + + +
+
+
+ ); + } + + // 4. Initial / Hidden State + return ( +
+
+

+ +

+

+ {isSuccess ? ( + + ) : ( + + )} +

+
+ +
+ ); +} diff --git a/frontend/src/components/taskSelection/action.js b/frontend/src/components/taskSelection/action.js index 172669ed08..82166a0fde 100644 --- a/frontend/src/components/taskSelection/action.js +++ b/frontend/src/components/taskSelection/action.js @@ -34,9 +34,10 @@ import { ActionTabsNav } from './actionTabsNav'; import { LockedTaskModalContent } from './lockedTasks'; import { SessionAboutToExpire, SessionExpired } from './extendSession'; import { MappingTypes } from '../mappingTypes'; -import { usePriorityAreasQuery, useTaskDetail } from '../../api/projects'; +import { usePriorityAreasQuery, useTaskDetail, useOsmFeaturesQuery } from '../../api/projects'; import OtherTabInfo from './OtherTabInfo'; import { InfoBox } from '../projectDetail/infoBox'; +import { OsmDataControls } from './OsmDataControls'; const Editor = lazy(() => import('../editor')); const RapidEditor = lazy(() => import('../rapidEditor')); @@ -91,6 +92,8 @@ export function TaskMapAction({ project, tasks, activeTasks, getTasks, action, e const [showMapChangesModal, setShowMapChangesModal] = useState(false); const [showSessionExpiringDialog, setShowSessionExpiringDialog] = useState(false); const [showSessionExpiredDialog, setSessionTimeExpiredDialog] = useState(false); + const [showOsmFeatures, setShowOsmFeatures] = useState(false); + const [osmLayerOpacity, setOsmLayerOpacity] = useState(1); const activeTask = activeTasks?.[0]; const timer = new Date(activeTask.lastUpdated); @@ -100,6 +103,11 @@ export function TaskMapAction({ project, tasks, activeTasks, getTasks, action, e const { data: priorityArea, isError: isPriorityAreaError } = usePriorityAreasQuery( project.projectId, ); + const { + isFetching: isOsmFetching, + isSuccess: isOsmSuccess, + isError: isOsmError, + } = useOsmFeaturesQuery(project.projectId, tasksIds[0], !!project.sandbox && showOsmFeatures); const contributors = taskDetail?.taskHistory ? getTaskContributors(taskDetail.taskHistory, userDetails.username) @@ -244,6 +252,10 @@ export function TaskMapAction({ project, tasks, activeTasks, getTasks, action, e imagery={formatImageryUrlCallback(project.imagery)} sandboxId={project.database} gpxUrl={getTaskGpxUrlCallback(project.projectId, tasksIds)} + projectId={project.projectId} + taskId={tasksIds[0]} + showOsmFeatures={showOsmFeatures} + osmLayerOpacity={osmLayerOpacity} /> ) : (
Rapid sandbox editor is under developemnt
@@ -451,6 +463,17 @@ export function TaskMapAction({ project, tasks, activeTasks, getTasks, action, e )} {activeSection === 'instructions' && ( <> + {project.sandbox && ( + + )} diff --git a/frontend/src/components/taskSelection/messages.js b/frontend/src/components/taskSelection/messages.js index 5582d59420..f3fa02a687 100644 --- a/frontend/src/components/taskSelection/messages.js +++ b/frontend/src/components/taskSelection/messages.js @@ -866,4 +866,64 @@ export default defineMessages({ id: 'project.detail.sandbox.tooltip', defaultMessage: 'This is a training project. Edits will not be saved to OpenStreetMap.', }, + osmDataControlsTitle: { + id: 'project.tasks.osm_data_controls.title', + defaultMessage: 'OSM Data Controls', + }, + osmDataControlsOpacityDescription: { + id: 'project.tasks.osm_data_controls.opacity.description', + defaultMessage: 'Adjust OSM data layer opacity', + }, + osmDataControlsOpacityHidden: { + id: 'project.tasks.osm_data_controls.opacity.hidden', + defaultMessage: '0% (Hidden)', + }, + osmDataControlsOpacitySolid: { + id: 'project.tasks.osm_data_controls.opacity.solid', + defaultMessage: '100% (Solid)', + }, + osmDataControlsFetching: { + id: 'project.tasks.osm_data_controls.fetching', + defaultMessage: 'Fetching OSM features...', + }, + osmDataControlsErrorTitle: { + id: 'project.tasks.osm_data_controls.error.title', + defaultMessage: 'Error Loading Features', + }, + osmDataControlsErrorDescription: { + id: 'project.tasks.osm_data_controls.error.description', + defaultMessage: 'Failed to fetch OSM features for this task.', + }, + osmDataControlsReset: { + id: 'project.tasks.osm_data_controls.reset', + defaultMessage: 'Reset', + }, + osmDataControlsLayerTitle: { + id: 'project.tasks.osm_data_controls.layer.title', + defaultMessage: 'OSM Features Layer', + }, + osmDataControlsLayerDescriptionInitial: { + id: 'project.tasks.osm_data_controls.layer.description.initial', + defaultMessage: 'Load and display existing OpenStreetMap features.', + }, + osmDataControlsLayerDescriptionReady: { + id: 'project.tasks.osm_data_controls.layer.description.ready', + defaultMessage: 'OSM features are loaded and ready.', + }, + osmDataControlsLoadButton: { + id: 'project.tasks.osm_data_controls.load.button', + defaultMessage: 'Load OSM Features', + }, + osmDataControlsShowButton: { + id: 'project.tasks.osm_data_controls.show.button', + defaultMessage: 'Show OSM Features', + }, + osmDataControlsHideTitle: { + id: 'project.tasks.osm_data_controls.hide.title', + defaultMessage: 'Hide OSM features', + }, + osmDataControlsShowTitle: { + id: 'project.tasks.osm_data_controls.show.title', + defaultMessage: 'Show OSM features', + }, }); diff --git a/tests/api/unit/services/test_overpass_service.py b/tests/api/unit/services/test_overpass_service.py new file mode 100644 index 0000000000..3eeee17eca --- /dev/null +++ b/tests/api/unit/services/test_overpass_service.py @@ -0,0 +1,45 @@ +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from backend.config import settings +from backend.services.overpass_service import OVERPASS_API_URL, OverpassService + + +@pytest.mark.anyio +async def test_fetch_osm_features_sets_osm_user_agent(): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"elements": []} + + geometry_geojson = { + "type": "Polygon", + "coordinates": [ + [ + [85.3, 27.7], + [85.31, 27.7], + [85.31, 27.71], + [85.3, 27.71], + [85.3, 27.7], + ] + ], + } + + with patch( + "backend.services.overpass_service.httpx.AsyncClient.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = mock_response + + await OverpassService.fetch_osm_features_for_boundary( + geometry_geojson, timeout=1 + ) + + mock_post.assert_awaited_once() + request_url = mock_post.call_args.args[0] + request_headers = mock_post.call_args.kwargs["headers"] + + assert request_url == OVERPASS_API_URL + assert request_headers["Accept"] == "application/json" + assert request_headers["Content-Type"] == "application/x-www-form-urlencoded" + assert request_headers["User-Agent"] == settings.OSM_USER_AGENT