Skip to content

Commit 93b1041

Browse files
isaacgountonclaude
andcommitted
fix(nara): call the v2 catalog API the adapter claims to wrap
Every NARA search failed. The adapter called `/api/v2/search` with `rows`/`offset` and read `results[]` — a shape the catalog no longer serves. That path now returns the catalog's HTML single-page app, so `r.json()` raised on the first request and the source contributed nothing to any corpus. The key was also documented as optional ("works without an API key, set NARA_API_KEY for higher rate limits") and `is_available()` returned True unconditionally. Without the header the API serves HTML rather than JSON, so a keyless install reported NARA as a live source that could never return a clip. It now gates on NARA_API_KEY like the other key-backed adapters, and the install text says why. Corrected against the live API: - endpoint `/api/v2/search` -> `/api/v2/records/search` - paging `rows` + `offset` -> `limit` + 1-based `page` (offset/from/start are silently ignored) - envelope `results[]` -> `body.hits.hits[]._source.record` - files `objects[].url` -> `digitalObjects[].objectUrl` - kind `mimeType` -> extension, with the `objectType` label ("Audio/Visual File (MP4)") as fallback - filter `type=moving-image` -> `typeOfMaterials`, the catalog's own facet value, so the video/image split happens server-side (47,075 hits -> 3,970 for "apollo" video) - also adds `availableOnline=true`: a record with no digitised file can never yield a candidate v2 exposes no dimensions or duration, so width/height/duration stay 0 rather than being invented, and `scopeAndContentNote` is gone — the description now lives across `generalNotes`, `shotList` and `subjects`, which `_describe` flattens into the text channel used for ranking. Result ordering is not stable across identical requests, so consecutive pages can overlap; noted at the call site rather than papered over. Verified against the live catalog: video and image searches both return downloadable public-domain candidates, and `page` tiles results exactly. Five tests pin the endpoint, the paging params, the kind filter, the envelope walk and the tag flattening. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QsqAJj445U3Uj9gmDTfXXV
1 parent 08e2151 commit 93b1041

2 files changed

Lines changed: 224 additions & 74 deletions

File tree

‎tests/tools/test_stock_source_adapters.py‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,3 +219,126 @@ def test_search_returns_empty_when_the_source_has_no_results(
219219
)
220220
== []
221221
)
222+
223+
224+
# ---------------------------------------------------------------------
225+
# NARA — the v2 catalog API.
226+
#
227+
# The adapter used to call `/api/v2/search` with `rows`/`offset` and read
228+
# `results[]`. That path now serves the catalog's HTML app rather than
229+
# JSON, so every search raised on `.json()`. These tests pin the shape of
230+
# the working endpoint so a silent regression back to v1 is caught here
231+
# rather than as an empty corpus.
232+
# ---------------------------------------------------------------------
233+
234+
_NARA_RECORD = {
235+
"naId": 7500,
236+
"title": "APOLLO 8 NASA MOON FOOTAGE",
237+
"generalNotes": ["Following is a general summary of scenes in the roll."],
238+
"shotList": "CU Radar antenna. Overall view of earth.",
239+
"subjects": [{"heading": "National Aeronautics and Space Administration."}],
240+
"digitalObjects": [
241+
{
242+
"objectId": "14869152",
243+
"objectFilename": "33050.mp4",
244+
"objectType": "Audio/Visual File (MP4)",
245+
"objectUrl": "https://catalog.archives.gov/media/33050.mp4",
246+
"objectFileSize": 12345,
247+
},
248+
{
249+
"objectId": "14869153",
250+
"objectFilename": "97-445.jpg",
251+
"objectType": "Image (JPG)",
252+
"objectUrl": "https://catalog.archives.gov/media/97-445.jpg",
253+
},
254+
],
255+
}
256+
257+
258+
class _NARAResponse:
259+
"""A 200 carrying one record in the v2 envelope."""
260+
261+
status_code = 200
262+
263+
def raise_for_status(self):
264+
return None
265+
266+
def json(self):
267+
return {"body": {"hits": {"hits": [{"_source": {"record": _NARA_RECORD}}]}}}
268+
269+
270+
def _nara_search(monkeypatch, kind, page=1):
271+
seen = {}
272+
273+
def fake_get(url, headers=None, params=None, timeout=None):
274+
seen["url"] = url
275+
seen["headers"] = headers or {}
276+
seen["params"] = params or {}
277+
return _NARAResponse()
278+
279+
_install_fake_transport(monkeypatch, fake_get)
280+
results = get_source("nara").search(
281+
"apollo", SearchFilters(kind=kind, per_page=5, page=page)
282+
)
283+
return seen, results
284+
285+
286+
def test_nara_requires_a_key():
287+
# v2 serves HTML without x-api-key, so "no key" is unavailable, not
288+
# merely rate-limited.
289+
source = get_source("nara")
290+
assert source.is_available() is False
291+
292+
293+
def test_nara_calls_the_v2_records_endpoint(monkeypatch):
294+
seen, _ = _nara_search(monkeypatch, "video", page=3)
295+
296+
assert seen["url"].endswith("/api/v2/records/search")
297+
assert seen["headers"]["x-api-key"] == "test-nara"
298+
# v2 paginates by 1-based `page`; offset/from/start are ignored.
299+
assert seen["params"]["page"] == 3
300+
assert "offset" not in seen["params"]
301+
assert seen["params"]["limit"] == 5
302+
assert seen["params"]["availableOnline"] == "true"
303+
304+
305+
def test_nara_pushes_the_kind_filter_server_side(monkeypatch):
306+
seen, _ = _nara_search(monkeypatch, "video")
307+
assert seen["params"]["typeOfMaterials"] == "Moving Images"
308+
309+
seen, _ = _nara_search(monkeypatch, "image")
310+
assert seen["params"]["typeOfMaterials"] == "Photographs and other Graphic Materials"
311+
312+
# "any" must not constrain the search to one material type.
313+
seen, _ = _nara_search(monkeypatch, "any")
314+
assert "typeOfMaterials" not in seen["params"]
315+
316+
317+
def test_nara_extracts_objects_from_the_v2_envelope(monkeypatch):
318+
_, videos = _nara_search(monkeypatch, "video")
319+
320+
assert len(videos) == 1
321+
clip = videos[0]
322+
assert clip.download_url == "https://catalog.archives.gov/media/33050.mp4"
323+
assert clip.kind == "video"
324+
assert clip.source_url == "https://catalog.archives.gov/id/7500"
325+
assert clip.source_id == "7500_14869152"
326+
# v2 reports no dimensions or duration; the Candidate says so rather
327+
# than inventing them.
328+
assert (clip.width, clip.height, clip.duration) == (0, 0, 0.0)
329+
330+
_, images = _nara_search(monkeypatch, "image")
331+
assert [i.kind for i in images] == ["image"]
332+
assert images[0].download_url.endswith("97-445.jpg")
333+
334+
335+
def test_nara_tags_flatten_the_scattered_prose_fields(monkeypatch):
336+
# v2 dropped scopeAndContentNote; the description now lives across
337+
# generalNotes, shotList and subjects.
338+
_, videos = _nara_search(monkeypatch, "video")
339+
tags = videos[0].source_tags
340+
341+
assert "APOLLO 8 NASA MOON FOOTAGE" in tags
342+
assert "general summary of scenes" in tags
343+
assert "Radar antenna" in tags
344+
assert "National Aeronautics" in tags

‎tools/video/stock_sources/nara.py‎

Lines changed: 101 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,16 @@
55
significant film and video holdings — all U.S. federal government work
66
and therefore public domain.
77
8-
No API key required for basic searching. For higher rate limits, email
9-
Catalog_API@nara.gov to request a key. Rate limit: ~10,000 queries per
10-
month per API key.
8+
An API key IS required: without an ``x-api-key`` header the catalog serves
9+
its HTML single-page app instead of JSON, so every search fails to parse.
10+
Request one from Catalog_API@nara.gov and set ``NARA_API_KEY``. Rate limit
11+
is ~10,000 queries per month per key.
1112
1213
Fetch pattern
1314
-------------
14-
Two-stage like NASA. The search endpoint returns metadata records. Each
15-
record may contain digital objects (files) in ``objects``. We follow
16-
those to find downloadable video files.
15+
Two-stage like NASA. ``/records/search`` returns metadata records under
16+
``body.hits.hits[]._source.record``; each record may carry ``digitalObjects``
17+
(files), which is what we follow to find downloadable media.
1718
1819
What NARA is good for
1920
---------------------
@@ -33,9 +34,20 @@
3334

3435
_log = logging.getLogger(__name__)
3536

36-
_SEARCH_URL = "https://catalog.archives.gov/api/v2/search"
37+
_SEARCH_URL = "https://catalog.archives.gov/api/v2/records/search"
3738
_LICENSE = "Public domain (U.S. federal government work)"
3839

40+
# The catalog's own facet values (aggregation "typeOfMaterials"), used to
41+
# push the video/image split server-side instead of over-fetching and
42+
# discarding. Anything else is left unfiltered.
43+
_MATERIAL_FILTER = {
44+
"video": "Moving Images",
45+
"image": "Photographs and other Graphic Materials",
46+
}
47+
48+
_VIDEO_EXT = ("mp4", "mov", "avi", "wmv", "mkv", "webm", "m4v", "mpg", "mpeg")
49+
_IMAGE_EXT = ("jpg", "jpeg", "png", "tif", "tiff", "gif", "bmp")
50+
3951

4052
class NARASource:
4153
"""U.S. National Archives adapter. Satisfies `StockSource`."""
@@ -45,41 +57,46 @@ class NARASource:
4557
provider = "nara"
4658
priority = 35
4759
install_instructions = (
48-
"NARA works without an API key. "
49-
"Set NARA_API_KEY in .env for higher rate limits."
60+
"NARA requires an API key: without one the catalog returns HTML, not JSON.\n"
61+
"Request a key from Catalog_API@nara.gov, then set NARA_API_KEY in .env."
5062
)
5163
supports = {"video": True, "image": True}
5264

5365
def is_available(self) -> bool:
54-
# NARA is always available (no key required)
55-
return True
66+
return bool(os.environ.get("NARA_API_KEY"))
5667

5768
def search(self, query: str, filters: SearchFilters) -> list[Candidate]:
5869
import requests
5970

71+
api_key = os.environ.get("NARA_API_KEY")
72+
if not api_key:
73+
raise RuntimeError(
74+
"NARA_API_KEY is not set. " + self.install_instructions
75+
)
76+
6077
kind = (filters.kind or "video").lower()
6178

6279
params: dict[str, Any] = {
6380
"q": query,
64-
"rows": max(1, min(filters.per_page, 50)),
65-
"offset": (max(1, filters.page) - 1) * filters.per_page,
81+
"limit": max(1, min(filters.per_page, 50)),
82+
# 1-based. The v2 API ignores offset/from/start entirely.
83+
#
84+
# Ordering is not stable: identical requests come back with
85+
# relevance ties shuffled, so consecutive pages can overlap or
86+
# skip a record. Nothing to fix here — de-dup downstream on
87+
# `source_id` if a caller walks several pages.
88+
"page": max(1, filters.page),
89+
# Records with no digitised file can never yield a candidate.
90+
"availableOnline": "true",
6691
}
67-
68-
# Filter by type if possible
69-
if kind == "video":
70-
params["type"] = "moving-image"
71-
elif kind == "image":
72-
params["type"] = "still-image"
73-
74-
headers: dict[str, str] = {}
75-
api_key = os.environ.get("NARA_API_KEY")
76-
if api_key:
77-
headers["x-api-key"] = api_key
92+
material = _MATERIAL_FILTER.get(kind)
93+
if material:
94+
params["typeOfMaterials"] = material
7895

7996
try:
8097
r = requests.get(
8198
_SEARCH_URL,
82-
headers=headers,
99+
headers={"x-api-key": api_key},
83100
params=params,
84101
timeout=30,
85102
)
@@ -91,53 +108,72 @@ def search(self, query: str, filters: SearchFilters) -> list[Candidate]:
91108
_log.warning("NARA search failed: %s", e)
92109
raise
93110

94-
results = data.get("results", []) or []
111+
hits = (
112+
(data.get("body") or {})
113+
.get("hits", {})
114+
.get("hits", [])
115+
) or []
116+
95117
out: list[Candidate] = []
118+
for hit in hits:
119+
record = ((hit.get("_source") or {}).get("record")) or {}
120+
out.extend(self._extract_candidates(record, kind, filters))
121+
return out
96122

97-
for item in results:
98-
candidates = self._extract_candidates(item, kind, filters)
99-
out.extend(candidates)
123+
@staticmethod
124+
def _describe(record: dict) -> str:
125+
"""Flatten the record's prose fields into one searchable tag blob.
100126
101-
return out
127+
v2 dropped ``scopeAndContentNote``; what description exists is spread
128+
across ``generalNotes`` (list of str), ``shotList`` (str) and
129+
``subjects`` (list of dicts with a ``heading``).
130+
"""
131+
parts: list[str] = [str(record.get("title") or "")]
132+
133+
notes = record.get("generalNotes")
134+
if isinstance(notes, list):
135+
parts += [str(n) for n in notes if isinstance(n, str)]
136+
137+
shot_list = record.get("shotList")
138+
if isinstance(shot_list, str):
139+
parts.append(shot_list)
140+
141+
subjects = record.get("subjects")
142+
if isinstance(subjects, list):
143+
parts += [
144+
str(s.get("heading"))
145+
for s in subjects
146+
if isinstance(s, dict) and s.get("heading")
147+
]
148+
149+
return " ".join(p for p in parts if p).strip()
102150

103151
def _extract_candidates(
104-
self, item: dict, kind: str, filters: SearchFilters
152+
self, record: dict, kind: str, filters: SearchFilters
105153
) -> list[Candidate]:
106154
"""Extract downloadable candidates from a NARA catalog record."""
107-
naid = str(item.get("naId", "") or "")
155+
naid = str(record.get("naId", "") or "")
108156
if not naid:
109157
return []
110158

111-
title = item.get("title", "") or ""
112-
description = item.get("scopeAndContentNote", "") or ""
113-
source_tags = f"{title} {description}".strip()
159+
source_tags = self._describe(record)
114160
source_url = f"https://catalog.archives.gov/id/{naid}"
115161

116-
# Look for digital objects
117-
objects = item.get("objects", []) or []
118-
if not objects:
119-
# Try alternate field names
120-
digital = item.get("digitalObjects", []) or []
121-
objects = digital
122-
123162
out: list[Candidate] = []
124-
for obj in objects:
125-
file_url = obj.get("url") or obj.get("fileUrl") or ""
163+
for obj in record.get("digitalObjects") or []:
164+
file_url = obj.get("objectUrl") or ""
126165
if not file_url:
127166
continue
128167

129-
# Determine kind from mime type or file extension
130-
mime = (obj.get("mimeType", "") or "").lower()
131-
ext = file_url.rsplit(".", 1)[-1].lower() if "." in file_url else ""
168+
# v2 has no mime type; it labels files like "Audio/Visual File (MP4)"
169+
# or "Image (JPG)". The extension is the reliable signal, with the
170+
# label as a fallback for the odd extensionless URL.
171+
label = str(obj.get("objectType") or "").lower()
172+
name = str(obj.get("objectFilename") or file_url)
173+
ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
132174

133-
is_video = (
134-
"video" in mime
135-
or ext in ("mp4", "mov", "avi", "wmv", "mkv", "webm")
136-
)
137-
is_image = (
138-
"image" in mime
139-
or ext in ("jpg", "jpeg", "png", "tif", "tiff", "gif")
140-
)
175+
is_video = ext in _VIDEO_EXT or (not ext and "audio/visual" in label)
176+
is_image = ext in _IMAGE_EXT or (not ext and "image" in label)
141177

142178
if kind == "video" and not is_video:
143179
continue
@@ -146,36 +182,27 @@ def _extract_candidates(
146182
if not is_video and not is_image:
147183
continue
148184

149-
candidate_kind = "video" if is_video else "image"
150-
width = int(obj.get("width") or 0)
151-
height = int(obj.get("height") or 0)
152-
duration = float(obj.get("duration") or 0)
153-
154-
# Duration filters (client-side)
155-
if candidate_kind == "video":
156-
if filters.min_duration and duration and duration < filters.min_duration:
157-
continue
158-
if filters.max_duration and duration and duration > filters.max_duration:
159-
continue
160-
185+
# v2 reports no dimensions or duration, so duration filters can only
186+
# be applied downstream once the file is probed. Leaving these at 0
187+
# keeps the Candidate honest rather than inventing values.
161188
out.append(
162189
Candidate(
163190
source=self.name,
164191
source_id=f"{naid}_{obj.get('objectId', len(out))}",
165192
source_url=source_url,
166193
download_url=file_url,
167-
kind=candidate_kind,
168-
width=width,
169-
height=height,
170-
duration=duration,
194+
kind="video" if is_video else "image",
195+
width=0,
196+
height=0,
197+
duration=0.0,
171198
creator="U.S. National Archives",
172199
license=_LICENSE,
173200
source_tags=source_tags,
174-
thumbnail_url=obj.get("thumbnailUrl", "") or "",
201+
thumbnail_url="",
175202
extra={
176203
"naId": naid,
177-
"mime": mime,
178-
"fileSize": obj.get("fileSize"),
204+
"objectType": obj.get("objectType"),
205+
"fileSize": obj.get("objectFileSize"),
179206
},
180207
)
181208
)

0 commit comments

Comments
 (0)