Skip to content

Commit 170c3ab

Browse files
author
OpenMontage Review
committed
Merge reviewed stock cohort member PR OpenMontageInc#857
# Conflicts: # tests/tools/test_stock_source_adapters.py
2 parents 9971542 + fd2197b commit 170c3ab

1 file changed

Lines changed: 303 additions & 0 deletions

File tree

Lines changed: 303 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,303 @@
1+
"""Openverse open-media image adapter.
2+
3+
Openverse (https://openverse.org) is a WordPress Foundation project that
4+
federates openly licensed media from Flickr, Wikimedia Commons, museum
5+
collections (Met, NYPL, ...), and dozens of smaller archives behind one
6+
API. No account is required for anonymous use — the public endpoints
7+
allow 20 requests/minute and 200/day per IP.
8+
9+
Scope note: Openverse indexes **images and audio only**. There is no
10+
video endpoint (`/v1/video/` returns 404), so this adapter reports
11+
``supports = {"video": False, "image": True}`` and returns an empty list
12+
for video searches. Use ``wikimedia`` or ``archive_org`` for footage.
13+
14+
Licensing: Openverse holds works under many CC variants, including
15+
non-commercial (NC) and no-derivatives (ND) licenses that cannot be used
16+
in a monetised or edited video. This adapter therefore asks the API for
17+
``commercial,modification`` by default so every returned candidate is
18+
usable in a real edit. Override with ``OPENVERSE_LICENSE_TYPE`` if you
19+
deliberately want a wider (or narrower) pool.
20+
21+
Optional credentials: set ``OPENVERSE_CLIENT_ID`` and
22+
``OPENVERSE_CLIENT_SECRET`` (register an app at
23+
https://api.openverse.org/v1/auth_tokens/register/) to authenticate and
24+
lift the anonymous rate limits. Without them the adapter still works.
25+
"""
26+
from __future__ import annotations
27+
28+
import os
29+
import time
30+
from pathlib import Path
31+
from typing import Any
32+
33+
from .base import Candidate, SearchFilters
34+
35+
36+
_SEARCH_URL = "https://api.openverse.org/v1/images/"
37+
_TOKEN_URL = "https://api.openverse.org/v1/auth_tokens/token/"
38+
_USER_AGENT = "OpenMontage/0.1 (+https://github.com/calesthio/OpenMontage)"
39+
40+
# Some hosts behind Openverse (Flickr's CDN in particular) reject any
41+
# User-Agent containing "Bot" and expect an image Accept header. Both are
42+
# still honest identification — just not the string that trips the filter.
43+
_DOWNLOAD_HEADERS = {
44+
"User-Agent": _USER_AGENT,
45+
"Accept": "image/avif,image/webp,image/jpeg,image/png,*/*;q=0.8",
46+
}
47+
48+
# Anonymous access is capped at 20/min and 200/day. Anything wider than
49+
# this per call just wastes a request budget the corpus builder needs for
50+
# several queries.
51+
_MAX_PAGE_SIZE = 80
52+
53+
# Default to licenses that permit commercial use *and* derivative works —
54+
# the two flags every edited video needs.
55+
_DEFAULT_LICENSE_TYPE = "commercial,modification"
56+
57+
# Openverse "aspect_ratio" values differ from our filter vocabulary.
58+
_ASPECT_RATIO = {
59+
"landscape": "wide",
60+
"portrait": "tall",
61+
"square": "square",
62+
}
63+
64+
# Client-credentials tokens are valid for ~24h; refresh a minute early.
65+
_TOKEN_CACHE: dict[str, Any] = {"token": None, "expires_at": 0.0}
66+
67+
68+
class OpenverseSource:
69+
"""Adapter for Openverse image search (keyless, CC-licensed)."""
70+
71+
name = "openverse"
72+
display_name = "Openverse"
73+
provider = "openverse"
74+
priority = 21
75+
install_instructions = (
76+
"Works without an API key (anonymous: 20 req/min, 200 req/day). "
77+
"Optional: set OPENVERSE_CLIENT_ID and OPENVERSE_CLIENT_SECRET in "
78+
".env to raise the rate limit. Images only — Openverse has no video "
79+
"endpoint."
80+
)
81+
supports = {"video": False, "image": True}
82+
83+
def is_available(self) -> bool:
84+
# No key required; `requests` is a core dependency. Availability is
85+
# deliberately network-free so preflight stays fast and offline-safe.
86+
return True
87+
88+
def search(self, query: str, filters: SearchFilters) -> list[Candidate]:
89+
import requests # lazy
90+
91+
kind = (filters.kind or "video").lower()
92+
if kind == "video":
93+
# Openverse has no video index. Returning [] keeps the fan-out
94+
# quiet instead of raising a "source cannot do that" error.
95+
return []
96+
97+
params: dict[str, Any] = {
98+
"q": query,
99+
"page": max(1, filters.page),
100+
"page_size": max(1, min(filters.per_page, _MAX_PAGE_SIZE)),
101+
"license_type": _license_type(),
102+
"mature": "false",
103+
}
104+
aspect_ratio = _ASPECT_RATIO.get((filters.orientation or "").lower())
105+
if aspect_ratio:
106+
params["aspect_ratio"] = aspect_ratio
107+
if filters.min_width:
108+
# Coarse server-side hint; the precise width check happens in
109+
# `_result_to_candidate`.
110+
params["size"] = "large"
111+
112+
r = requests.get(
113+
_SEARCH_URL,
114+
params=params,
115+
headers=self._headers(),
116+
timeout=30,
117+
)
118+
r.raise_for_status()
119+
data = r.json()
120+
121+
out: list[Candidate] = []
122+
for result in data.get("results") or []:
123+
cand = _result_to_candidate(result, filters)
124+
if cand is not None:
125+
out.append(cand)
126+
return out
127+
128+
def download(self, candidate: Candidate, out_path: Path) -> Path:
129+
import requests # lazy
130+
131+
if not candidate.download_url:
132+
raise ValueError(f"Candidate {candidate.clip_id} has no download_url")
133+
134+
out_path = Path(out_path)
135+
out_path.parent.mkdir(parents=True, exist_ok=True)
136+
137+
# Openverse proxies many hosts (Flickr's CDN especially), and some
138+
# of them answer 403/429 when several files are pulled back to back.
139+
# A short backoff turns those transient throttles into successes
140+
# instead of dropping otherwise usable candidates.
141+
last_error: Exception | None = None
142+
for attempt in range(3):
143+
try:
144+
with requests.get(
145+
candidate.download_url,
146+
stream=True,
147+
timeout=180,
148+
headers=_DOWNLOAD_HEADERS,
149+
) as r:
150+
if r.status_code in (403, 429) and attempt < 2:
151+
raise requests.HTTPError(f"{r.status_code} throttled")
152+
r.raise_for_status()
153+
with open(out_path, "wb") as f:
154+
for chunk in r.iter_content(chunk_size=1 << 16):
155+
if chunk:
156+
f.write(chunk)
157+
return out_path
158+
except Exception as e: # noqa: BLE001 — re-raised below if final
159+
last_error = e
160+
if attempt < 2:
161+
time.sleep(0.6 * (2**attempt))
162+
raise last_error if last_error else RuntimeError("download failed")
163+
164+
def _headers(self) -> dict[str, str]:
165+
headers = {"User-Agent": _USER_AGENT, "Accept": "application/json"}
166+
token = _access_token()
167+
if token:
168+
headers["Authorization"] = f"Bearer {token}"
169+
return headers
170+
171+
172+
def _license_type() -> str:
173+
return (os.environ.get("OPENVERSE_LICENSE_TYPE") or _DEFAULT_LICENSE_TYPE).strip()
174+
175+
176+
def _access_token() -> str | None:
177+
"""Return a cached client-credentials token, or None when unconfigured.
178+
179+
Failures are non-fatal: anonymous access still works, so a bad secret
180+
should degrade rather than break a corpus build.
181+
"""
182+
client_id = os.environ.get("OPENVERSE_CLIENT_ID")
183+
client_secret = os.environ.get("OPENVERSE_CLIENT_SECRET")
184+
if not (client_id and client_secret):
185+
return None
186+
187+
now = time.monotonic()
188+
if _TOKEN_CACHE["token"] and now < _TOKEN_CACHE["expires_at"]:
189+
return _TOKEN_CACHE["token"]
190+
191+
import requests # lazy
192+
193+
try:
194+
r = requests.post(
195+
_TOKEN_URL,
196+
data={
197+
"grant_type": "client_credentials",
198+
"client_id": client_id,
199+
"client_secret": client_secret,
200+
},
201+
headers={"User-Agent": _USER_AGENT},
202+
timeout=30,
203+
)
204+
r.raise_for_status()
205+
payload = r.json()
206+
except Exception:
207+
return None
208+
209+
token = payload.get("access_token")
210+
if not token:
211+
return None
212+
expires_in = int(payload.get("expires_in") or 3600)
213+
_TOKEN_CACHE["token"] = token
214+
_TOKEN_CACHE["expires_at"] = now + max(60, expires_in - 60)
215+
return token
216+
217+
218+
def _result_to_candidate(
219+
result: dict[str, Any], filters: SearchFilters
220+
) -> Candidate | None:
221+
download_url = (result.get("url") or "").strip()
222+
if not download_url:
223+
return None
224+
225+
width = int(result.get("width") or 0)
226+
height = int(result.get("height") or 0)
227+
if filters.min_width is not None and width and width < filters.min_width:
228+
return None
229+
if filters.orientation and not _matches_orientation(
230+
filters.orientation, width, height
231+
):
232+
return None
233+
234+
title = (result.get("title") or "").strip()
235+
tag_names = " ".join(
236+
str(tag.get("name") or "")
237+
for tag in (result.get("tags") or [])
238+
if isinstance(tag, dict) and tag.get("name")
239+
)
240+
provider = (result.get("source") or result.get("provider") or "").strip()
241+
source_tags = " ".join(
242+
part for part in (title, tag_names, provider) if part
243+
).strip()[:500]
244+
245+
return Candidate(
246+
source=OpenverseSource.name,
247+
source_id=str(result.get("id") or ""),
248+
source_url=str(result.get("foreign_landing_url") or ""),
249+
download_url=download_url,
250+
kind="image",
251+
width=width,
252+
height=height,
253+
duration=0.0,
254+
creator=str(result.get("creator") or ""),
255+
license=_license_label(result),
256+
source_tags=source_tags,
257+
thumbnail_url=str(result.get("thumbnail") or download_url),
258+
extra={
259+
"provider": provider,
260+
"attribution": result.get("attribution") or "",
261+
"license_url": result.get("license_url") or "",
262+
"detail_url": result.get("detail_url") or "",
263+
"filesize": result.get("filesize"),
264+
"filetype": result.get("filetype"),
265+
"indexed_on": result.get("indexed_on"),
266+
"mature": result.get("mature"),
267+
},
268+
)
269+
270+
271+
def _license_label(result: dict[str, Any]) -> str:
272+
"""Human-readable licence string, e.g. ``CC BY-SA 4.0``.
273+
274+
`Candidate.license` may hold either a label or a URL; a label plus the
275+
canonical URL in `extra` keeps attribution cheap for the agent.
276+
"""
277+
code = str(result.get("license") or "").strip()
278+
version = str(result.get("license_version") or "").strip()
279+
if not code:
280+
return str(result.get("license_url") or "unknown")
281+
282+
lowered = code.lower()
283+
if lowered == "cc0":
284+
return f"CC0 {version}".strip()
285+
if lowered in {"pdm", "public domain mark"}:
286+
return "Public Domain Mark"
287+
if lowered in {"by", "by-sa", "by-nd", "by-nc", "by-nc-sa", "by-nc-nd"}:
288+
return f"CC {code.upper()} {version}".strip()
289+
# Unknown code — prefer the canonical URL when the API supplied one.
290+
return str(result.get("license_url") or f"CC {code.upper()} {version}".strip())
291+
292+
293+
def _matches_orientation(orientation: str, width: int, height: int) -> bool:
294+
if not width or not height:
295+
return True
296+
orientation = orientation.lower()
297+
if orientation == "landscape":
298+
return width >= height
299+
if orientation == "portrait":
300+
return height > width
301+
if orientation == "square":
302+
return abs(width - height) <= max(1, int(0.02 * max(width, height)))
303+
return True

0 commit comments

Comments
 (0)