|
| 1 | +"""Openverse open-media image adapter. |
| 2 | +
|
| 3 | +Openverse (https://openverse.org) is a WordPress Foundation project that |
| 4 | +federates openly licensed media from Flickr, Wikimedia Commons, museum |
| 5 | +collections (Met, NYPL, ...), and dozens of smaller archives behind one |
| 6 | +API. No account is required for anonymous use — the public endpoints |
| 7 | +allow 20 requests/minute and 200/day per IP. |
| 8 | +
|
| 9 | +Scope note: Openverse indexes **images and audio only**. There is no |
| 10 | +video endpoint (`/v1/video/` returns 404), so this adapter reports |
| 11 | +``supports = {"video": False, "image": True}`` and returns an empty list |
| 12 | +for video searches. Use ``wikimedia`` or ``archive_org`` for footage. |
| 13 | +
|
| 14 | +Licensing: Openverse holds works under many CC variants, including |
| 15 | +non-commercial (NC) and no-derivatives (ND) licenses that cannot be used |
| 16 | +in a monetised or edited video. This adapter therefore asks the API for |
| 17 | +``commercial,modification`` by default so every returned candidate is |
| 18 | +usable in a real edit. Override with ``OPENVERSE_LICENSE_TYPE`` if you |
| 19 | +deliberately want a wider (or narrower) pool. |
| 20 | +
|
| 21 | +Optional credentials: set ``OPENVERSE_CLIENT_ID`` and |
| 22 | +``OPENVERSE_CLIENT_SECRET`` (register an app at |
| 23 | +https://api.openverse.org/v1/auth_tokens/register/) to authenticate and |
| 24 | +lift the anonymous rate limits. Without them the adapter still works. |
| 25 | +""" |
| 26 | +from __future__ import annotations |
| 27 | + |
| 28 | +import os |
| 29 | +import time |
| 30 | +from pathlib import Path |
| 31 | +from typing import Any |
| 32 | + |
| 33 | +from .base import Candidate, SearchFilters |
| 34 | + |
| 35 | + |
| 36 | +_SEARCH_URL = "https://api.openverse.org/v1/images/" |
| 37 | +_TOKEN_URL = "https://api.openverse.org/v1/auth_tokens/token/" |
| 38 | +_USER_AGENT = "OpenMontage/0.1 (+https://github.com/calesthio/OpenMontage)" |
| 39 | + |
| 40 | +# Some hosts behind Openverse (Flickr's CDN in particular) reject any |
| 41 | +# User-Agent containing "Bot" and expect an image Accept header. Both are |
| 42 | +# still honest identification — just not the string that trips the filter. |
| 43 | +_DOWNLOAD_HEADERS = { |
| 44 | + "User-Agent": _USER_AGENT, |
| 45 | + "Accept": "image/avif,image/webp,image/jpeg,image/png,*/*;q=0.8", |
| 46 | +} |
| 47 | + |
| 48 | +# Anonymous access is capped at 20/min and 200/day. Anything wider than |
| 49 | +# this per call just wastes a request budget the corpus builder needs for |
| 50 | +# several queries. |
| 51 | +_MAX_PAGE_SIZE = 80 |
| 52 | + |
| 53 | +# Default to licenses that permit commercial use *and* derivative works — |
| 54 | +# the two flags every edited video needs. |
| 55 | +_DEFAULT_LICENSE_TYPE = "commercial,modification" |
| 56 | + |
| 57 | +# Openverse "aspect_ratio" values differ from our filter vocabulary. |
| 58 | +_ASPECT_RATIO = { |
| 59 | + "landscape": "wide", |
| 60 | + "portrait": "tall", |
| 61 | + "square": "square", |
| 62 | +} |
| 63 | + |
| 64 | +# Client-credentials tokens are valid for ~24h; refresh a minute early. |
| 65 | +_TOKEN_CACHE: dict[str, Any] = {"token": None, "expires_at": 0.0} |
| 66 | + |
| 67 | + |
| 68 | +class OpenverseSource: |
| 69 | + """Adapter for Openverse image search (keyless, CC-licensed).""" |
| 70 | + |
| 71 | + name = "openverse" |
| 72 | + display_name = "Openverse" |
| 73 | + provider = "openverse" |
| 74 | + priority = 21 |
| 75 | + install_instructions = ( |
| 76 | + "Works without an API key (anonymous: 20 req/min, 200 req/day). " |
| 77 | + "Optional: set OPENVERSE_CLIENT_ID and OPENVERSE_CLIENT_SECRET in " |
| 78 | + ".env to raise the rate limit. Images only — Openverse has no video " |
| 79 | + "endpoint." |
| 80 | + ) |
| 81 | + supports = {"video": False, "image": True} |
| 82 | + |
| 83 | + def is_available(self) -> bool: |
| 84 | + # No key required; `requests` is a core dependency. Availability is |
| 85 | + # deliberately network-free so preflight stays fast and offline-safe. |
| 86 | + return True |
| 87 | + |
| 88 | + def search(self, query: str, filters: SearchFilters) -> list[Candidate]: |
| 89 | + import requests # lazy |
| 90 | + |
| 91 | + kind = (filters.kind or "video").lower() |
| 92 | + if kind == "video": |
| 93 | + # Openverse has no video index. Returning [] keeps the fan-out |
| 94 | + # quiet instead of raising a "source cannot do that" error. |
| 95 | + return [] |
| 96 | + |
| 97 | + params: dict[str, Any] = { |
| 98 | + "q": query, |
| 99 | + "page": max(1, filters.page), |
| 100 | + "page_size": max(1, min(filters.per_page, _MAX_PAGE_SIZE)), |
| 101 | + "license_type": _license_type(), |
| 102 | + "mature": "false", |
| 103 | + } |
| 104 | + aspect_ratio = _ASPECT_RATIO.get((filters.orientation or "").lower()) |
| 105 | + if aspect_ratio: |
| 106 | + params["aspect_ratio"] = aspect_ratio |
| 107 | + if filters.min_width: |
| 108 | + # Coarse server-side hint; the precise width check happens in |
| 109 | + # `_result_to_candidate`. |
| 110 | + params["size"] = "large" |
| 111 | + |
| 112 | + r = requests.get( |
| 113 | + _SEARCH_URL, |
| 114 | + params=params, |
| 115 | + headers=self._headers(), |
| 116 | + timeout=30, |
| 117 | + ) |
| 118 | + r.raise_for_status() |
| 119 | + data = r.json() |
| 120 | + |
| 121 | + out: list[Candidate] = [] |
| 122 | + for result in data.get("results") or []: |
| 123 | + cand = _result_to_candidate(result, filters) |
| 124 | + if cand is not None: |
| 125 | + out.append(cand) |
| 126 | + return out |
| 127 | + |
| 128 | + def download(self, candidate: Candidate, out_path: Path) -> Path: |
| 129 | + import requests # lazy |
| 130 | + |
| 131 | + if not candidate.download_url: |
| 132 | + raise ValueError(f"Candidate {candidate.clip_id} has no download_url") |
| 133 | + |
| 134 | + out_path = Path(out_path) |
| 135 | + out_path.parent.mkdir(parents=True, exist_ok=True) |
| 136 | + |
| 137 | + # Openverse proxies many hosts (Flickr's CDN especially), and some |
| 138 | + # of them answer 403/429 when several files are pulled back to back. |
| 139 | + # A short backoff turns those transient throttles into successes |
| 140 | + # instead of dropping otherwise usable candidates. |
| 141 | + last_error: Exception | None = None |
| 142 | + for attempt in range(3): |
| 143 | + try: |
| 144 | + with requests.get( |
| 145 | + candidate.download_url, |
| 146 | + stream=True, |
| 147 | + timeout=180, |
| 148 | + headers=_DOWNLOAD_HEADERS, |
| 149 | + ) as r: |
| 150 | + if r.status_code in (403, 429) and attempt < 2: |
| 151 | + raise requests.HTTPError(f"{r.status_code} throttled") |
| 152 | + r.raise_for_status() |
| 153 | + with open(out_path, "wb") as f: |
| 154 | + for chunk in r.iter_content(chunk_size=1 << 16): |
| 155 | + if chunk: |
| 156 | + f.write(chunk) |
| 157 | + return out_path |
| 158 | + except Exception as e: # noqa: BLE001 — re-raised below if final |
| 159 | + last_error = e |
| 160 | + if attempt < 2: |
| 161 | + time.sleep(0.6 * (2**attempt)) |
| 162 | + raise last_error if last_error else RuntimeError("download failed") |
| 163 | + |
| 164 | + def _headers(self) -> dict[str, str]: |
| 165 | + headers = {"User-Agent": _USER_AGENT, "Accept": "application/json"} |
| 166 | + token = _access_token() |
| 167 | + if token: |
| 168 | + headers["Authorization"] = f"Bearer {token}" |
| 169 | + return headers |
| 170 | + |
| 171 | + |
| 172 | +def _license_type() -> str: |
| 173 | + return (os.environ.get("OPENVERSE_LICENSE_TYPE") or _DEFAULT_LICENSE_TYPE).strip() |
| 174 | + |
| 175 | + |
| 176 | +def _access_token() -> str | None: |
| 177 | + """Return a cached client-credentials token, or None when unconfigured. |
| 178 | +
|
| 179 | + Failures are non-fatal: anonymous access still works, so a bad secret |
| 180 | + should degrade rather than break a corpus build. |
| 181 | + """ |
| 182 | + client_id = os.environ.get("OPENVERSE_CLIENT_ID") |
| 183 | + client_secret = os.environ.get("OPENVERSE_CLIENT_SECRET") |
| 184 | + if not (client_id and client_secret): |
| 185 | + return None |
| 186 | + |
| 187 | + now = time.monotonic() |
| 188 | + if _TOKEN_CACHE["token"] and now < _TOKEN_CACHE["expires_at"]: |
| 189 | + return _TOKEN_CACHE["token"] |
| 190 | + |
| 191 | + import requests # lazy |
| 192 | + |
| 193 | + try: |
| 194 | + r = requests.post( |
| 195 | + _TOKEN_URL, |
| 196 | + data={ |
| 197 | + "grant_type": "client_credentials", |
| 198 | + "client_id": client_id, |
| 199 | + "client_secret": client_secret, |
| 200 | + }, |
| 201 | + headers={"User-Agent": _USER_AGENT}, |
| 202 | + timeout=30, |
| 203 | + ) |
| 204 | + r.raise_for_status() |
| 205 | + payload = r.json() |
| 206 | + except Exception: |
| 207 | + return None |
| 208 | + |
| 209 | + token = payload.get("access_token") |
| 210 | + if not token: |
| 211 | + return None |
| 212 | + expires_in = int(payload.get("expires_in") or 3600) |
| 213 | + _TOKEN_CACHE["token"] = token |
| 214 | + _TOKEN_CACHE["expires_at"] = now + max(60, expires_in - 60) |
| 215 | + return token |
| 216 | + |
| 217 | + |
| 218 | +def _result_to_candidate( |
| 219 | + result: dict[str, Any], filters: SearchFilters |
| 220 | +) -> Candidate | None: |
| 221 | + download_url = (result.get("url") or "").strip() |
| 222 | + if not download_url: |
| 223 | + return None |
| 224 | + |
| 225 | + width = int(result.get("width") or 0) |
| 226 | + height = int(result.get("height") or 0) |
| 227 | + if filters.min_width is not None and width and width < filters.min_width: |
| 228 | + return None |
| 229 | + if filters.orientation and not _matches_orientation( |
| 230 | + filters.orientation, width, height |
| 231 | + ): |
| 232 | + return None |
| 233 | + |
| 234 | + title = (result.get("title") or "").strip() |
| 235 | + tag_names = " ".join( |
| 236 | + str(tag.get("name") or "") |
| 237 | + for tag in (result.get("tags") or []) |
| 238 | + if isinstance(tag, dict) and tag.get("name") |
| 239 | + ) |
| 240 | + provider = (result.get("source") or result.get("provider") or "").strip() |
| 241 | + source_tags = " ".join( |
| 242 | + part for part in (title, tag_names, provider) if part |
| 243 | + ).strip()[:500] |
| 244 | + |
| 245 | + return Candidate( |
| 246 | + source=OpenverseSource.name, |
| 247 | + source_id=str(result.get("id") or ""), |
| 248 | + source_url=str(result.get("foreign_landing_url") or ""), |
| 249 | + download_url=download_url, |
| 250 | + kind="image", |
| 251 | + width=width, |
| 252 | + height=height, |
| 253 | + duration=0.0, |
| 254 | + creator=str(result.get("creator") or ""), |
| 255 | + license=_license_label(result), |
| 256 | + source_tags=source_tags, |
| 257 | + thumbnail_url=str(result.get("thumbnail") or download_url), |
| 258 | + extra={ |
| 259 | + "provider": provider, |
| 260 | + "attribution": result.get("attribution") or "", |
| 261 | + "license_url": result.get("license_url") or "", |
| 262 | + "detail_url": result.get("detail_url") or "", |
| 263 | + "filesize": result.get("filesize"), |
| 264 | + "filetype": result.get("filetype"), |
| 265 | + "indexed_on": result.get("indexed_on"), |
| 266 | + "mature": result.get("mature"), |
| 267 | + }, |
| 268 | + ) |
| 269 | + |
| 270 | + |
| 271 | +def _license_label(result: dict[str, Any]) -> str: |
| 272 | + """Human-readable licence string, e.g. ``CC BY-SA 4.0``. |
| 273 | +
|
| 274 | + `Candidate.license` may hold either a label or a URL; a label plus the |
| 275 | + canonical URL in `extra` keeps attribution cheap for the agent. |
| 276 | + """ |
| 277 | + code = str(result.get("license") or "").strip() |
| 278 | + version = str(result.get("license_version") or "").strip() |
| 279 | + if not code: |
| 280 | + return str(result.get("license_url") or "unknown") |
| 281 | + |
| 282 | + lowered = code.lower() |
| 283 | + if lowered == "cc0": |
| 284 | + return f"CC0 {version}".strip() |
| 285 | + if lowered in {"pdm", "public domain mark"}: |
| 286 | + return "Public Domain Mark" |
| 287 | + if lowered in {"by", "by-sa", "by-nd", "by-nc", "by-nc-sa", "by-nc-nd"}: |
| 288 | + return f"CC {code.upper()} {version}".strip() |
| 289 | + # Unknown code — prefer the canonical URL when the API supplied one. |
| 290 | + return str(result.get("license_url") or f"CC {code.upper()} {version}".strip()) |
| 291 | + |
| 292 | + |
| 293 | +def _matches_orientation(orientation: str, width: int, height: int) -> bool: |
| 294 | + if not width or not height: |
| 295 | + return True |
| 296 | + orientation = orientation.lower() |
| 297 | + if orientation == "landscape": |
| 298 | + return width >= height |
| 299 | + if orientation == "portrait": |
| 300 | + return height > width |
| 301 | + if orientation == "square": |
| 302 | + return abs(width - height) <= max(1, int(0.02 * max(width, height))) |
| 303 | + return True |
0 commit comments