Rir assignment check hardening

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 
 ## Unreleased
+### Added
+- `RdapBootstrapError` (subclass of `RdapNotFoundError`) raised on a self-bootstrap lookup miss, so callers can distinguish "no registry was queried" from an authoritative 404
+### Fixed
+- `RIRAssignmentLookup`: validate downloaded delegated-stats data (HTTP status + completeness against the file's summary count) and never overwrite a good cache with a failed/truncated fetch, so a partial download no longer makes allocated ASNs read as unassigned
+- `RIRAssignmentLookup`: cache freshness now uses elapsed seconds instead of floored `timedelta.days`, so `cache_days=1` refreshes at 24h rather than ~48h
+- `RIRAssignmentLookup.load_data`: a load that fails partway no longer memoizes partial state on the instance, so a retry re-attempts all downloads instead of silently serving a subset of RIRs
 
 
 ## 1.7.0

rdap/__init__.py

@@ -6,13 +6,20 @@
 from rdap.client import RdapClient
 
 # exceptions to base namespace
-from rdap.exceptions import RdapException, RdapNotFoundError
+from rdap.exceptions import (
+    RdapBootstrapError,
+    RdapException,
+    RdapNotFoundError,
+    RIRAssignmentError,
+)
 
 # objects to base namespace
 from rdap.objects import RdapAsn
 
 __all__ = [
+    "RIRAssignmentError",
     "RdapAsn",
+    "RdapBootstrapError",
     "RdapClient",
     "RdapException",
     "RdapNotFoundError",

rdap/assignment.py

@@ -1,9 +1,14 @@
 import datetime
+import logging
 import os
 from typing import ClassVar
 
 import requests
 
+from rdap.exceptions import RIRAssignmentError
+
+logger = logging.getLogger(__name__)
+
 
 class RIRAssignmentLookup:
     """Fetch RIR assignement status lists from ripe and lookup
@@ -44,6 +49,57 @@ def parse_data(self, line):
         except IndexError:
             return None
 
+    def expected_asn_count(self, text):
+        """Number of asn records the file declares in its summary line
+        (e.g. `ripencc|*|asn|*|48678|summary`), or `None` if absent. Compared
+        to the records actually present to detect a truncated download.
+        """
+        for line in text.splitlines():
+            parts = line.split("|")
+            if (
+                len(parts) >= 6
+                and parts[1] == "*"
+                and parts[2] == "asn"
+                and parts[5].strip() == "summary"
+            ):
+                try:
+                    return int(parts[4])
+                except ValueError:
+                    return None
+        return None
+
+    def count_asn_records(self, text):
+        """Count asn record lines (excludes version/summary/malformed lines)."""
+        count = 0
+        for line in text.splitlines():
+            parts = line.split("|")
+            # record lines: registry|cc|asn|start|value|date|status[|ext...];
+            # summary has 6 fields, version's 3rd field is a serial not "asn"
+            if len(parts) < 7 or parts[2] != "asn":
+                continue
+            count += 1
+        return count
+
+    def validate_content(self, source, text):
+        """Raise `RIRAssignmentError` if delegated-stats `text` is empty or
+        truncated. Completeness is only checked when a summary count is present
+        (`source` is a rir name or path, used for the error message).
+        """
+        if not text or not text.strip():
+            raise RIRAssignmentError(f"RIR data for {source} is empty")
+
+        actual = self.count_asn_records(text)
+        if actual == 0:
+            # non-empty but no asn records: e.g. a proxy/error page served 200
+            raise RIRAssignmentError(f"RIR data for {source} has no asn records")
+
+        expected = self.expected_asn_count(text)
+        if expected is not None and actual < expected:
+            raise RIRAssignmentError(
+                f"RIR data for {source} looks truncated: "
+                f"{actual} asn records present, summary declares {expected}"
+            )
+
     def load_data(self, data_path=".", cache_days=1):
         """Reads RIR assignment data into memory
 
@@ -61,23 +117,34 @@ def load_data(self, data_path=".", cache_days=1):
         - data_path (`str`): directory path to where downloaded files are to be saved
         - cache_days (`int`): maximum age of downloaded files before they will be
             downloaded again
+
+        Raises `RIRAssignmentError` rather than silently loading partial data
+        (which would make allocated ASNs read as unassigned).
         """
         if not hasattr(self, "_data_files"):
-            self._data_files = []
+            data_files = []
 
             for rir in self.rir_lists:
                 rir_file_path = os.path.join(
                     data_path,
                     f"delegated-{rir}-extended-latest",
                 )
                 self.download_data(rir, rir_file_path, cache_days)
-                self._data_files.append(rir_file_path)
+                data_files.append(rir_file_path)
+
+            # memoize only once every file downloaded/validated: a partial
+            # list would make a retry on this instance skip the download
+            # phase and serve a subset of RIRs, with their absent ASNs
+            # reading as unassigned
+            self._data_files = data_files
 
         if not hasattr(self, "_data"):
-            self._data = {}
+            data = {}
 
             for rir_file_path in self._data_files:
-                print(rir_file_path)
+                # download_data guarantees each file is present and valid (or
+                # raised), so parse directly here
+                logger.debug("loading RIR assignment data from %s", rir_file_path)
                 with open(rir_file_path) as fh:
                     for line in fh.read().splitlines():
                         asns = self.parse_data(line)
@@ -86,34 +153,84 @@ def load_data(self, data_path=".", cache_days=1):
                             continue
 
                         try:
-                            for data in asns:
-                                self._data[int(data["asn"])] = data["status"]
+                            for parsed in asns:
+                                data[int(parsed["asn"])] = parsed["status"]
                         except (TypeError, ValueError):
                             pass
 
+            # same rule as _data_files: memoize only the complete dataset
+            self._data = data
+
         return self._data
 
+    def _cache_expired(self, file_path, cache_days):
+        """True if `file_path` is missing or older than `cache_days`
+        (measured in elapsed seconds).
+        """
+        if not os.path.exists(file_path):
+            return True
+        age = datetime.datetime.now() - datetime.datetime.fromtimestamp(
+            os.path.getmtime(file_path)
+        )
+        return age.total_seconds() > cache_days * 86400
+
+    def _file_valid(self, rir, file_path):
+        """True if `file_path` exists and passes content validation."""
+        if not os.path.exists(file_path):
+            return False
+        try:
+            with open(file_path) as fh:
+                self.validate_content(rir, fh.read())
+        except (OSError, RIRAssignmentError):
+            return False
+        return True
+
     def download_data(self, rir, file_path, cache_days=1):
         """Download RIR network assignment status data from RIPE
-        https://ftp.ripe.net/pub/stats/{rir}/delegated-{rir}-extended-latesy
+        https://ftp.ripe.net/pub/stats/{rir}/delegated-{rir}-extended-latest
+
+        The body is checked (HTTP status + completeness) and written atomically.
+        A stale or corrupt cache is re-downloaded; a failed/truncated fetch never
+        clobbers a valid cache (kept if present, else `RIRAssignmentError`).
         """
-        # Only download/re-download the file if it doesn't exist
-        # or the file is older than cache_days
-        if (
-            not os.path.exists(file_path)
-            or (
-                datetime.datetime.now()
-                - datetime.datetime.fromtimestamp(os.path.getmtime(file_path))
-            ).days
-            > cache_days
+        # reuse a still-fresh cache only if it is actually valid; a stale or
+        # corrupt cache falls through and is re-downloaded
+        if not self._cache_expired(file_path, cache_days) and self._file_valid(
+            rir, file_path
         ):
-            url = (
-                f"https://ftp.ripe.net/pub/stats/{rir}/delegated-{rir}-extended-latest"
-            )
-            response = requests.get(url, timeout=30)
+            return
 
-            with open(file_path, "w") as file:
-                file.write(response.text)
+        url = f"https://ftp.ripe.net/pub/stats/{rir}/delegated-{rir}-extended-latest"
+
+        try:
+            response = requests.get(url, timeout=30)
+            response.raise_for_status()
+            text = response.text
+            self.validate_content(rir, text)
+        except (requests.RequestException, RIRAssignmentError) as exc:
+            # fall back to a valid cache rather than clobber it with a bad fetch
+            if self._file_valid(rir, file_path):
+                logger.warning(
+                    "RIR data fetch for %s failed (%s); keeping cached copy",
+                    rir,
+                    exc,
+                )
+                return
+            raise RIRAssignmentError(
+                f"could not fetch valid RIR data for {rir} and no usable cache exists: {exc}"
+            ) from exc
+
+        # write to a pid-unique temp file and atomically replace: concurrent
+        # runs can't interleave, an interrupted write can't leave a partial
+        # cache, and the temp file is cleaned up if the replace never happened
+        tmp_path = f"{file_path}.{os.getpid()}.tmp"
+        try:
+            with open(tmp_path, "w") as file:
+                file.write(text)
+            os.replace(tmp_path, file_path)
+        finally:
+            if os.path.exists(tmp_path):
+                os.remove(tmp_path)
 
     def get_status(self, asn):
         """Get RIR assignment status for an ASN"""

rdap/client.py

@@ -11,7 +11,7 @@
 import rdap.bootstrap
 from rdap.config import Config
 from rdap.context import RdapRequestContext
-from rdap.exceptions import RdapHTTPError, RdapNotFoundError
+from rdap.exceptions import RdapBootstrapError, RdapHTTPError, RdapNotFoundError
 from rdap.objects import RdapAsn, RdapDomain, RdapEntity, RdapHistory, RdapNetwork
 
 
@@ -282,9 +282,10 @@ def get_asn(self, asn):
         asn = int(asn)
         try:
             url = self.asn_url(asn)
-        # catch bootstrap Lookup errors and report as not found
+        # bootstrap has no service for this ASN -- no registry query was made,
+        # so this is a bootstrap miss, not an authoritative not-found
         except LookupError as excinfo:
-            raise RdapNotFoundError(str(excinfo))
+            raise RdapBootstrapError(str(excinfo)) from excinfo
 
         # save reqest to get url for following entity lookups
         query = f"/autnum/{asn}"

rdap/exceptions.py

@@ -8,3 +8,26 @@ class RdapHTTPError(RdapException):
 
 class RdapNotFoundError(RdapHTTPError):
     """RDAP query returned 404 Not Found."""
+
+
+class RdapBootstrapError(RdapNotFoundError):
+    """No RDAP service could be resolved for the resource from bootstrap data.
+
+    Raised when self-bootstrap lookup finds no service for an ASN -- i.e.
+    no registry query was made at all, which is NOT the same as an authoritative
+    404. A newly-assigned block can be absent from bootstrap for a while, so
+    callers must not treat this as proof the resource does not exist.
+
+    Subclass of RdapNotFoundError for backwards compatibility (existing
+    ``except RdapNotFoundError`` handlers still catch it); catch this first to
+    distinguish "never queried" from "registry said no".
+    """
+
+
+class RIRAssignmentError(RdapException):
+    """RIR delegated-stats data could not be fetched or is incomplete/corrupt.
+
+    Raised instead of caching/loading a bad file, since consumers treat an
+    absent ASN as 'unassigned' -- a truncated file would make live ASNs look
+    reclaimed.
+    """

tests/data/assignment/sample-complete

@@ -0,0 +1,6 @@
+2|ripencc|1783720799|4|19700101|20260710|+0200
+ripencc|*|asn|*|4|summary
+ripencc|NL|asn|100|1|20200101|allocated|uuid-a
+ripencc|DE|asn|101|1|20200101|allocated|uuid-b
+ripencc|IN|asn|219272|1|20260706|allocated|uuid-c
+ripencc|SG|asn|300|5|20200101|assigned|uuid-d

tests/data/assignment/sample-truncated

@@ -0,0 +1,3 @@
+2|ripencc|1783720799|3|19700101|20260710|+0200
+ripencc|*|asn|*|3|summary
+ripencc|NL|asn|100|1|20200101|allocated|uuid-a

tests/data/iana/bootstrap/asn.json

@@ -1 +1 @@
-{"description":"RDAP bootstrap file for Autonomous System Number allocations","publication":"2025-01-17T20:00:02Z","services":[[["36864-37887","327680-328703","328704-329727"],["https://rdap.afrinic.net/rdap/","http://rdap.afrinic.net/rdap/"]],[["4608-4865","7467-7722","9216-10239","17408-18431","23552-24575","37888-38911","45056-46079","55296-56319","58368-59391","63488-63999","64000-64098","64297-64395","131072-132095","132096-133119","133120-133631","133632-134556","134557-135580","135581-136505","136506-137529","137530-138553","138554-139577","139578-140601","140602-141625","141626-142649","142650-143673","143674-144697","144698-145721","145722-146745","146746-147769","147770-148793","148794-149817","149818-150841","150842-151865","151866-152889","152890-153913","153914-154937","154938-155961"],["https://rdap.apnic.net/"]],[["1-1876","1902-2042","2044-2046","2048-2106","2137-2584","2615-2772","2823-2829","2880-3153","3354-4607","4866-5376","5632-6655","6912-7466","7723-8191","10240-12287","13312-15359","16384-17407","18432-20479","21504-23455","23457-23551","25600-26623","26624-27647","29696-30719","31744-32767","32768-33791","35840-36863","39936-40959","46080-47103","53248-54271","54272-55295","62464-63487","64198-64296","393216-394239","394240-395164","395165-396188","396189-397212","397213-398236","398237-399260","399261-400284","400285-401308","401309-402332"],["https://rdap.arin.net/registry/","http://rdap.arin.net/registry/"]],[["1877-1901","2043","2047","2107-2136","2585-2614","2773-2822","2830-2879","3154-3353","5377-5631","6656-6911","8192-9215","12288-13311","15360-16383","20480-21503","24576-25599","28672-29695","30720-31743","33792-34815","34816-35839","38912-39935","40960-41983","41984-43007","43008-44031","44032-45055","47104-48127","48128-49151","49152-50175","50176-51199","51200-52223","56320-57343","57344-58367","59392-60415","60416-61439","61952-62463","64396-64495","196608-197631","197632-198655","198656-199679","199680-200191","200192-201215","201216-202239","202240-203263","203264-204287","204288-205211","205212-206235","206236-207259","207260-208283","208284-209307","209308-210331","210332-211355","211356-212379","212380-213403","213404-214427","214428-215451","215452-216475"],["https://rdap.db.ripe.net/"]],[["27648-28671","52224-53247","61440-61951","64099-64197","262144-263167","263168-263679","263680-264604","264605-265628","265629-266652","266653-267676","267677-268700","268701-269724","269725-270748","270749-271772","271773-272796","272797-273820","273821-274844"],["https://rdap.lacnic.net/rdap/"]]],"version":"1.0"}
+{"description":"RDAP bootstrap file for Autonomous System Number allocations","publication":"2026-06-01T20:00:01Z","services":[[["36864-37887","327680-328703","328704-329727","329728-330751"],["https://rdap.afrinic.net/rdap/","http://rdap.afrinic.net/rdap/"]],[["4608-4865","7467-7722","9216-10239","17408-18431","23552-24575","37888-38911","45056-46079","55296-56319","58368-59391","63488-63999","64000-64098","64297-64395","131072-132095","132096-133119","133120-133631","133632-134556","134557-135580","135581-136505","136506-137529","137530-138553","138554-139577","139578-140601","140602-141625","141626-142649","142650-143673","143674-144697","144698-145721","145722-146745","146746-147769","147770-148793","148794-149817","149818-150841","150842-151865","151866-152889","152890-153913","153914-154937","154938-155961"],["https://rdap.apnic.net/"]],[["1-1876","1902-2042","2044-2046","2048-2106","2137-2584","2615-2772","2823-2829","2880-3153","3354-4607","4866-5376","5632-6655","6912-7466","7723-8191","10240-12287","13312-15359","16384-17407","18432-20479","21504-23455","23457-23551","25600-26623","26624-27647","29696-30719","31744-32767","32768-33791","35840-36863","39936-40959","46080-47103","53248-54271","54272-55295","62464-63487","64198-64296","393216-394239","394240-395164","395165-396188","396189-397212","397213-398236","398237-399260","399261-400284","400285-401308","401309-402332","402333-403356","403357-404380"],["https://rdap.arin.net/registry/","http://rdap.arin.net/registry/"]],[["1877-1901","2043","2047","2107-2136","2585-2614","2773-2822","2830-2879","3154-3353","5377-5631","6656-6911","8192-9215","12288-13311","15360-16383","20480-21503","24576-25599","28672-29695","30720-31743","33792-34815","34816-35839","38912-39935","40960-41983","41984-43007","43008-44031","44032-45055","47104-48127","48128-49151","49152-50175","50176-51199","51200-52223","56320-57343","57344-58367","59392-60415","60416-61439","61952-62463","64396-64495","196608-197631","197632-198655","198656-199679","199680-200191","200192-201215","201216-202239","202240-203263","203264-204287","204288-205211","205212-206235","206236-207259","207260-208283","208284-209307","209308-210331","210332-211355","211356-212379","212380-213403","213404-214427","214428-215451","215452-216475","216476-217499","217500-218523","218524-219547"],["https://rdap.db.ripe.net/"]],[["27648-28671","52224-53247","61440-61951","64099-64197","262144-263167","263168-263679","263680-264604","264605-265628","265629-266652","266653-267676","267677-268700","268701-269724","269725-270748","270749-271772","271773-272796","272797-273820","273821-274844","274845-275868"],["https://rdap.lacnic.net/rdap/"]]],"version":"1.0"}

tests/test_asn.py

@@ -1,7 +1,10 @@
+from unittest.mock import patch
+
 import pytest
 import pytest_filedata
 
 from rdap import RdapAsn, RdapNotFoundError
+from rdap.exceptions import RdapBootstrapError
 
 
 def assert_parsed(data, parsed):
@@ -25,6 +28,18 @@ def test_rdap_asn_lookup_not_found(rdapc):
         rdapc.get_asn(65535)
 
 
+def test_rdap_asn_bootstrap_miss_raises_bootstrap_error(rdapc):
+    # a bootstrap lookup miss (no service resolved -> no registry query) must
+    # raise RdapBootstrapError, not be indistinguishable from an authoritative
+    # 404. It stays a RdapNotFoundError subclass for backwards compatibility.
+    assert issubclass(RdapBootstrapError, RdapNotFoundError)
+    with (
+        patch.object(rdapc, "asn_url", side_effect=LookupError("no service")),
+        pytest.raises(RdapBootstrapError),
+    ):
+        rdapc.get_asn(219273)
+
+
 @pytest.mark.network
 def test_rdap_asn_lookup_no_client(rdapc):
     asn = rdapc.get_asn(63311)