Catch out-of-range numeric character references (#591)

CONTRIBUTORS.rst

@@ -14,6 +14,7 @@ bug report!
 * `John Beimler <http://john.beimler.org/>`_
 * `Beat Bolli <https://drbeat.li/>`_
 * `François Boulogne <http://www.sciunto.org/>`_
+* `Chris <https://github.com/ChrisJr404>`_
 * `Adrian Damian <https://death.andgravity.com/>`_
 * `Jason Diamond <http://injektilo.org/>`_
 * `Jakub Kuczys <https://github.com/jack1142>`_

changelog.d/20260818_233239_chrisjr404_charref_overflow.rst

@@ -0,0 +1,6 @@
+Fixed
+-----
+
+*   Stop out-of-range numeric character references like ``�``,
+    ``�``, and ``�`` from aborting the loose parser, and
+    keep them as literal text instead. (#591)

feedparser/mixin.py

@@ -373,7 +373,15 @@ def handle_charref(self, ref):
                 c = int(ref[1:], 16)
             else:
                 c = int(ref)
-            text = chr(c).encode("utf-8")
+            try:
+                text = chr(c).encode("utf-8")
+            except (ValueError, OverflowError):
+                # The code point is outside the valid Unicode range or is a
+                # lone surrogate that can't be encoded as UTF-8. Rather than
+                # letting the exception abort the whole parse, keep the
+                # reference as literal text, matching how the HTML processor
+                # handles out-of-range references.
+                text = "&#%s;" % ref
         self.elementstack[-1][2].append(text)
 
     def handle_entityref(self, ref):

tests/test_numeric_character_references.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import pytest
+
+import feedparser
+
+
+def _title_feed(charref: str) -> str:
+    return (
+        '<rss version="2.0"><channel>'
+        f"<item><title>before{charref}after</title></item>"
+        "</channel></rss>"
+    )
+
+
+@pytest.mark.parametrize(
+    "charref, expected",
+    [
+        # A value that overflows the Unicode range.
+        ("&#xFFFFFFFF;", "before&#xffffffff;after"),
+        # The first code point above the maximum (0x10FFFF).
+        ("&#x110000;", "before&#x110000;after"),
+        # A lone surrogate that can't be encoded as UTF-8.
+        ("&#xD800;", "before&#xd800;after"),
+        # The decimal spelling of an out-of-range value.
+        ("&#4294967295;", "before&#4294967295;after"),
+    ],
+)
+def test_out_of_range_charref_does_not_crash(charref, expected):
+    """Out-of-range numeric character references must not abort the parse.
+
+    Previously ``&#xFFFFFFFF;``, ``&#x110000;``, and ``&#xD800;`` raised
+    ``OverflowError``, ``ValueError``, and ``UnicodeEncodeError`` respectively
+    from the loose parser. See issue #591.
+    """
+
+    result = feedparser.parse(_title_feed(charref))
+    assert result.entries[0].title == expected
+
+
+def test_valid_charref_is_still_resolved():
+    """Well-formed references keep resolving to their character."""
+
+    result = feedparser.parse(_title_feed("&#160;"))
+    assert result.entries[0].title == "before\xa0after"