From a01e79388d5a08d5123307724dcf41d306f4822f Mon Sep 17 00:00:00 2001
From: "Chris (ChrisJr404)" <11917633+ChrisJr404@users.noreply.github.com>
Date: Tue, 18 Aug 2026 23:34:17 -0400
Subject: [PATCH] Catch out-of-range numeric character references in the loose
parser
The loose parser resolved every numeric character reference with
chr().encode("utf-8"), so a reference whose value overflowed the Unicode
range (�), landed above the maximum code point (�),
or was a lone surrogate (�) raised OverflowError, ValueError, or
UnicodeEncodeError and aborted the whole parse.
Wrap the conversion so those references fall back to their literal text,
which is how the HTML processor already handles them. Fixes #591.
---
CONTRIBUTORS.rst | 1 +
...818_233239_chrisjr404_charref_overflow.rst | 6 +++
feedparser/mixin.py | 10 ++++-
tests/test_numeric_character_references.py | 45 +++++++++++++++++++
4 files changed, 61 insertions(+), 1 deletion(-)
create mode 100644 changelog.d/20260818_233239_chrisjr404_charref_overflow.rst
create mode 100644 tests/test_numeric_character_references.py
diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst
index 5d9f94fe..79d13b29 100644
--- a/CONTRIBUTORS.rst
+++ b/CONTRIBUTORS.rst
@@ -14,6 +14,7 @@ bug report!
* `John Beimler `_
* `Beat Bolli `_
* `François Boulogne `_
+* `Chris `_
* `Adrian Damian `_
* `Jason Diamond `_
* `Jakub Kuczys `_
diff --git a/changelog.d/20260818_233239_chrisjr404_charref_overflow.rst b/changelog.d/20260818_233239_chrisjr404_charref_overflow.rst
new file mode 100644
index 00000000..8ce309a1
--- /dev/null
+++ b/changelog.d/20260818_233239_chrisjr404_charref_overflow.rst
@@ -0,0 +1,6 @@
+Fixed
+-----
+
+* Stop out-of-range numeric character references like ``�``,
+ ``�``, and ``�`` from aborting the loose parser, and
+ keep them as literal text instead. (#591)
diff --git a/feedparser/mixin.py b/feedparser/mixin.py
index 3acee1b1..f6c4ef56 100644
--- a/feedparser/mixin.py
+++ b/feedparser/mixin.py
@@ -373,7 +373,15 @@ def handle_charref(self, ref):
c = int(ref[1:], 16)
else:
c = int(ref)
- text = chr(c).encode("utf-8")
+ try:
+ text = chr(c).encode("utf-8")
+ except (ValueError, OverflowError):
+ # The code point is outside the valid Unicode range or is a
+ # lone surrogate that can't be encoded as UTF-8. Rather than
+ # letting the exception abort the whole parse, keep the
+ # reference as literal text, matching how the HTML processor
+ # handles out-of-range references.
+ text = "&#%s;" % ref
self.elementstack[-1][2].append(text)
def handle_entityref(self, ref):
diff --git a/tests/test_numeric_character_references.py b/tests/test_numeric_character_references.py
new file mode 100644
index 00000000..0fa76abe
--- /dev/null
+++ b/tests/test_numeric_character_references.py
@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import pytest
+
+import feedparser
+
+
+def _title_feed(charref: str) -> str:
+ return (
+ ''
+ f"- before{charref}after
"
+ ""
+ )
+
+
+@pytest.mark.parametrize(
+ "charref, expected",
+ [
+ # A value that overflows the Unicode range.
+ ("�", "before�after"),
+ # The first code point above the maximum (0x10FFFF).
+ ("�", "before�after"),
+ # A lone surrogate that can't be encoded as UTF-8.
+ ("�", "before�after"),
+ # The decimal spelling of an out-of-range value.
+ ("�", "before�after"),
+ ],
+)
+def test_out_of_range_charref_does_not_crash(charref, expected):
+ """Out-of-range numeric character references must not abort the parse.
+
+ Previously ``�``, ``�``, and ``�`` raised
+ ``OverflowError``, ``ValueError``, and ``UnicodeEncodeError`` respectively
+ from the loose parser. See issue #591.
+ """
+
+ result = feedparser.parse(_title_feed(charref))
+ assert result.entries[0].title == expected
+
+
+def test_valid_charref_is_still_resolved():
+ """Well-formed references keep resolving to their character."""
+
+ result = feedparser.parse(_title_feed(" "))
+ assert result.entries[0].title == "before\xa0after"