diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst index 5d9f94fe..79d13b29 100644 --- a/CONTRIBUTORS.rst +++ b/CONTRIBUTORS.rst @@ -14,6 +14,7 @@ bug report! * `John Beimler `_ * `Beat Bolli `_ * `François Boulogne `_ +* `Chris `_ * `Adrian Damian `_ * `Jason Diamond `_ * `Jakub Kuczys `_ diff --git a/changelog.d/20260818_233239_chrisjr404_charref_overflow.rst b/changelog.d/20260818_233239_chrisjr404_charref_overflow.rst new file mode 100644 index 00000000..8ce309a1 --- /dev/null +++ b/changelog.d/20260818_233239_chrisjr404_charref_overflow.rst @@ -0,0 +1,6 @@ +Fixed +----- + +* Stop out-of-range numeric character references like ``�``, + ``�``, and ``�`` from aborting the loose parser, and + keep them as literal text instead. (#591) diff --git a/feedparser/mixin.py b/feedparser/mixin.py index 3acee1b1..f6c4ef56 100644 --- a/feedparser/mixin.py +++ b/feedparser/mixin.py @@ -373,7 +373,15 @@ def handle_charref(self, ref): c = int(ref[1:], 16) else: c = int(ref) - text = chr(c).encode("utf-8") + try: + text = chr(c).encode("utf-8") + except (ValueError, OverflowError): + # The code point is outside the valid Unicode range or is a + # lone surrogate that can't be encoded as UTF-8. Rather than + # letting the exception abort the whole parse, keep the + # reference as literal text, matching how the HTML processor + # handles out-of-range references. + text = "&#%s;" % ref self.elementstack[-1][2].append(text) def handle_entityref(self, ref): diff --git a/tests/test_numeric_character_references.py b/tests/test_numeric_character_references.py new file mode 100644 index 00000000..0fa76abe --- /dev/null +++ b/tests/test_numeric_character_references.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import pytest + +import feedparser + + +def _title_feed(charref: str) -> str: + return ( + '' + f"before{charref}after" + "" + ) + + +@pytest.mark.parametrize( + "charref, expected", + [ + # A value that overflows the Unicode range. + ("�", "before�after"), + # The first code point above the maximum (0x10FFFF). + ("�", "before�after"), + # A lone surrogate that can't be encoded as UTF-8. + ("�", "before�after"), + # The decimal spelling of an out-of-range value. + ("�", "before�after"), + ], +) +def test_out_of_range_charref_does_not_crash(charref, expected): + """Out-of-range numeric character references must not abort the parse. + + Previously ``�``, ``�``, and ``�`` raised + ``OverflowError``, ``ValueError``, and ``UnicodeEncodeError`` respectively + from the loose parser. See issue #591. + """ + + result = feedparser.parse(_title_feed(charref)) + assert result.entries[0].title == expected + + +def test_valid_charref_is_still_resolved(): + """Well-formed references keep resolving to their character.""" + + result = feedparser.parse(_title_feed(" ")) + assert result.entries[0].title == "before\xa0after"