replace raw null and surrogate code points during tokenization

cssselect/parser.py

@@ -991,6 +991,9 @@ def _compile(pattern: str) -> MatchFunc:
 _sub_newline_escape = re.compile(r"\\(?:\n|\r\n|\r|\f)").sub
 _sub_string_control_char = re.compile(r"[\x00-\x1f\x7f]").sub
 
+# CSS Syntax Level 3, §3.3: fold a raw U+0000 or surrogate code point to U+FFFD.
+_sub_invalid_input_char = re.compile("[\x00\ud800-\udfff]").sub
+
 # Same as r'\1', but faster on CPython
 _replace_simple = operator.methodcaller("group", 1)
 
@@ -1048,6 +1051,8 @@ def _serialize_ident(value: str) -> str:
 
 
 def tokenize(s: str) -> Iterator[Token]:
+    # Preprocess the input stream (§3.3); the substitution is length-preserving.
+    s = _sub_invalid_input_char("\N{REPLACEMENT CHARACTER}", s)
     pos = 0
     len_s = len(s)
     while pos < len_s:

tests/test_cssselect.py

@@ -1085,6 +1085,22 @@ def test_unicode_escapes(self) -> None:
         assert unescape_ident(r"\41 B") == "AB"
         assert unescape_ident(r"\-foo") == "-foo"
 
+    def test_input_preprocessing(self) -> None:
+        # CSS Syntax §3.3: a raw U+0000 or surrogate code point (not an
+        # escape) is folded to U+FFFD before tokenizing.
+        css_to_xpath = GenericTranslator().css_to_xpath
+        assert css_to_xpath('*[aval="x\x00y"]') == (
+            "descendant-or-self::*[@aval = 'x�y']"
+        )
+        assert css_to_xpath('*[aval="x\ud800y"]') == (
+            "descendant-or-self::*[@aval = 'x�y']"
+        )
+        assert css_to_xpath(':contains("x\udfffy")') == (
+            "descendant-or-self::*[contains(., 'x�y')]"
+        )
+        # A raw NUL in an identifier becomes a valid U+FFFD name character.
+        assert str(next(tokenize("foo\x00bar"))) == "<IDENT 'foo�bar' at 0>"
+
     def test_xpath_pseudo_elements(self) -> None:
         class CustomTranslator(GenericTranslator):
             def xpath_pseudo_element(