diff --git a/ly/lex/html.py b/ly/lex/html.py index 7def595..bfe04f0 100644 --- a/ly/lex/html.py +++ b/ly/lex/html.py @@ -95,8 +95,9 @@ class StringSQEnd(String, _token.StringEnd, _token.Leaver): rx = r"'" +# [0-9], not \d: a numeric character reference is ASCII digits only class EntityRef(_token.Character): - rx = r"\&(#\d+|#[xX][0-9A-Fa-f]+|[A-Za-z_:][\w.:_-]*);" + rx = r"\&(#[0-9]+|#[xX][0-9A-Fa-f]+|[A-Za-z_:][\w.:_-]*);" class LilyPondTag(Tag): diff --git a/ly/lex/lilypond.py b/ly/lex/lilypond.py index 65034d7..b1ff8b1 100644 --- a/ly/lex/lilypond.py +++ b/ly/lex/lilypond.py @@ -28,6 +28,9 @@ from . import _token from . import Parser, FallthroughParser +# digit patterns use [0-9]: \d also matches Unicode digits that are not legal +# LilyPond. \d in a negated class excludes those too, so it stays + # an identifier allowing letters and single hyphens in between re_identifier = r"[^\W\d_]+([_-][^\W\d_]+)*" @@ -45,7 +48,7 @@ re_duration = rf"(\\(maxima|longa|breve){re_identifier_end}|(1|2|4|8|16|32|64|128|256|512|1024|2048)(?!\d))" re_dot = r"\." -re_scaling = r"\*[\t ]*\d+(/\d+)?" +re_scaling = r"\*[\t ]*[0-9]+(/[0-9]+)?" @@ -72,15 +75,15 @@ class Value(_token.Item, _token.Numeric): class DecimalValue(Value): - rx = r"-?\d+(\.\d+)?" + rx = r"-?[0-9]+(\.[0-9]+)?" class IntegerValue(DecimalValue): - rx = r"\d+" + rx = r"[0-9]+" class Fraction(Value): - rx = r"\d+/\d+" + rx = r"[0-9]+/[0-9]+" class Delimiter(_token.Token): @@ -289,11 +292,11 @@ class ScriptAbbreviation(Articulation, _token.Leaver): class Fingering(Articulation, _token.Leaver): - rx = r"\d+" + rx = r"[0-9]+" class StringNumber(Articulation): - rx = r"\\\d+" + rx = r"\\[0-9]+" class Slur(_token.Token): @@ -379,7 +382,7 @@ class ChordSeparator(ChordItem): class ChordStepNumber(ChordItem): - rx = r"\d+[-+]?" + rx = r"[0-9]+[-+]?" class DotChord(ChordItem): @@ -571,7 +574,7 @@ def update_state(self, state): class TempoSeparator(Delimiter): - rx = r"[-~](?=\s*\d)" + rx = r"[-~](?=\s*[0-9])" class Partial(Command): @@ -740,7 +743,7 @@ class FigureBracket(Figure): class FigureStep(Figure): """A step figure number or the underscore.""" - rx = r"_|\d+" + rx = r"_|[0-9]+" class FigureAccidental(Figure): diff --git a/ly/lex/scheme.py b/ly/lex/scheme.py index f8d3e5a..9c31c7d 100644 --- a/ly/lex/scheme.py +++ b/ly/lex/scheme.py @@ -138,20 +138,21 @@ def test_match(cls, match): return match.group() in data.scheme_constants() +# [0-9], not \d, which also matches Unicode digits that are not legal LilyPond class Number(_token.Item, _token.Numeric): rx = (r"(" - r"-?\d+|" + r"-?[0-9]+|" r"#(b[0-1]+|o[0-7]+|x[0-9a-fA-F]+)|" r"[-+]inf.0|[-+]?nan.0" r")(?=$|[)\s])") class Fraction(Number): - rx = r"-?\d+/\d+(?=$|[)\s])" + rx = r"-?[0-9]+/[0-9]+(?=$|[)\s])" class Float(Number): - rx = r"-?((\d+(\.\d*)|\.\d+)(E\d+)?)(?=$|[)\s])" + rx = r"-?(([0-9]+(\.[0-9]*)|\.[0-9]+)(E[0-9]+)?)(?=$|[)\s])" class VectorStart(OpenParen): diff --git a/tests/test_lex.py b/tests/test_lex.py new file mode 100644 index 0000000..66ba613 --- /dev/null +++ b/tests/test_lex.py @@ -0,0 +1,61 @@ +"""Tests for the lexers.""" +import ly.document +import ly.lex._token +import ly.lex.html +import ly.lex.lilypond + + +DIGIT_TOKENS = (ly.lex._token.Numeric, + ly.lex.lilypond.Fingering, + ly.lex.lilypond.StringNumber, + ly.lex.lilypond.ChordStepNumber, + ly.lex.lilypond.FigureStep, + ly.lex.lilypond.TempoSeparator, + ly.lex.lilypond.Scaling) + +TO_FULLWIDTH = str.maketrans('0123456789', '0123456789') + +SOURCES = [ + '#(display 42)', + '#(x 1/2)', + '#(x 1.5)', + '\\tempo 4 = 60 - 80', + '\\time 3/4', + 'c4', + 'c-4', + '\\override NoteHead.font-size = 2', + '\\chords { c1:5 }', + '\\figures { <1> }', + '\\score { { c1*2 } }', + '\\score { \\new TabStaff { c4\\4 } }', +] + + +def digit_tokens(text, mode=None): + """Return the text of every token the lexer only reads as such because of + the digits in it.""" + doc = ly.document.Document(text, mode) + return [str(t) for block in doc + for t in doc.tokens(block) if isinstance(t, DIGIT_TOKENS)] + + +def test_ascii_digits_are_read(): + for source in SOURCES: + assert digit_tokens(source), source + + +def test_fullwidth_digits_are_not_read(): + for source in SOURCES: + fullwidth = source.translate(TO_FULLWIDTH) + assert digit_tokens(fullwidth) == [], fullwidth + + +def entity_refs(text): + doc = ly.document.Document(text, 'html') + return [str(t) for block in doc + for t in doc.tokens(block) if isinstance(t, ly.lex.html.EntityRef)] + + +def test_only_ascii_digits_form_a_numeric_entity(): + assert entity_refs('A') == ['A'] + assert entity_refs('A'.translate(TO_FULLWIDTH)) == []