diff --git a/code/shared/src/main/scala/maf/language/sexp/SExpParser.scala b/code/shared/src/main/scala/maf/language/sexp/SExpParser.scala index 3d9c47e30..e44cd5849 100644 --- a/code/shared/src/main/scala/maf/language/sexp/SExpParser.scala +++ b/code/shared/src/main/scala/maf/language/sexp/SExpParser.scala @@ -85,7 +85,24 @@ class SExpLexer extends Lexical with SExpTokens: def eol: Parser[Any] = acceptIf(n => n == '\n')(n => "") def notEol: Parser[Char] = acceptIf(n => n != '\n')(n => "") def comment: Parser[String] = ';' ~> rep(notEol) ^^ (_.mkString) - def nonRelevant: Parser[Unit] = rep(comment | whitespaceChar | eol) ^^ (_ => ()) + + def multiLineComment: Parser[String] = + ('#' ~ '|' ~> rep(multiLineComment | not('|' ~ '#') ~> any) <~ '|' <~ '#') ^^ (_.mkString) + + def datumSkipper: Parser[Unit] = ( + (boolean | number | character | string | dot) ^^ (_ => ()) + | leftParen ~> rep(datumSkipperWrapper) <~ rightParen ^^ (_ => ()) + | leftBracket ~> rep(datumSkipperWrapper) <~ rightBracket ^^ (_ => ()) + | hashParen ~> rep(datumSkipperWrapper) <~ rightParen ^^ (_ => ()) + | quotes ~> datumSkipperWrapper ^^ (_ => ()) + | identifier ^^ (_ => ()) + ) + def datumSkipperWrapper: Parser[Unit] = nonRelevant ~> datumSkipper <~ nonRelevant + def datumComment: Parser[Unit] = '#' ~ ';' ~> datumSkipperWrapper + + def hashBangComment: Parser[String] = '#' ~ '!' ~> rep(notEol) ^^ (_.mkString) + + def nonRelevant: Parser[Unit] = rep(comment | multiLineComment | datumComment | hashBangComment | whitespaceChar | eol) ^^ (_ => ()) def any: Parser[Char] = chrExcept() def chr(c: Char): Parser[Char] = elem(s"character $c", _ == c) diff --git a/code/shared/src/test/scala/maf/test/language/sexp/CommentsTest.scala b/code/shared/src/test/scala/maf/test/language/sexp/CommentsTest.scala new file mode 100644 index 000000000..d33840b42 --- /dev/null +++ b/code/shared/src/test/scala/maf/test/language/sexp/CommentsTest.scala @@ -0,0 +1,44 @@ +package maf.test.language.sexp + +import org.scalatest.prop._ +import org.scalatest.propspec._ +import maf.language.sexp._ +import maf.test.ParserTest + +class CommentsTest extends AnyPropSpec with TableDrivenPropertyChecks { + val lexical = new SExpLexer + + val comments = Table( + ("input", "expected"), + // Line comments + ("; standard line comment\n 42", "42"), + ("#! hash bang line comment\n 42", "42"), + // Multi-line block comments + ("#| block comment |# 42", "42"), + ("#| outer #| inner |# outer |# 42", "42"), + ("#| multiple \n lines \n inside |# 42", "42"), + // Datum comments + ("#; (1 2 3) 42", "42"), + ("#; #| comment in datum |# (1 2 3) 42", "42"), + ("#; #(1 2 3) 42", "42"), + ("#; 'foo 42", "42"), + ("#; ,@foo 42", "42"), + // Nested / Stacked datum comments + ("#; #; 1 2 3", "3"), + ("#; #; (1 2) 3 4", "4"), + // Mixed comments + ("#; 1 #| block |# #; 2 42", "42"), + ("#; (1 #; 2 3) 42", "42") + ) + + property("SExpLexer should skip comments correctly", ParserTest) { + forAll(comments) { (s, exp) => + val parser = lexical.token + parser(new scala.util.parsing.input.CharArrayReader(s.toCharArray.nn)) match { + case lexical.Success(res, next) => + assert(res.chars == exp) + case res => throw new Exception(s"Parse failure: $res") + } + } + } +}