📝 CodeRabbit Chat: Protect strings and comments during compiler translation passes

scripts/compile.cjs

@@ -9,15 +9,99 @@ const passes = [
   'excepReplace', 'domReplace'
 ]
 
+function protectSource(source) {
+  let prefix = 'BYNIXPROTECTED'
+  while (source.includes(prefix)) prefix += 'X'
+  const protectedText = []
+  let position = 0
+
+  function mask(text) {
+    if (!text) return ''
+    const token = `${prefix}${protectedText.length}TOKEN`
+    protectedText.push(text)
+    return token
+  }
+
+  function quoted(quote) {
+    const start = position++
+    while (position < source.length) {
+      if (source[position] === '\\') {
+        position += 2
+      } else if (source[position++] === quote) {
+        break
+      }
+    }
+    return mask(source.slice(start, position))
+  }
+
+  function template() {
+    let start = position++
+    let output = ''
+    while (position < source.length) {
+      if (source[position] === '\\') {
+        position += 2
+      } else if (source.startsWith('${', position)) {
+        output += mask(source.slice(start, position)) + '${'
+        position += 2
+        output += code('}')
+        start = position
+      } else if (source[position++] === '`') {
+        return output + mask(source.slice(start, position))
+      }
+    }
+    return output + mask(source.slice(start, position))
+  }
+
+  function code(terminator) {
+    let output = ''
+    let depth = 0
+    while (position < source.length) {
+      const character = source[position]
+      if (character === terminator && depth === 0) {
+        position++
+        return output + character
+      }
+      if (character === '"' || character === "'") {
+        output += quoted(character)
+      } else if (character === '`') {
+        output += template()
+      } else if (source.startsWith('//', position) || (character === '#' && !source.startsWith('#!', position))) {
+        const start = position
+        while (position < source.length && source[position] !== '\n') position++
+        const text = source.slice(start, position)
+        output += mask(character === '#' ? `//${text.slice(1)}` : text)
+      } else if (source.startsWith('/*', position) || source.startsWith('**', position)) {
+        const dialect = character === '*'
+        const start = position
+        const end = source.indexOf(dialect ? '**' : '*/', position + 2)
+        position = end === -1 ? source.length : end + 2
+        const text = source.slice(start, position)
+        output += mask(dialect ? `/*${text.slice(2, -2)}*/` : text)
+      } else {
+        if (character === '{') depth++
+        if (character === '}') depth--
+        output += character
+        position++
+      }
+    }
+    return output
+  }
+
+  const masked = code()
+  const token = new RegExp(`${prefix}(\\d+)TOKEN`, 'g')
+  return { masked, restore: text => text.replace(token, (match, index) => protectedText[Number(index)]) }
+}
+
 function compileSource(source) {
-  let code = addSemicolons(parseCode(source))
+  const { masked, restore } = protectSource(source)
+  let code = addSemicolons(parseCode(masked))
   if (parsing(code) === false) {
     throw new Error(parsingMsg.message)
   }
   for (const pass of passes) {
     code = replacements[pass](code)
   }
-  return code
+  return restore(code)
 }
 
 module.exports = { compileSource }

src/translate.bs

@@ -1,8 +1,7 @@
 const fs = require('node:fs')
 const path = require('node:path')
 const { extensions } = require('../js/bsc.js')
-const { parsing, parsingMsg, parseCode, addSemicolons } = require('../js/parser.js')
-const replacements = require('bynixscript/dist/parser/parsingDecl.js')
+const { compileSource } = require('../scripts/compile.cjs')
 
 func resolveFile(file):
   if typeof file !== 'string' || file.trim() === '':
@@ -27,15 +26,7 @@ func readSource(file):
 end
 
 func translate(source):
-  var code = addSemicolons(parseCode(source))
-  if parsing(code) === false:
-    throw new Err(parsingMsg.message)
-  end
-  const passes = 'flowReplace utilityReplace funcReplace condReplace forEachReplace reassignReplace assignReplace logReplace interactReplace mathReplace commentReplace asyncReplace forReplace convReplace checkReplace oopReplace excepReplace domReplace'.split(' ')
-  for const pass of passes:
-    code = replacements[pass](code)
-  end
-  return code
+  return compileSource(source)
 end
 
 module.exports = { resolveFile, readSource, translate, extensions }

tests/cli.test.cjs

@@ -4,7 +4,7 @@ const fs = require('node:fs')
 const os = require('node:os')
 const path = require('node:path')
 const { spawnSync } = require('node:child_process')
-const { Script } = require('node:vm')
+const { Script, runInNewContext } = require('node:vm')
 const { compileSource } = require('../scripts/compile.cjs')
 
 const root = path.resolve(__dirname, '..')
@@ -43,6 +43,87 @@ test('every CLI source compiles through the build compiler', () => {
   assert.throws(() => new Script(compileSource('func invalid():\n  print("x")\n')), SyntaxError)
 })
 
+test('bootstrap compiler preserves quoted and template text while translating code', () => {
+  const source = [
+    "const single = 'func print(1) is_end(\"x\") # \\'quoted\\''",
+    'const double = "handle: print(2) rand(3) \\"quoted\\""',
+    'const template = `first print(3) # **',
+    'end \\`escaped\\` ${single} ${`nested print(4) ${double}`}`',
+    'const interpolated = `print(5) ${"needle".is_includes("need")}`',
+    'func show():',
+    '  print(single, double, template, interpolated)',
+    'end',
+    'show()'
+  ].join('\n')
+  const output = compileSource(source)
+  assert.match(output, /function show\(\) \{/)
+  for (const literal of [
+    "'func print(1) is_end(\"x\") # \\'quoted\\''",
+    '"handle: print(2) rand(3) \\"quoted\\""',
+    '`first print(3) # **\nend \\`escaped\\` ${single} ${`nested print(4) ${double}`}`',
+    '`print(5) ${"needle".includes("need")}`'
+  ]) {
+    assert.ok(output.includes(literal), `Missing unchanged literal: ${literal}`)
+  }
+  const values = []
+  runInNewContext(output, { console: { log: (...args) => values.push(args) } })
+  assert.deepEqual(values, [[
+    "func print(1) is_end(\"x\") # 'quoted'",
+    'handle: print(2) rand(3) "quoted"',
+    'first print(3) # **\nend `escaped` func print(1) is_end("x") # \'quoted\' nested print(4) handle: print(2) rand(3) "quoted"',
+    'print(5) true'
+  ]])
+})
+
+test('bootstrap compiler preserves line and block comments without executing their contents', () => {
+  const source = [
+    'const events = []',
+    '# print("from hash") func hidden():',
+    '// print("from slash") is_end("x")',
+    '/* print("from block")',
+    'end func hidden(): */',
+    '** print("from dialect block")',
+    'end is_includes("x") **',
+    'events.push("safe")',
+    'print(events.join(","))'
+  ].join('\n')
+  const output = compileSource(source)
+  for (const comment of [
+    '// print("from hash") func hidden():',
+    '// print("from slash") is_end("x")',
+    '/* print("from block")\nend func hidden(): */',
+    '/* print("from dialect block")\nend is_includes("x") */'
+  ]) {
+    assert.ok(output.includes(comment), `Missing unchanged comment: ${comment}`)
+  }
+  const values = []
+  runInNewContext(output, { console: { log: value => values.push(value) } })
+  assert.deepEqual(values, ['safe'])
+})
+
+test('packaged compiler preserves literals and ignores comments', context => {
+  const directory = fixture(context)
+  const source = [
+    'const value = `print(1) # \\`quoted\\`',
+    'end ${"func is_end(2)"}`',
+    '# print("unsafe hash")',
+    '/* print("unsafe block")',
+    'end */',
+    'print(value)'
+  ].join('\n')
+  fs.writeFileSync(path.join(directory, 'protected.bys'), source)
+  const compilation = invoke(['compile', 'protected.bys'], directory)
+  assert.equal(compilation.status, 0, compilation.stderr)
+  const output = fs.readFileSync(path.join(directory, 'protected.js'), 'utf8')
+  assert.ok(output.includes('`print(1) # \\`quoted\\`\nend ${"func is_end(2)"}`'))
+  assert.ok(output.includes('// print("unsafe hash")'))
+  assert.ok(output.includes('/* print("unsafe block")\nend */'))
+  const result = invoke(['run', 'protected.bys'], directory)
+  assert.equal(result.status, 0, result.stderr)
+  assert.equal(result.stdout.trim(), 'print(1) # `quoted`\nend func is_end(2)')
+  assert.doesNotMatch(result.stdout, /unsafe/)
+})
+
 test('global and command help succeeds', () => {
   assert.match(invoke(['--help']).stdout, /run <file>/)
   for (const command of ['run', 'compile', 'print', 'delete']) {