perf(rolldown): speed up session logs

packages/rolldown/src/node/rolldown/__tests__/reader-index-cache.test.ts

@@ -0,0 +1,114 @@
+import type { ReaderIndex } from '../reader-index-codec'
+import { Buffer } from 'node:buffer'
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'pathe'
+import { afterEach, describe, expect, it } from 'vitest'
+import { RolldownEventsReader } from '../events-reader'
+import { decodeReaderIndex, encodeReaderIndex } from '../reader-index-codec'
+
+const dirs: string[] = []
+afterEach(() => {
+  for (const dir of dirs.splice(0))
+    rmSync(dir, { recursive: true, force: true })
+})
+
+function createIndex(): ReaderIndex {
+  const location = (offset: number, length = 10) => ({ offset, length })
+  return {
+    lastBytes: 5 * 1024 ** 3,
+    lastTimestamp: 1793700062071,
+    moduleEventIndex: new Map([
+      ['/src/ä/main.ts', {
+        resolveIds: new Map([['r1', { start: location(0), end: location(10) }]]),
+        loads: new Map(),
+        // Offsets past 4 GiB must survive the round trip.
+        transforms: new Map([['t1', { start: location(4.5 * 1024 ** 3), end: location(4.5 * 1024 ** 3 + 20, 300) }]]),
+      }],
+    ]),
+    stringRefIndex: new Map([['ref-1', location(42, 7)]]),
+    pendingHookCallStarts: new Map([
+      ['p1', { module: '\0virtual', location: location(1) }],
+      ['p2', { location: location(2) }],
+    ]),
+    assetsReadyLocations: [location(99, 3)],
+  }
+}
+
+describe('reader index codec', () => {
+  const stat = { size: 123, mtime: 456 }
+
+  it('round-trips every index field', async () => {
+    const index = createIndex()
+    const decoded = await decodeReaderIndex(Buffer.concat(await encodeReaderIndex(index, stat)), stat)
+    expect(decoded).toEqual(index)
+  })
+
+  it('rejects an index written for another log version', async () => {
+    const buffer = Buffer.concat(await encodeReaderIndex(createIndex(), stat))
+    expect(await decodeReaderIndex(buffer, { size: 124, mtime: 456 })).toBeUndefined()
+    expect(await decodeReaderIndex(buffer, { size: 123, mtime: 457 })).toBeUndefined()
+  })
+
+  it('rejects truncated or padded data', async () => {
+    const buffer = Buffer.concat(await encodeReaderIndex(createIndex(), stat))
+    expect(await decodeReaderIndex(buffer.subarray(0, buffer.length - 1), stat)).toBeUndefined()
+    expect(await decodeReaderIndex(Buffer.concat([buffer, Buffer.from([0])]), stat)).toBeUndefined()
+  })
+})
+
+describe('rolldown log cache', () => {
+  function writeLog() {
+    const dir = mkdtempSync(join(tmpdir(), 'rolldown-log-cache-'))
+    dirs.push(dir)
+    const base = { build_id: 'bid_1', session_id: 'sid_1' }
+    const lines = [
+      { timestamp: 1, action: 'BuildStart', ...base },
+      { timestamp: 2, action: 'HookResolveIdCallStart', importer: null, module_request: '/src/main.ts', import_kind: 'import-statement', plugin_name: 'p', plugin_id: 0, trigger: 'automatic', call_id: 'c1', ...base },
+      { timestamp: 3, action: 'HookResolveIdCallEnd', resolved_id: '/src/main.ts', is_external: false, plugin_name: 'p', plugin_id: 0, trigger: 'automatic', call_id: 'c1', ...base },
+      { timestamp: 4, action: 'HookTransformCallStart', module_id: '/src/main.ts', content: 'a', plugin_name: 'p', plugin_id: 0, trigger: 'automatic', call_id: 'c2', ...base },
+      { timestamp: 5, action: 'HookTransformCallEnd', module_id: '/src/main.ts', content: 'b', plugin_name: 'p', plugin_id: 0, trigger: 'automatic', call_id: 'c2', ...base },
+      { timestamp: 6, action: 'BuildEnd', ...base },
+    ]
+    const filepath = join(dir, 'logs.json')
+    writeFileSync(filepath, `${lines.map(line => JSON.stringify(line)).join('\n')}\n`)
+    return filepath
+  }
+
+  const indexOf = (reader: RolldownEventsReader) => {
+    const state = reader as unknown as Record<string, unknown>
+    return [state.moduleEventIndex, state.stringRefIndex, state.pendingHookCallStarts, state.assetsReadyLocations]
+  }
+
+  it('restores the summary with module metrics but without the index, then restores the same index on a full read', async () => {
+    const filepath = writeLog()
+    const cold = RolldownEventsReader.get(filepath)
+    await cold.read()
+    expect((cold as unknown as { moduleEventIndex: Map<string, unknown> }).moduleEventIndex.size).toBeGreaterThan(0)
+    const coldIndex = structuredClone(indexOf(cold))
+    const coldEvents = cold.manager.eventCount
+    const coldModules = cold.manager.module_build_metrics.size
+    const coldMetrics = cold.manager.module_build_metrics.get('/src/main.ts')!
+    expect(coldMetrics.transforms).toHaveLength(1)
+    cold.dispose()
+
+    const warm = RolldownEventsReader.get(filepath)
+    try {
+      await warm.readSummary()
+      expect(warm.manager.eventCount).toBe(coldEvents)
+      expect((warm as unknown as { moduleEventIndex: Map<string, unknown> }).moduleEventIndex.size).toBe(0)
+      // The summary keeps the compacted metrics the module list and chunk sizes rely on.
+      const warmMetrics = warm.manager.module_build_metrics.get('/src/main.ts')
+      expect(warm.manager.module_build_metrics.size).toBe(coldModules)
+      expect(warmMetrics?.transforms.at(-1)?.transformed_code_size).toBe(coldMetrics.transforms.at(-1)!.transformed_code_size)
+      expect(warmMetrics?.resolve_ids[0]?.duration).toBe(coldMetrics.resolve_ids[0]!.duration)
+
+      await warm.read()
+      expect(indexOf(warm)).toEqual(coldIndex)
+      expect(warm.manager.eventCount).toBe(coldEvents)
+    }
+    finally {
+      warm.dispose()
+    }
+  })
+})

packages/rolldown/src/node/rolldown/events-reader.ts

@@ -15,6 +15,7 @@ const MAX_READERS = 32
 const MAX_MODULE_METRICS_CACHE = 32
 const MAX_MODULE_METRICS_CACHE_BYTES = 64 * 1024 * 1024
 const READ_STREAM_HIGH_WATER_MARK = 1024 * 1024
+const PARSE_SLICE_MS = 8
 const LINE_FEED = '\n'.charCodeAt(0)
 const CARRIAGE_RETURN = '\r'.charCodeAt(0)
 const ACTION_PREVIEW_BYTES = 512
@@ -341,7 +342,7 @@ export class RolldownEventsReader {
     if (restoredSummary) {
       this.clearModuleMetricsCache()
       this.pendingModuleMetrics.clear()
-      this.summaryOnly = restoredSummary !== 'complete'
+      this.summaryOnly = true
       this.packageSummaryOnly = false
       this.assetsHydrated = false
       return
@@ -455,6 +456,7 @@ export class RolldownEventsReader {
       lineStartOffset = nextLineStartOffset
     }
 
+    let sliceStart = performance.now()
     for await (const chunk of stream) {
       const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
       let segmentStart = 0
@@ -482,6 +484,11 @@ export class RolldownEventsReader {
           }
           await processLine(line, location)
         }
+        // Large logs take seconds to parse; yield so the server keeps answering meanwhile.
+        if (performance.now() - sliceStart > PARSE_SLICE_MS) {
+          await new Promise(resolve => setImmediate(resolve))
+          sliceStart = performance.now()
+        }
 
         segmentStart = newline + 1
         resetLine(absoluteOffset + segmentStart)

packages/rolldown/src/node/rolldown/log-cache.ts

@@ -4,14 +4,15 @@ import type { ModuleBuildMetrics, PluginBuildMetrics } from '../../shared/types'
 import type { RolldownEventsManager } from './events-manager'
 import fs from 'node:fs/promises'
 import { basename, dirname, join } from 'pathe'
+import { decodeReaderIndex, encodeReaderIndex } from './reader-index-codec'
 
 export const ROLLDOWN_LOG_CACHE_DIR = '.cache'
 export const ROLLDOWN_LOG_SESSION_SUMMARY_CACHE_FILENAME = 'session-summary.json'
-export const ROLLDOWN_LOG_READER_INDEX_CACHE_FILENAME = 'reader-index.json'
+export const ROLLDOWN_LOG_READER_INDEX_CACHE_FILENAME = 'reader-index.bin'
 export const ROLLDOWN_LOG_PACKAGE_SUMMARY_CACHE_FILENAME = 'package-summary.json'
 export const ROLLDOWN_LOG_PLUGIN_SUMMARY_CACHE_FILENAME = 'plugin-summary.json'
 export const ROLLDOWN_LOG_MODULE_METRICS_SUMMARY_CACHE_FILENAME = 'module-metrics-summary.json'
-const ROLLDOWN_LOG_CACHE_VERSION = 2
+const ROLLDOWN_LOG_CACHE_VERSION = 3
 
 type ManagerSnapshot = ReturnType<RolldownEventsManager['snapshot']>
 
@@ -91,21 +92,6 @@ interface RolldownPluginSummaryCacheSnapshot {
   plugins: Array<[number, PluginBuildMetricsSummary]>
 }
 
-interface RolldownReaderIndexCacheSnapshot {
-  reader: {
-    lastBytes: number
-    lastTimestamp: number
-    moduleEventIndex: Array<[string, {
-      resolveIds: Array<[string, IndexedHookCall]>
-      loads: Array<[string, IndexedHookCall]>
-      transforms: Array<[string, IndexedHookCall]>
-    }]>
-    stringRefIndex: Array<[string, LineLocation]>
-    pendingHookCallStarts: Array<[string, PendingHookCallStart]>
-    assetsReadyLocations: LineLocation[]
-  }
-}
-
 interface RolldownPackageSummaryCacheSnapshot {
   packageSummaryTimestamp: number
   manager: ManagerSnapshot
@@ -167,6 +153,46 @@ class RolldownLogCacheFile<T> {
   }
 }
 
+class RolldownReaderIndexCacheFile {
+  private constructor(
+    readonly filepath: string,
+  ) {
+  }
+
+  static forLogFile(logFilepath: string) {
+    if (basename(logFilepath) !== 'logs.json')
+      return undefined
+
+    return new RolldownReaderIndexCacheFile(join(dirname(logFilepath), ROLLDOWN_LOG_CACHE_DIR, ROLLDOWN_LOG_READER_INDEX_CACHE_FILENAME))
+  }
+
+  async read(stat: Stats) {
+    try {
+      return await decodeReaderIndex(await fs.readFile(this.filepath), toLogStat(stat))
+    }
+    catch {
+      return undefined
+    }
+  }
+
+  async write(stat: Stats, state: RolldownLogCacheState) {
+    try {
+      const chunks = await encodeReaderIndex(state, toLogStat(stat))
+      const temporaryFilepath = `${this.filepath}.tmp`
+      await fs.mkdir(dirname(this.filepath), { recursive: true })
+      await fs.writeFile(temporaryFilepath, chunks)
+      await fs.rename(temporaryFilepath, this.filepath)
+    }
+    catch {
+      // The cache is best-effort. The reader can always rebuild from logs.json.
+    }
+  }
+}
+
+function toLogStat(stat: Stats) {
+  return { size: stat.size, mtime: stat.mtime.getTime() }
+}
+
 function createMetricSummary(): PluginMetricSummary {
   return {
     count: 0,
@@ -297,7 +323,7 @@ export class RolldownLogCache {
   private sessionSummaryCache: RolldownLogCacheFile<RolldownSessionSummaryCacheSnapshot> | undefined
   private moduleMetricsSummaryCache: RolldownLogCacheFile<RolldownModuleMetricsSummaryCacheSnapshot> | undefined
   private pluginSummaryCache: RolldownLogCacheFile<RolldownPluginSummaryCacheSnapshot> | undefined
-  private readerIndexCache: RolldownLogCacheFile<RolldownReaderIndexCacheSnapshot> | undefined
+  private readerIndexCache: RolldownReaderIndexCacheFile | undefined
   private packageSummaryCache: RolldownLogCacheFile<RolldownPackageSummaryCacheSnapshot> | undefined
   private completeSessionWriteAttempted = false
   private summaryWriteAttempted = false
@@ -308,7 +334,7 @@ export class RolldownLogCache {
       this.sessionSummaryCache = RolldownLogCacheFile.forLogFile(logFilepath, ROLLDOWN_LOG_SESSION_SUMMARY_CACHE_FILENAME)
       this.moduleMetricsSummaryCache = RolldownLogCacheFile.forLogFile(logFilepath, ROLLDOWN_LOG_MODULE_METRICS_SUMMARY_CACHE_FILENAME)
       this.pluginSummaryCache = RolldownLogCacheFile.forLogFile(logFilepath, ROLLDOWN_LOG_PLUGIN_SUMMARY_CACHE_FILENAME)
-      this.readerIndexCache = RolldownLogCacheFile.forLogFile(logFilepath, ROLLDOWN_LOG_READER_INDEX_CACHE_FILENAME)
+      this.readerIndexCache = RolldownReaderIndexCacheFile.forLogFile(logFilepath)
     }
     this.packageSummaryCache = RolldownLogCacheFile.forLogFile(logFilepath, ROLLDOWN_LOG_PACKAGE_SUMMARY_CACHE_FILENAME)
   }
@@ -351,11 +377,14 @@ export class RolldownLogCache {
     if (!sessionSummaryCache || !moduleMetricsCache || !pluginSummaryCacheExists || !indexCache)
       return false
 
-    if (sessionSummaryCache.reader.lastBytes !== stat.size || indexCache.reader.lastBytes !== stat.size)
+    if (sessionSummaryCache.reader.lastBytes !== stat.size || indexCache.lastBytes !== stat.size)
       return false
 
     this.restoreSessionSnapshot(sessionSummaryCache, state, moduleMetricsCache)
-    this.restoreReaderIndexSnapshot(indexCache, state)
+    state.moduleEventIndex = indexCache.moduleEventIndex
+    state.stringRefIndex = indexCache.stringRefIndex
+    state.pendingHookCallStarts = indexCache.pendingHookCallStarts
+    state.assetsReadyLocations = indexCache.assetsReadyLocations
     this.completeSessionWriteAttempted = true
     this.summaryWriteAttempted = true
     return true
@@ -365,27 +394,18 @@ export class RolldownLogCache {
     if (state.lastBytes || state.manager.eventCount)
       return false
 
-    const [sessionSummaryCache, moduleMetricsCache, _pluginSummaryCache, indexCache] = await Promise.all([
+    // The module list needs the metrics; the index is loaded on a full read.
+    const [sessionSummaryCache, moduleMetricsCache] = await Promise.all([
       this.sessionSummaryCache?.read(stat),
       this.moduleMetricsSummaryCache?.read(stat),
-      this.pluginSummaryCache?.read(stat),
-      this.readerIndexCache?.read(stat),
     ])
-    if (!sessionSummaryCache)
-      return false
-
-    if (sessionSummaryCache.reader.lastBytes !== stat.size)
+    if (!sessionSummaryCache || sessionSummaryCache.reader.lastBytes !== stat.size)
       return false
 
     this.restoreSessionSnapshot(sessionSummaryCache, state, moduleMetricsCache)
-    const hasCompleteCache = !!moduleMetricsCache && !!_pluginSummaryCache && !!indexCache
-    if (indexCache?.reader.lastBytes === stat.size)
-      this.restoreReaderIndexSnapshot(indexCache, state)
-    else
-      this.clearReaderIndexState(state)
+    this.clearReaderIndexState(state)
     this.summaryWriteAttempted = true
-    this.completeSessionWriteAttempted = hasCompleteCache
-    return hasCompleteCache ? 'complete' : 'summary'
+    return 'summary' as const
   }
 
   async restorePackageSummary(stat: Stats, state: RolldownLogCacheState) {
@@ -416,17 +436,19 @@ export class RolldownLogCache {
       this.sessionSummaryCache?.write(stat, this.createSessionSummarySnapshot(state)),
       this.moduleMetricsSummaryCache?.write(stat, this.createModuleMetricsSummarySnapshot(state)),
       this.pluginSummaryCache?.write(stat, this.createPluginSummarySnapshot(state)),
-      this.readerIndexCache?.write(stat, this.createReaderIndexSnapshot(state)),
+      this.readerIndexCache?.write(stat, state),
     ])
   }
 
   async writeSummary(stat: Stats, state: RolldownLogCacheState) {
     this.summaryWriteAttempted = true
+    // Writes the same sidecars as a complete session, so don't write them again for the full read.
+    this.completeSessionWriteAttempted = true
     await Promise.all([
       this.sessionSummaryCache?.write(stat, this.createSessionSummarySnapshot(state)),
       this.moduleMetricsSummaryCache?.write(stat, this.createModuleMetricsSummarySnapshot(state)),
       this.pluginSummaryCache?.write(stat, this.createPluginSummarySnapshot(state)),
-      this.readerIndexCache?.write(stat, this.createReaderIndexSnapshot(state)),
+      this.readerIndexCache?.write(stat, state),
     ])
   }
 
@@ -471,23 +493,6 @@ export class RolldownLogCache {
     }
   }
 
-  private createReaderIndexSnapshot(state: RolldownLogCacheState): RolldownReaderIndexCacheSnapshot {
-    return {
-      reader: {
-        lastBytes: state.lastBytes,
-        lastTimestamp: state.lastTimestamp,
-        moduleEventIndex: Array.from(state.moduleEventIndex.entries()).map(([module, index]) => [module, {
-          resolveIds: Array.from(index.resolveIds.entries()),
-          loads: Array.from(index.loads.entries()),
-          transforms: Array.from(index.transforms.entries()),
-        }]),
-        stringRefIndex: Array.from(state.stringRefIndex.entries()),
-        pendingHookCallStarts: Array.from(state.pendingHookCallStarts.entries()),
-        assetsReadyLocations: state.assetsReadyLocations,
-      },
-    }
-  }
-
   private createPackageSummarySnapshot(state: RolldownLogCacheState): RolldownPackageSummaryCacheSnapshot {
     return {
       packageSummaryTimestamp: state.packageSummaryTimestamp,
@@ -521,17 +526,6 @@ export class RolldownLogCache {
     })
   }
 
-  private restoreReaderIndexSnapshot(snapshot: RolldownReaderIndexCacheSnapshot, state: RolldownLogCacheState) {
-    state.moduleEventIndex = new Map(snapshot.reader.moduleEventIndex.map(([module, item]) => [module, {
-      resolveIds: new Map(item.resolveIds),
-      loads: new Map(item.loads),
-      transforms: new Map(item.transforms),
-    }]))
-    state.stringRefIndex = new Map(snapshot.reader.stringRefIndex)
-    state.pendingHookCallStarts = new Map(snapshot.reader.pendingHookCallStarts)
-    state.assetsReadyLocations = snapshot.reader.assetsReadyLocations
-  }
-
   private restorePackageSummarySnapshot(snapshot: RolldownPackageSummaryCacheSnapshot, state: RolldownLogCacheState) {
     state.manager.restore(snapshot.manager)
     state.packageSummaryTimestamp = snapshot.packageSummaryTimestamp