diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/StreamingMemoryProbeTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/StreamingMemoryProbeTests.swift new file mode 100644 index 00000000..58cf9908 --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/StreamingMemoryProbeTests.swift @@ -0,0 +1,227 @@ +import Darwin +import Foundation +import XCTest +@testable import BitMatchEngine + +#if os(macOS) +final class StreamingMemoryProbeTests: XCTestCase { + /// Opt in and run this test alone: process-footprint measurements are + /// meaningless when another test is allocating in the same process. + /// The source is sparse, but the real fan-out writer copies every byte to + /// one and then two disposable destinations. No verification or report + /// pass runs, so the samples isolate copy-phase memory. + func testSparse512MiBCopyKeepsMemoryBoundedWithOneAndTwoDestinations() async throws { + guard ProcessInfo.processInfo.environment["BITMATCH_COPY_MEMORY_TEST"] == "1" else { + throw XCTSkip("Set BITMATCH_COPY_MEMORY_TEST=1 and run this test in isolation") + } + let size = Int64(512 * 1_024 * 1_024) + let available = (try FileManager.default.attributesOfFileSystem( + forPath: FileManager.default.temporaryDirectory.path)[.systemFreeSize] as? NSNumber)?.int64Value ?? 0 + guard available > size * 2 + 2 * 1_024 * 1_024 * 1_024 else { + throw XCTSkip("Not enough free space for disposable copy outputs") + } + + for destinationCount in [1, 2] { + try await measureCopy(destinationCount: destinationCount) + } + } + + /// This separate opt-in probe exercises the byte-comparison loop used by + /// Paranoid folder Compare. The pinned Paranoid transfer verifier uses a + /// different loop. Sparse inputs make a full 4 MiB-at-a-time pass without + /// consuming 1 GiB of disk. + func testSparse512MiBByteComparisonKeepsMemoryBounded() async throws { + guard ProcessInfo.processInfo.environment["BITMATCH_COMPARE_MEMORY_TEST"] == "1" else { + throw XCTSkip("Set BITMATCH_COMPARE_MEMORY_TEST=1 and run this test in isolation") + } + + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("bitmatch-compare-memory-\(UUID())", isDirectory: true) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + let size = UInt64(512 * 1_024 * 1_024) + let source = root.appendingPathComponent("source.bin") + let destination = root.appendingPathComponent("destination.bin") + for file in [source, destination] { + XCTAssertTrue(FileManager.default.createFile(atPath: file.path, contents: nil)) + let handle = try FileHandle(forWritingTo: file) + try handle.truncate(atOffset: size) + try handle.close() + } + + let samples = ComparisonFootprintSamples() + samples.started() + let match = try await ChecksumEngine.shared.performByteComparison( + sourceURL: source, destinationURL: destination, + progressCallback: { _, _ in samples.sample() } + ) + let measured = samples.snapshot() + XCTAssertTrue(match) + XCTAssertGreaterThanOrEqual(measured.chunks, 128) + XCTAssertGreaterThan(measured.initialMiB, 0) + XCTAssertEqual(measured.missingSamples, 0, "Could not measure physical footprint") + print("compare memory initialMiB=\(measured.initialMiB) peakMiB=\(measured.peakMiB) finalMiB=\(measured.finalMiB) chunks=\(measured.chunks)") + XCTAssertLessThan(measured.peakMiB - measured.initialMiB, 128, + "Byte comparison footprint grew with two 512 MiB files") + } + + private func measureCopy(destinationCount: Int) async throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("bitmatch-copy-memory-\(UUID())", isDirectory: true) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + + let size = Int64(512 * 1_024 * 1_024) + let source = root.appendingPathComponent("source", isDirectory: true) + try FileManager.default.createDirectory(at: source, withIntermediateDirectories: true) + let clip = source.appendingPathComponent("clip.bin") + XCTAssertTrue(FileManager.default.createFile(atPath: clip.path, contents: nil)) + let writer = try FileHandle(forWritingTo: clip) + try writer.truncate(atOffset: UInt64(size)) + try writer.close() + + let destinations = try (0.. DestinationWriter.FanOutDestination in + let url = root.appendingPathComponent("backup-\(index)", isDirectory: true) + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + return .init(index: index, root: try PinnedDestinationDirectory.open(destination: url, rootComponents: [])) + } + let samples = CopyFootprintSamples(expectedSize: size) + let hooks = DestinationWriter.FanOutHooks(sourceDidOpen: { _ in samples.opened() }) + + try await DestinationWriter.copyAllSafelyFanOut( + from: source, + toPinnedRoots: destinations, + verificationMode: .standard, + workers: 1, + checksumService: ChecksumEngine.shared, + preEnumeratedFiles: [clip], + hooks: hooks, + onIOBytes: { index, phase, count, _, _ in + switch phase { + case .writing: samples.wrote(index: index, count: count) + case .readingBack: break + } + }, + onProgress: { index, _, byteCount, wasReused in + samples.completed(index: index, byteCount: byteCount, wasReused: wasReused) + }, + onError: { _, _, error in samples.failed(error) } + ) + + let measured = samples.snapshot() + XCTAssertTrue(measured.errors.isEmpty, "Copy failed: \(measured.errors)") + XCTAssertGreaterThan(measured.initialMiB, 0) + XCTAssertEqual(measured.missingSamples, 0, "Could not measure physical footprint") + XCTAssertEqual(measured.completedIndexes, Set(0.. = [] + private var errors: [String] = [] + private var missingSamples = 0 + + init(expectedSize: Int64) { self.expectedSize = expectedSize } + + func opened() { + lock.withLock { + if let footprint = TransferDiagnosticStore.physicalFootprintMB() { + initialMiB = footprint + peakMiB = footprint + } else { + missingSamples += 1 + } + } + } + + func wrote(index: Int, count: Int) { + lock.withLock { + writtenBytes[index, default: 0] += Int64(count) + if let footprint = TransferDiagnosticStore.physicalFootprintMB() { + peakMiB = max(peakMiB, footprint) + } else { + missingSamples += 1 + } + } + } + + func completed(index: Int, byteCount: Int64, wasReused: Bool) { + lock.withLock { + if wasReused { errors.append("Unexpected reuse at destination \(index)") } + if byteCount != expectedSize { errors.append("Unexpected progress size at destination \(index)") } + completedIndexes.insert(index) + } + } + + func failed(_ error: Error) { + lock.withLock { errors.append(error.localizedDescription) } + } + + func snapshot() -> (initialMiB: Int, peakMiB: Int, finalMiB: Int, + writtenBytes: [Int: Int64], completedIndexes: Set, + errors: [String], missingSamples: Int) { + lock.withLock { + let finalMiB = TransferDiagnosticStore.physicalFootprintMB() + if finalMiB == nil { missingSamples += 1 } + return (initialMiB, peakMiB, finalMiB ?? 0, + writtenBytes, completedIndexes, errors, missingSamples) + } + } +} + +private final class ComparisonFootprintSamples: @unchecked Sendable { + private let lock = NSLock() + private var initialMiB = 0 + private var peakMiB = 0 + private var chunks = 0 + private var missingSamples = 0 + + func started() { + lock.withLock { + if let footprint = TransferDiagnosticStore.physicalFootprintMB() { + initialMiB = footprint + peakMiB = footprint + } else { + missingSamples += 1 + } + } + } + + func sample() { + lock.withLock { + chunks += 1 + if let footprint = TransferDiagnosticStore.physicalFootprintMB() { + peakMiB = max(peakMiB, footprint) + } else { + missingSamples += 1 + } + } + } + + func snapshot() -> (initialMiB: Int, peakMiB: Int, finalMiB: Int, + chunks: Int, missingSamples: Int) { + lock.withLock { + let finalMiB = TransferDiagnosticStore.physicalFootprintMB() + if finalMiB == nil { missingSamples += 1 } + return (initialMiB, peakMiB, finalMiB ?? 0, chunks, missingSamples) + } + } +} +#endif