From 19e4f807d3f1c47d211315fabcbb42c95706d2c2 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 20:54:57 -0400 Subject: [PATCH 01/56] Check explicitly recorded metadata files against saved hashes --- .../BitMatchEngine/SavedChecksumCheck.swift | 8 +- .../SavedChecksumRecordedMetadataTests.swift | 103 ++++++++++++++++++ 2 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/SavedChecksumRecordedMetadataTests.swift diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/SavedChecksumCheck.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/SavedChecksumCheck.swift index dc21cb45..b9d2044c 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/SavedChecksumCheck.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/SavedChecksumCheck.swift @@ -186,7 +186,10 @@ public struct SavedChecksumCheck: Sendable { try Task.checkCancellation() try await PauseGate.waitIfCurrentIsPaused() let path = try resolver.resolve(file) - if Self.isEvidenceOrMetadata(path) { continue } + // Unrecorded Finder state is noise. A preserved companion named in + // the evidence is part of the transfer set and must be checked. + if Self.isReservedEvidence(path) + || (FolderComparer.isFinderMetadata(path) && discovery.expectedFiles[path] == nil) { continue } #if canImport(Darwin) if discovery.expectedFiles[path] == nil, TransferStatusMarker.validatedStatus(at: file, relativePath: path) != nil { continue } @@ -668,8 +671,7 @@ public struct SavedChecksumCheck: Sendable { return path } - private static func isEvidenceOrMetadata(_ path: String) -> Bool { - if FolderComparer.isFinderMetadata(path) { return true } + private static func isReservedEvidence(_ path: String) -> Bool { let parts = path.split(separator: "/") guard !parts.isEmpty else { return true } return parts.contains { $0.caseInsensitiveCompare("Reports") == .orderedSame } diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SavedChecksumRecordedMetadataTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SavedChecksumRecordedMetadataTests.swift new file mode 100644 index 00000000..f67cee7d --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SavedChecksumRecordedMetadataTests.swift @@ -0,0 +1,103 @@ +import CryptoKit +import Foundation +import XCTest +@testable import BitMatchEngine + +final class SavedChecksumRecordedMetadataTests: XCTestCase { + /// The same preserve-all pipeline and MHL handoff as the real mounted + /// exFAT regression, without requiring a disk image in the normal suite. + func testPreservedRecordedAppleDoubleIsCheckedAndDamageIsNeverHidden() async throws { + let fm = FileManager.default + let root = fm.temporaryDirectory.appendingPathComponent("recorded-metadata-\(UUID())") + let source = root.appendingPathComponent("Card") + let target = root.appendingPathComponent("Backup") + for directory in [source, target] { try fm.createDirectory(at: directory, withIntermediateDirectories: true) } + defer { try? fm.removeItem(at: root) } + try Data("synthetic footage".utf8).write(to: source.appendingPathComponent("clip.bin")) + let companion = metadata() + try companion.write(to: source.appendingPathComponent("._clip.bin")) + XCTAssertEqual(try AppleDoubleSelection.review(source: source), ["._clip.bin"]) + let operation = try await TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared) + .performFileOperation(sourceURL: source, destinationURLs: [target], verificationMode: .standard, + settings: .init(), estimatedTotalBytes: nil, progressCallback: { _ in }, onFileResult: nil) + XCTAssertEqual(operation.results.count, 2) + XCTAssertTrue(operation.results.allSatisfy { $0.success && $0.verificationResult?.isValid == true }) + let plan = TransferCompletion.ascmhlPlan(results: operation.results, sourceFiles: operation.sourceManifest, + destinations: [target], source: source, settings: .init()) + XCTAssertEqual(plan.jobs.count, 1) + let issues = try TransferCompletion.writeASCMHL(plan.jobs, planIssues: plan.issues, + startTime: operation.startTime, source: source, toolVersion: "recorded-metadata-test") + XCTAssertTrue(issues.isEmpty) + let output = try XCTUnwrap(plan.jobs.first?.root) + let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + let evidence = try await checker.discover(under: output) + XCTAssertEqual(Set(evidence.expectedFiles.keys), ["clip.bin", "._clip.bin"]) + let checked = try await checker.check(evidence) { _ in } + XCTAssertTrue(checked.isIntact) + XCTAssertEqual(Set(checked.matchingPaths), ["clip.bin", "._clip.bin"]) + XCTAssertTrue(checked.missingPaths.isEmpty) + + // A valid metadata header does not authorize ignoring its recorded hash. + var changed = companion + changed[changed.count - 1] ^= 1 + let destinationCompanion = output.appendingPathComponent("._clip.bin") + try changed.write(to: destinationCompanion) + let damaged = try await checker.check(evidence) { _ in } + XCTAssertFalse(damaged.isIntact) + XCTAssertEqual(damaged.changedPaths, ["._clip.bin"]) + XCTAssertTrue(damaged.missingPaths.isEmpty) + + try fm.removeItem(at: destinationCompanion) + let missing = try await checker.check(evidence) { _ in } + XCTAssertFalse(missing.isIntact) + XCTAssertEqual(missing.missingPaths, ["._clip.bin"]) + XCTAssertEqual(try Data(contentsOf: source.appendingPathComponent("._clip.bin")), companion) + } + + func testUnrecordedFinderMetadataStaysIgnoredAndReservedTreesStayReserved() async throws { + let fm = FileManager.default + let root = fm.temporaryDirectory.appendingPathComponent("unrecorded-metadata-\(UUID())") + try fm.createDirectory(at: root, withIntermediateDirectories: true) + defer { try? fm.removeItem(at: root) } + let media = Data("synthetic footage".utf8) + try media.write(to: root.appendingPathComponent("clip.bin")) + try metadata().write(to: root.appendingPathComponent("._clip.bin")) + try Data("Finder view state".utf8).write(to: root.appendingPathComponent(".DS_Store")) + let report = root.appendingPathComponent("record.json") + let record = SavedChecksumCheck.Record(url: report, date: Date(), kind: .bitMatchReport) + let expected = SavedChecksumCheck.ExpectedFile(relativePath: "clip.bin", checksum: sha(media), algorithm: .sha256, record: record) + let discovery = SavedChecksumCheck.Discovery(root: root, expectedFiles: ["clip.bin": expected], records: [record]) + let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + let result = try await checker.check(discovery) { _ in } + XCTAssertTrue(result.isIntact) + XCTAssertEqual(result.matchingPaths, ["clip.bin"]) + XCTAssertTrue(result.newPaths.isEmpty) + + for reserved in ["Reports/._clip.bin", "ascmhl/._clip.bin"] { + let file = root.appendingPathComponent(reserved) + try fm.createDirectory(at: file.deletingLastPathComponent(), withIntermediateDirectories: true) + try metadata().write(to: file) + let input = SavedChecksumCheck.ExpectedFile(relativePath: reserved, checksum: sha(metadata()), algorithm: .sha256, record: record) + let reservedDiscovery = SavedChecksumCheck.Discovery(root: root, expectedFiles: [reserved: input], records: [record]) + let reservedResult = try await checker.check(reservedDiscovery) { _ in } + XCTAssertFalse(reservedResult.isIntact) + XCTAssertEqual(reservedResult.missingPaths, [reserved]) + } + } + + private func metadata() -> Data { + var bytes = [UInt8](repeating: 0, count: 70) + func put(_ value: UInt32, _ offset: Int) { + for index in 0..<4 { bytes[offset + index] = UInt8(truncatingIfNeeded: value >> (24 - index * 8)) } + } + put(0x00051607, 0); put(0x00020000, 4) + bytes[25] = 1 + put(9, 26); put(38, 30); put(32, 34) + bytes[69] = 17 + return Data(bytes) + } + + private func sha(_ bytes: Data) -> String { + SHA256.hash(data: bytes).map { String(format: "%02x", $0) }.joined() + } +} From 59282ce33185e670233a1584de5a9156fb5dbbe1 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 20:59:46 -0400 Subject: [PATCH 02/56] Test mounted-drive recovery and filesystem faults --- .../MountedDriveRecoveryMatrixTests.swift | 411 ++++++++++++++++++ Scripts/test-mounted-drive-recovery.sh | 69 +++ 2 files changed, 480 insertions(+) create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedDriveRecoveryMatrixTests.swift create mode 100755 Scripts/test-mounted-drive-recovery.sh diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedDriveRecoveryMatrixTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedDriveRecoveryMatrixTests.swift new file mode 100644 index 00000000..745a566c --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedDriveRecoveryMatrixTests.swift @@ -0,0 +1,411 @@ +#if os(macOS) +import CryptoKit +import Darwin +import Foundation +import Synchronization +import XCTest +@testable import BitMatchEngine + +/// Opt-in tests against an explicitly owned mounted image, never a user's drive. +/// Hook-driven interleavings below are labelled separately from kernel faults. +final class MountedDriveRecoveryMatrixTests: XCTestCase, @unchecked Sendable { + private var mount: URL! + private var scratch: URL! + private var source: URL! + private var backup: URL! + private var filesystem = "" + // Do not allocate media buffers when the opt-in suite is merely discovered. + private lazy var payload = Data(repeating: 0x61, count: 12 * 1024 * 1024 + 137) + + override func setUpWithError() throws { + guard let path = ProcessInfo.processInfo.environment["BITMATCH_RECOVERY_MOUNT"], + let expected = ProcessInfo.processInfo.environment["BITMATCH_RECOVERY_TYPE"] else { + throw XCTSkip("Opt in with BITMATCH_RECOVERY_MOUNT and BITMATCH_RECOVERY_TYPE") + } + mount = URL(fileURLWithPath: path) + guard mount.path.hasPrefix("/private/tmp/bitmatch-recovery-"), + FileManager.default.fileExists(atPath: mount.appendingPathComponent(".bitmatch-recovery-owned").path) else { + throw failure("The mounted fixture does not have our ownership marker") + } + var facts = statfs() + guard statfs(mount.path, &facts) == 0 else { throw posix() } + var host = statfs() + guard statfs("/private/tmp", &host) == 0 else { throw posix() } + guard facts.f_fsid.val.0 != host.f_fsid.val.0 || facts.f_fsid.val.1 != host.f_fsid.val.1 else { + throw failure("Fixture is on the host filesystem, not a disposable mount") + } + filesystem = withUnsafeBytes(of: facts.f_fstypename) { + String(decoding: $0.prefix(while: { $0 != 0 }), as: UTF8.self) + } + XCTAssertEqual(filesystem, expected) + guard filesystem == expected else { throw failure("Wrong mounted filesystem") } + scratch = FileManager.default.temporaryDirectory.appendingPathComponent("bitmatch-recovery-source-\(UUID())") + source = scratch.appendingPathComponent("Card") + let readOnly = ProcessInfo.processInfo.environment["BITMATCH_RECOVERY_READ_ONLY"] == "1" + backup = mount.appendingPathComponent(readOnly ? "readonly-target" : "attempt-\(UUID())") + try FileManager.default.createDirectory(at: source, withIntermediateDirectories: true) + if !readOnly { try FileManager.default.createDirectory(at: backup, withIntermediateDirectories: true) } + try payload.write(to: source.appendingPathComponent("clip.bin")) + print("RECOVERY_CASE filesystem=\(filesystem) test=\(name)") + } + + override func tearDownWithError() throws { + if ProcessInfo.processInfo.environment["BITMATCH_RECOVERY_READ_ONLY"] != "1", + let backup, FileManager.default.fileExists(atPath: backup.path) { + try FileManager.default.removeItem(at: backup) + } + if let scratch, FileManager.default.fileExists(atPath: scratch.path) { + try FileManager.default.removeItem(at: scratch) + } + } + + private var fat: Bool { filesystem == "exfat" || filesystem == "msdos" } + private var copied: URL { + SafetyValidator.resolvedDestinationRoot(source: source, destination: backup, settings: .init()) + } + private func run(_ mode: VerificationMode = .standard, + hooks: DestinationWriter.FanOutHooks? = nil, + destination: URL? = nil) async throws -> FileOperation { + try await TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared, fanOutHooks: hooks) + .performFileOperation(sourceURL: source, destinationURLs: [destination ?? backup], + verificationMode: mode, settings: .init(), + progressCallback: { _ in }, onFileResult: nil) + } + private func assertSourceIntact(file: StaticString = #filePath, line: UInt = #line) throws { + XCTAssertEqual(try Data(contentsOf: source.appendingPathComponent("clip.bin")), payload, file: file, line: line) + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: source.path), ["clip.bin"], file: file, line: line) + } + private func assertNoStaging(file: StaticString = #filePath, line: UInt = #line) throws { + let entries = try XCTUnwrap(FileManager.default.enumerator(at: backup, includingPropertiesForKeys: nil), file: file, line: line) + for case let entry as URL in entries { + XCTAssertFalse(entry.lastPathComponent.hasPrefix(".bitmatch.tmp."), file: file, line: line) + } + } + private func assertIncomplete(file: StaticString = #filePath, line: UInt = #line) throws { + let entries = try FileManager.default.contentsOfDirectory(at: copied, includingPropertiesForKeys: nil) + let markers = entries.filter { TransferStatusMarker.hasExactName($0.lastPathComponent) } + XCTAssertFalse(markers.isEmpty, file: file, line: line) + for marker in markers { + XCTAssertEqual(TransferStatusMarker.validatedStatus(at: marker, relativePath: marker.lastPathComponent), .incomplete, file: file, line: line) + } + XCTAssertFalse(FileManager.default.fileExists(atPath: copied.appendingPathComponent("ascmhl").path), file: file, line: line) + } + + /// Real writes; cancellation is requested at a deterministic chunk boundary. + func testCancelMidCopyLeavesNoTrustedResultAndRetryPreservesPartialBytes() async throws { + let reached = Mutex(false) + let hooks = DestinationWriter.FanOutHooks(beforeWrite: { _, _, chunk in + if chunk == 1 { + reached.withLock { $0 = true } + throw CancellationError() + } + }) + let task = Task.detached { try await self.run(hooks: hooks) } + do { _ = try await task.value; XCTFail("A cancelled write returned completion") } + catch is CancellationError { } + XCTAssertTrue(reached.withLock { $0 }) + try assertSourceIntact() + try assertNoStaging() + if fat { + try assertIncomplete() + let partial = try Data(contentsOf: copied.appendingPathComponent("clip.bin")) + XCTAssertGreaterThan(partial.count, 0) + XCTAssertLessThan(partial.count, payload.count) + let retry = try await run() + XCTAssertTrue(retry.results.contains { !$0.success }) + XCTAssertEqual(try Data(contentsOf: copied.appendingPathComponent("clip.bin")), partial) + } else { + XCTAssertFalse(FileManager.default.fileExists(atPath: copied.appendingPathComponent("clip.bin").path)) + let retry = try await run() + XCTAssertTrue(retry.results.allSatisfy { $0.success && $0.verificationResult?.isValid == true }) + } + let fresh = backup.appendingPathComponent("fresh") + try FileManager.default.createDirectory(at: fresh, withIntermediateDirectories: true) + let recovered = try await run(destination: fresh) + XCTAssertTrue(recovered.results.allSatisfy { $0.success && $0.verificationResult?.isValid == true }) + try assertSourceIntact() + } + + func testCancelMidReadbackPropagatesAndIdenticalRetryRechecksBytes() async throws { + let reached = Mutex(false) + let hooks = DestinationWriter.FanOutHooks(destinationDidRead: { _, _, _ in + reached.withLock { $0 = true } + withUnsafeCurrentTask { $0?.cancel() } + }) + let task = Task.detached { try await self.run(hooks: hooks) } + do { _ = try await task.value; XCTFail("Cancelled readback returned completion") } + catch is CancellationError { } + XCTAssertTrue(reached.withLock { $0 }) + XCTAssertEqual(try Data(contentsOf: copied.appendingPathComponent("clip.bin")), payload) + if fat { try assertIncomplete() } + let reads = Mutex(0) + let retry = try await run(hooks: .init(destinationDidRead: { _, _, count in reads.withLock { $0 += count } })) + XCTAssertTrue(retry.results.allSatisfy { $0.success && $0.verificationResult?.isValid == true }) + XCTAssertGreaterThanOrEqual(reads.withLock { $0 }, payload.count) + try assertSourceIntact() + } + + func testRetryIdenticalAndChangedDestinationAcrossAllModes() async throws { + _ = try await run() + let file = copied.appendingPathComponent("clip.bin") + for mode in VerificationMode.allCases { + let retry = try await run(mode) + XCTAssertEqual(retry.results.count, 1) + XCTAssertEqual(retry.results[0].success, mode != .quick) + XCTAssertEqual(try Data(contentsOf: file), payload) + } + let changed = Data(repeating: 0x99, count: payload.count) + try changed.write(to: file) + for mode in VerificationMode.allCases { + let retry = try await run(mode) + XCTAssertEqual(retry.results.count, 1) + XCTAssertFalse(retry.results[0].success) + XCTAssertNotEqual(retry.results[0].verificationResult?.isValid, true) + XCTAssertEqual(try Data(contentsOf: file), changed) + } + try assertSourceIntact() + } + + /// A competitor is created at the real publication boundary; O_EXCL/kernel + /// publication refusal must preserve it. The interleaving is hook-driven. + func testCompetingCreatorCannotBeOverwritten() async throws { + let competitor = Data("A different owner's existing bytes".utf8) + let output = copied + let install: @Sendable (Int, URL) throws -> Void = { _, _ in + try competitor.write(to: output.appendingPathComponent("clip.bin"), options: .withoutOverwriting) + } + var hooks = DestinationWriter.FanOutHooks() + if fat { hooks.beforeDirectFinalCreate = install } + else { hooks.beforePublish = install } + let operation = try await run(hooks: hooks) + XCTAssertEqual(operation.results.count, 1) + XCTAssertFalse(operation.results[0].success) + XCTAssertEqual(try Data(contentsOf: output.appendingPathComponent("clip.bin")), competitor) + try assertSourceIntact() + try assertNoStaging() + } + + func testCaseOnlyConflictNeverOverwritesOnInsensitiveFilesystem() async throws { + try FileManager.default.createDirectory(at: copied, withIntermediateDirectories: true) + let existing = copied.appendingPathComponent("CLIP.BIN") + let sentinel = Data("different existing clip".utf8) + try sentinel.write(to: existing) + let insensitive = FileManager.default.fileExists(atPath: copied.appendingPathComponent("clip.bin").path) + let operation = try await run() + XCTAssertEqual(operation.results.count, 1) + XCTAssertEqual(operation.results[0].success, !insensitive) + XCTAssertEqual(try Data(contentsOf: existing), sentinel) + if !insensitive { XCTAssertEqual(try Data(contentsOf: copied.appendingPathComponent("clip.bin")), payload) } + print("RECOVERY_CASE caseInsensitive=\(insensitive)") + try assertSourceIntact() + } + + func testSourceSymlinkRefusedWithoutReadingOrCopyingItsTarget() async throws { + let media = source.appendingPathComponent("clip.bin") + let target = scratch.appendingPathComponent("outside.bin") + try payload.write(to: target) + try FileManager.default.removeItem(at: media) + try FileManager.default.createSymbolicLink(at: media, withDestinationURL: target) + let sourceReads = Mutex(0) + do { + let operation = try await run(hooks: .init(sourceDidOpen: { _ in sourceReads.withLock { $0 += 1 } })) + XCTAssertFalse(operation.results.contains { $0.success }) + } catch { XCTAssertFalse(error is CancellationError) } + XCTAssertEqual(sourceReads.withLock { $0 }, 0) + XCTAssertEqual(try Data(contentsOf: target), payload) + XCTAssertEqual(try FileManager.default.destinationOfSymbolicLink(atPath: media.path), target.path) + XCTAssertFalse(FileManager.default.fileExists(atPath: copied.appendingPathComponent("clip.bin").path)) + } + + /// The symlink lives on the host APFS (FAT cannot create one). Its target is + /// the real mounted filesystem. This validates ancestor rejection, not FAT + /// symlink support. + func testDestinationSymlinkAncestorRefusedAndTargetUntouched() async throws { + let alias = scratch.appendingPathComponent("backup-alias") + try FileManager.default.createSymbolicLink(at: alias, withDestinationURL: backup) + let sentinel = backup.appendingPathComponent("keep.bin") + try Data("keep".utf8).write(to: sentinel) + do { + let operation = try await run(destination: alias.appendingPathComponent("nested")) + XCTAssertFalse(operation.results.contains { $0.success }) + } catch { XCTAssertFalse(error is CancellationError) } + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: backup.path), ["keep.bin"]) + XCTAssertEqual(try Data(contentsOf: sentinel), Data("keep".utf8)) + try assertSourceIntact() + } + + /// This is the sole deliberate source mutation: append one byte after the + /// first actual card read. The engine must report failure, never bless it. + func testSourceGrowthDuringCopyFailsClosed() async throws { + let mutated = Mutex(false) + let media = source.appendingPathComponent("clip.bin") + let hooks = DestinationWriter.FanOutHooks(sourceDidRead: { _, bytes in + let shouldMutate = mutated.withLock { value -> Bool in + if value { return false }; value = true; return true + } + if shouldMutate { + let handle = try FileHandle(forWritingTo: media) + defer { try? handle.close() } + try handle.seekToEnd(); try handle.write(contentsOf: Data([0x7f])) + } + return bytes + }) + let operation = try await run(hooks: hooks) + XCTAssertTrue(mutated.withLock { $0 }) + XCTAssertTrue(operation.results.contains { !$0.success }) + XCTAssertFalse(operation.results.contains { $0.verificationResult?.isValid == true }) + XCTAssertEqual(try Data(contentsOf: media), payload + Data([0x7f])) + if fat { try assertIncomplete() } + try assertNoStaging() + } + + func testMHLReadCancellationAndPublicationInterruptionNeverBecomeTrustedHistory() async throws { + let operation = try await run() + XCTAssertTrue(operation.results.allSatisfy { $0.success }) + let digest = SHA256.hash(data: payload).map { String(format: "%02x", $0) }.joined() + let files: [ASCMHLGenerator.VerifiedFile] = [.init(relativePath: "clip.bin", size: Int64(payload.count), expectedSHA256: digest)] + let directory = copied + let readReached = Mutex(false) + let task = Task.detached { + try ASCMHLGenerator.generateInitialHistory(destinationURL: directory, files: files, startTime: Date(), toolVersion: "mounted-recovery", + readHooks: .init(didRead: { _, _ in readReached.withLock { $0 = true }; withUnsafeCurrentTask { $0?.cancel() } })) + } + do { _ = try await task.value; XCTFail("MHL cancellation returned success") } + catch is CancellationError { } + XCTAssertTrue(readReached.withLock { $0 }) + XCTAssertFalse(FileManager.default.fileExists(atPath: directory.appendingPathComponent("ascmhl").path)) + var hooks = ASCMHLGenerator.ReadHooks() + if fat { + hooks.didWritePublicHistoryFile = { name in if name == "ascmhl_chain.xml" { throw CancellationError() } } + } else { + hooks.beforeExclusiveRename = { throw CancellationError() } + } + XCTAssertThrowsError(try ASCMHLGenerator.generateInitialHistory(destinationURL: directory, files: files, startTime: Date(), toolVersion: "mounted-recovery", readHooks: hooks)) { + XCTAssertTrue($0 is CancellationError) + } + let history = directory.appendingPathComponent("ascmhl") + if fat { + let status = history.appendingPathComponent(ASCMHLGenerator.publicationStatusName) + XCTAssertTrue(FileManager.default.fileExists(atPath: status.path)) + XCTAssertFalse(try ASCMHLGenerator.publicationIsComplete(at: status)) + let namesBefore = try directorySnapshot(history) + XCTAssertThrowsError(try ASCMHLGenerator.generateInitialHistory(destinationURL: directory, files: files, startTime: Date(), toolVersion: "retry")) + XCTAssertEqual(try directorySnapshot(history), namesBefore) + do { _ = try await SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared).discover(under: directory); XCTFail("Incomplete history was accepted") } + catch let error as SavedChecksumCheck.CheckError { XCTAssertEqual(error, .unreadableEvidence) } + } else { + XCTAssertFalse(FileManager.default.fileExists(atPath: history.path)) + } + XCTAssertEqual(try Data(contentsOf: directory.appendingPathComponent("clip.bin")), payload) + try assertSourceIntact() + } + + /// Fill the real destination only after safety preflight. ENOSPC is the + /// actual driver's response, not an injected NSError. The filler is ours. + func testActualDiskFullDuringCopyRetainsWarningAndPreservesSource() async throws { + let filler = backup.appendingPathComponent("filler.bin") + let filled = Mutex(false) + let observedErrno = Mutex(nil) + let maxFillBytes: Int64 = 4 * 1024 * 1024 * 1024 + let hooks = DestinationWriter.FanOutHooks(beforeWrite: { _, _, _ in + let first = filled.withLock { value -> Bool in if value { return false }; value = true; return true } + guard first else { return } + let fd = Darwin.open(filler.path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0o600) + guard fd >= 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } + defer { _ = Darwin.close(fd) } + let chunk = Data(repeating: 0x3d, count: 8 * 1024 * 1024) + var written: Int64 = 0 + while written < maxFillBytes { + let count = chunk.withUnsafeBytes { Darwin.write(fd, $0.baseAddress, $0.count) } + if count > 0 { written += Int64(count) } + else if count < 0 && errno == EINTR { continue } + else { observedErrno.withLock { $0 = errno }; break } + } + guard observedErrno.withLock({ $0 }) == ENOSPC else { + throw NSError(domain: "MountedRecoveryFixture", code: 2, + userInfo: [NSLocalizedDescriptionKey: "Fixture must reach actual ENOSPC within 4 GiB"]) + } + }) + let operation = try await run(hooks: hooks) + XCTAssertTrue(filled.withLock { $0 }) + XCTAssertEqual(observedErrno.withLock { $0 }, ENOSPC) + XCTAssertEqual(operation.results.count, 1) + XCTAssertFalse(operation.results[0].success) + XCTAssertNotEqual(operation.results[0].verificationResult?.isValid, true) + print("RECOVERY_CASE actual_fill_errno=\(observedErrno.withLock { $0 } ?? -1) operation_error=\(String(describing: operation.results[0].error))") + // Free our filler before cleanup; never delete an unrelated volume. + try FileManager.default.removeItem(at: filler) + if fat { + try assertIncomplete() + let partial = try Data(contentsOf: copied.appendingPathComponent("clip.bin")) + XCTAssertLessThan(partial.count, payload.count) + XCTAssertEqual(partial, Data(payload.prefix(partial.count))) + } else { + XCTAssertFalse(FileManager.default.fileExists(atPath: copied.appendingPathComponent("clip.bin").path)) + } + try assertNoStaging() + try assertSourceIntact() + } + + /// Run separately after the harness reattaches this image read-only. + func testReadOnlyMountedDestinationCannotProduceVerifiedCopy() async throws { + guard ProcessInfo.processInfo.environment["BITMATCH_RECOVERY_READ_ONLY"] == "1" else { + throw XCTSkip("Run this case on the read-only reattachment") + } + var volume = statfs() + XCTAssertEqual(statfs(mount.path, &volume), 0) + XCTAssertNotEqual(volume.f_flags & UInt32(MNT_RDONLY), 0) + do { + let operation = try await run() + XCTAssertFalse(operation.results.isEmpty) + XCTAssertTrue(operation.results.allSatisfy { !$0.success }) + XCTAssertFalse(operation.results.contains { $0.verificationResult?.isValid == true }) + } catch { XCTAssertFalse(error is CancellationError) } + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: backup.path), []) + try assertSourceIntact() + } + + /// Run separately because it deliberately detaches the fixture. No forced + /// detach: this is an actual disappearing mount before descriptor pinning, + /// not a claim about cable removal during a blocked kernel read. + func testDetachedDestinationAfterPreflightFailsWithoutWritingMountpoint() async throws { + guard ProcessInfo.processInfo.environment["BITMATCH_RECOVERY_DETACH"] == "1" else { + throw XCTSkip("Run this case separately because it detaches the owned fixture") + } + let mountPath = mount.path + let didDetach = Mutex(false) + let service = TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared, + destinationSetupHook: { _ in + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/hdiutil") + process.arguments = ["detach", "-quiet", mountPath] + try process.run(); process.waitUntilExit() + guard process.terminationStatus == 0 else { + throw NSError(domain: "MountedRecoveryFixture.detach", code: Int(process.terminationStatus)) + } + didDetach.withLock { $0 = true } + }) + do { + let operation = try await service.performFileOperation(sourceURL: source, destinationURLs: [backup], verificationMode: .standard, + settings: .init(), progressCallback: { _ in }, onFileResult: nil) + XCTAssertFalse(operation.results.isEmpty) + XCTAssertTrue(operation.results.allSatisfy { !$0.success }) + } catch { XCTAssertFalse(error is CancellationError) } + XCTAssertTrue(didDetach.withLock { $0 }) + XCTAssertFalse(FileManager.default.fileExists(atPath: backup.path), "Must not recreate output on the host mountpoint") + try assertSourceIntact() + } + + private func directorySnapshot(_ root: URL) throws -> [String: Data] { + var result: [String: Data] = [:] + for file in try FileManager.default.contentsOfDirectory(at: root, includingPropertiesForKeys: nil) { + result[file.lastPathComponent] = try Data(contentsOf: file) + } + return result + } + private func failure(_ message: String) -> NSError { + NSError(domain: "MountedRecoveryFixture", code: 1, userInfo: [NSLocalizedDescriptionKey: message]) + } + private func posix() -> NSError { NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } +} +#endif diff --git a/Scripts/test-mounted-drive-recovery.sh b/Scripts/test-mounted-drive-recovery.sh new file mode 100755 index 00000000..492e0555 --- /dev/null +++ b/Scripts/test-mounted-drive-recovery.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Opt-in, destructive ONLY to disposable images created below our tmp root. +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +PAIR=$(mktemp -d /private/tmp/bitmatch-recovery-XXXXXX) +LOG_DIR=${BITMATCH_RECOVERY_LOG_DIR:-/private/tmp/bitmatch-recovery-results-$(date +%Y%m%d-%H%M%S)} +BUILD_DIR=${BITMATCH_RECOVERY_BUILD_DIR:-/private/tmp/bitmatch-recovery-build} +mkdir -p "$LOG_DIR" +ACTIVE="" +cleanup() { + if [[ -n "$ACTIVE" ]] && mount | /usr/bin/grep -F " on $ACTIVE (" >/dev/null; then + if ! hdiutil detach -quiet "$ACTIVE"; then + echo "Busy image retained; do not delete mounted root: $PAIR" >&2 + return + fi + fi + rm -rf "$PAIR" +} +trap cleanup EXIT +require_execution() { + python3 - "$1" "$2" <<'PY' +import pathlib, re, sys +text = pathlib.Path(sys.argv[1]).read_text() +rows = re.findall(r'Executed (\d+) tests?, with (?:(\d+) tests? skipped and )?(\d+) failures', text) +if not rows: + raise SystemExit("No XCTest execution summary: " + sys.argv[1]) +count, skipped, failures = rows[-1] +if int(count) != int(sys.argv[2]) or int(skipped or 0) or int(failures): + raise SystemExit("Wrong execution count/skips/failures: " + repr(rows[-1]) + " in " + sys.argv[1]) +PY +} +swift build --build-tests --package-path "$ROOT/Packages/BitMatchEngine" --scratch-path "$BUILD_DIR" --jobs 4 > "$LOG_DIR/build.log" 2>&1 +failed=0 +for item in 'apfs:APFS:apfs' 'apfsx:Case-sensitive APFS:apfs' 'hfs:JHFS+:hfs' 'exfat:ExFAT:exfat' 'fat:MS-DOS FAT32:msdos'; do + name=${item%%:*}; rest=${item#*:}; fs=${rest%:*}; expected=${rest##*:} + mkdir "$PAIR/$name" + # FAT-family volume labels have tighter length limits than APFS. + hdiutil create -quiet -size 3g -type SPARSE -fs "$fs" -volname "BMR_$name" -o "$PAIR/$name.sparseimage" > "$LOG_DIR/$name-image.log" 2>&1 + hdiutil attach -quiet -nobrowse -mountpoint "$PAIR/$name" "$PAIR/$name.sparseimage" >> "$LOG_DIR/$name-image.log" 2>&1 + ACTIVE="$PAIR/$name" + touch "$ACTIVE/.bitmatch-recovery-owned" + mkdir "$ACTIVE/readonly-target" + BITMATCH_RECOVERY_MOUNT="$ACTIVE" BITMATCH_RECOVERY_TYPE="$expected" \ + swift test --skip-build --package-path "$ROOT/Packages/BitMatchEngine" --scratch-path "$BUILD_DIR" --jobs 4 \ + --filter MountedDriveRecoveryMatrixTests --skip 'testReadOnlyMountedDestination|testDetachedDestination' \ + > "$LOG_DIR/$name-recovery.log" 2>&1 || failed=1 + require_execution "$LOG_DIR/$name-recovery.log" 10 || failed=1 + BITMATCH_RECOVERY_MOUNT="$ACTIVE" BITMATCH_RECOVERY_TYPE="$expected" BITMATCH_RECOVERY_DETACH=1 \ + swift test --skip-build --package-path "$ROOT/Packages/BitMatchEngine" --scratch-path "$BUILD_DIR" --jobs 4 \ + --filter MountedDriveRecoveryMatrixTests/testDetachedDestinationAfterPreflightFailsWithoutWritingMountpoint \ + > "$LOG_DIR/$name-detach.log" 2>&1 || failed=1 + require_execution "$LOG_DIR/$name-detach.log" 1 || failed=1 + # The detach test must unmount normally; never force a still-busy image. + if mount | /usr/bin/grep -F " on $ACTIVE (" >/dev/null; then + hdiutil detach -quiet "$ACTIVE" + fi + hdiutil attach -quiet -readonly -nobrowse -mountpoint "$ACTIVE" "$PAIR/$name.sparseimage" >> "$LOG_DIR/$name-image.log" 2>&1 + BITMATCH_RECOVERY_MOUNT="$ACTIVE" BITMATCH_RECOVERY_TYPE="$expected" BITMATCH_RECOVERY_READ_ONLY=1 \ + swift test --skip-build --package-path "$ROOT/Packages/BitMatchEngine" --scratch-path "$BUILD_DIR" --jobs 4 \ + --filter MountedDriveRecoveryMatrixTests/testReadOnlyMountedDestinationCannotProduceVerifiedCopy \ + > "$LOG_DIR/$name-readonly.log" 2>&1 || failed=1 + require_execution "$LOG_DIR/$name-readonly.log" 1 || failed=1 + hdiutil detach -quiet "$ACTIVE" + ACTIVE="" + rm "$PAIR/$name.sparseimage" + echo "RECOVERY_FILESYSTEM $name finished; logs: $LOG_DIR/$name-*.log" +done +echo "RECOVERY_RESULT exit=$failed logs=$LOG_DIR" +exit "$failed" From 5396f6c57a6b881e769d1e47b8ce0e635df872a8 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 21:03:45 -0400 Subject: [PATCH 03/56] Exercise mounted drive pairs and saved-history handoffs --- .../MountedDriveTransferMatrixTests.swift | 413 ++++++++++++++++++ 1 file changed, 413 insertions(+) create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedDriveTransferMatrixTests.swift diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedDriveTransferMatrixTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedDriveTransferMatrixTests.swift new file mode 100644 index 00000000..721b4fb9 --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedDriveTransferMatrixTests.swift @@ -0,0 +1,413 @@ +#if os(macOS) +import CryptoKit +import Darwin +import Foundation +import Synchronization +import XCTest +@testable import BitMatchEngine + +/// Opt-in integration tests against actual macOS filesystem drivers. Sparse +/// images exercise filesystem semantics, not USB hardware, power loss or RAM. +/// Run BITMATCH_MOUNTED_DRIVE_MATRIX=smoke first, then =full with +/// swift test --package-path Packages/BitMatchEngine --jobs 4 +/// --disable-swift-testing --filter MountedDriveTransferMatrixTests. +/// Full runs 636 scenarios; =preserve runs 18 targeted history rechecks, and +/// =policies runs 24 collisions plus 36 metadata-policy scenarios. +final class MountedDriveTransferMatrixTests: XCTestCase { + func testMountedDriveTransferAndHandoffMatrix() async throws { + guard let selection = ProcessInfo.processInfo.environment["BITMATCH_MOUNTED_DRIVE_MATRIX"], + ["smoke", "full", "preserve", "policies"].contains(selection) else { + throw XCTSkip("Opt in with BITMATCH_MOUNTED_DRIVE_MATRIX=smoke, full, preserve or policies") + } + try await FileOperationsTestLock.shared.run { + let harness = try MountedDriveMatrixHarness() + do { + try await harness.run(selection: selection) + try harness.cleanup() + } catch { + do { try harness.cleanup() } + catch { XCTFail("Cleanup failed; owned paths retained: \(error)") } + throw error + } + } + } +} + +private final class MountedDriveMatrixHarness { + private struct Format { + let name: String + let hdi: String + let kernel: String + let caseSensitive: Bool + } + private struct Volume { + let format: Format + let mount: URL + let role: String + } + private struct Snapshot: Equatable { + let size: Int64 + let modified: Date + let sha: String + } + private let fm = FileManager.default + private let root: URL + private var mounted: [Volume] = [] + private var sources: [Volume] = [] + private var backups: [Volume] = [] + private var scenarios = 0 + private var resultRows = 0 + private var savedChecks = 0 + private var expectedMetadataFailures = 0 + private let start = ProcessInfo.processInfo.systemUptime + private let formats = [ + Format(name: "APFS", hdi: "APFS", kernel: "apfs", caseSensitive: false), + Format(name: "APFS-case", hdi: "Case-sensitive APFS", kernel: "apfs", caseSensitive: true), + Format(name: "HFS-journaled", hdi: "Journaled HFS+", kernel: "hfs", caseSensitive: false), + Format(name: "HFS-case-journaled", hdi: "Case-sensitive Journaled HFS+", kernel: "hfs", caseSensitive: true), + Format(name: "exFAT", hdi: "ExFAT", kernel: "exfat", caseSensitive: false), + Format(name: "FAT32", hdi: "MS-DOS FAT32", kernel: "msdos", caseSensitive: false) + ] + + init() throws { + root = URL(fileURLWithPath: "/private/tmp", isDirectory: true) + .appendingPathComponent("bitmatch-drive-matrix-\(UUID())", isDirectory: true) + try fm.createDirectory(at: root, withIntermediateDirectories: true) + print("DRIVE_MATRIX owned_root=\(root.path)") + } + + func run(selection: String) async throws { + let smoke = selection == "smoke" + // Smoke uses the two unlike families most relevant to the report; + // the full run requires all formats and never silently skips a driver. + let selected = smoke ? [formats[0], formats[4]] : formats + for (index, format) in selected.enumerated() { + sources.append(try mount(format: format, role: "source", index: index)) + backups.append(try mount(format: format, role: "backup", index: index)) + } + for volume in sources { try seed(at: volume.mount.appendingPathComponent("Camera Card")) } + if selection == "preserve" { + try await preservedSourceCases() + XCTAssertEqual(scenarios, 18) + } else if selection == "policies" { + try await collisionAndPolicyCases() + XCTAssertEqual(scenarios, 60) + } else if smoke { + try await transfer(source: sources[0], destinations: [backups[1]], mode: .standard, pipeline: true) + try await transfer(source: sources[1], destinations: [backups[0]], mode: .paranoid, pipeline: false) + XCTAssertEqual(scenarios, 2) + } else { + for source in sources { + for backup in backups { + for mode in VerificationMode.allCases { + for pipeline in [false, true] { + try await transfer(source: source, destinations: [backup], mode: mode, pipeline: pipeline) + } + } + } + } + XCTAssertEqual(scenarios, 288) + for (sourceIndex, source) in sources.enumerated() { + for left in backups.indices { + for right in backups.indices where right > left { + for (modeIndex, mode) in [VerificationMode.standard, .thorough, .paranoid].enumerated() { + try await transfer(source: source, destinations: [backups[left], backups[right]], + mode: mode, pipeline: (sourceIndex + left + right + modeIndex).isMultiple(of: 2)) + } + } + } + } + XCTAssertEqual(scenarios, 558) + // Preserve every source entry, including real FAT companions, onto + // APFS and re-check the generated history. This pins the mounted + // regression that was hidden by the normal exclusion default. + try await preservedSourceCases() + XCTAssertEqual(scenarios, 576) + try await collisionAndPolicyCases() + XCTAssertEqual(scenarios, 636) + } + print("DRIVE_MATRIX COMPLETE scenarios=\(scenarios) result_rows=\(resultRows) saved_checks=\(savedChecks) expected_metadata_failures=\(expectedMetadataFailures) seconds=\(String(format: "%.2f", ProcessInfo.processInfo.systemUptime - start))") + } + + private func collisionAndPolicyCases() async throws { + let initial = scenarios + for index in sources.indices { + for mode in VerificationMode.allCases { + try await collision(source: sources[index], backup: backups[index], mode: mode) + } + } + XCTAssertEqual(scenarios, initial + 24) + let companion = try makeRealAppleDouble(on: sources[4].mount) + for index in sources.indices { + let card = sources[index].mount.appendingPathComponent("Policy Card") + try fm.createDirectory(at: card, withIntermediateDirectories: true) + try Data("camera media".utf8).write(to: card.appendingPathComponent("clip.bin")) + try companion.write(to: card.appendingPathComponent("._clip.bin")) + try Data("real sidecar; keep me".utf8).write(to: card.appendingPathComponent("._notes.txt")) + let reviewed = try AppleDoubleSelection.review(source: card) + XCTAssertTrue(reviewed.contains("._clip.bin")) + XCTAssertFalse(reviewed.contains("._notes.txt")) + for mode in [VerificationMode.standard, .thorough, .paranoid] { + for exclude in [false, true] { + try await transfer(source: sources[index], destinations: [backups[index]], mode: mode, + pipeline: exclude, sourceName: "Policy Card", reviewedExclusions: exclude ? reviewed : [], + allowCompanionRefusal: !exclude) + } + } + } + XCTAssertEqual(scenarios, initial + 60) + } + + private func preservedSourceCases() async throws { + for source in sources { + for mode in [VerificationMode.standard, .thorough, .paranoid] { + try await transfer(source: source, destinations: [backups[0]], mode: mode, + pipeline: mode != .paranoid, reviewedExclusions: []) + } + } + } + + private func mount(format: Format, role: String, index: Int) throws -> Volume { + let image = root.appendingPathComponent("\(role)-\(index).sparseimage") + let mount = root.appendingPathComponent("mnt-\(role)-\(index)") + try fm.createDirectory(at: mount, withIntermediateDirectories: true) + try command("/usr/bin/hdiutil", ["create", "-quiet", "-size", "3g", "-type", "SPARSE", "-fs", format.hdi, + "-volname", "BM\(index)\(role)", "-o", image.path]) + let volume = Volume(format: format, mount: mount, role: role) + // Track before attach: a partially successful command must not leave an + // untracked mounted filesystem under a recursively removed directory. + mounted.append(volume) + try command("/usr/bin/hdiutil", ["attach", "-quiet", "-nobrowse", "-mountpoint", mount.path, image.path]) + var info = statfs() + guard statfs(mount.path, &info) == 0 else { throw posix() } + let actual = withUnsafeBytes(of: info.f_fstypename) { String(decoding: $0.prefix { $0 != 0 }, as: UTF8.self) } + try require(actual == format.kernel, "Expected \(format.kernel), got \(actual)") + let probe = mount.appendingPathComponent("CaseProbe") + try Data([1]).write(to: probe) + let actualSensitive = !fm.fileExists(atPath: mount.appendingPathComponent("caseprobe").path) + try require(actualSensitive == format.caseSensitive, "Wrong case behavior: \(format.name)") + try fm.removeItem(at: probe) + print("DRIVE_MATRIX mounted format=\(format.name) role=\(role) actual=\(actual) case_sensitive=\(actualSensitive)") + return volume + } + + private func seed(at card: URL) throws { + let files: [(String, Data)] = [ + ("zero.bin", Data()), ("tiny.txt", Data([0x17])), + ("Folder with spaces/quote' dollar$ bracket[].txt", Data("synthetic sidecar".utf8)), + ("Nested/Deeper/cafe\u{301}.txt", Data(repeating: 0x38, count: 4097)), + ("雪 café/clip.bin", Data(repeating: 0x57, count: 65537)), + ("large-odd.bin", Data(repeating: 0x91, count: 4 * 1024 * 1024 + 17)) + ] + for (path, data) in files { + let file = card.appendingPathComponent(path) + try fm.createDirectory(at: file.deletingLastPathComponent(), withIntermediateDirectories: true) + try data.write(to: file) + } + } + + private func transfer(source: Volume, destinations: [Volume], mode: VerificationMode, pipeline: Bool, + sourceName: String = "Camera Card", reviewedExclusions: [String]? = nil, allowCompanionRefusal: Bool = false) async throws { + let card = source.mount.appendingPathComponent(sourceName) + let before = try snapshot(card) + // Match the shipped default while retaining the complete manifest. + // Finder may add valid companions to the FAT-family fixture itself. + let recognizedCompanions = try AppleDoubleSelection.review(source: card) + let exclusions = reviewedExclusions ?? recognizedCompanions + let label = "\(source.format.name)->\(destinations.map { $0.format.name }.joined(separator: "+"))/\(mode.rawValue)/pipeline=\(pipeline)/excluded=\(exclusions.count)" + let targets = destinations.enumerated().map { $0.element.mount.appendingPathComponent("case-\(scenarios)-\($0.offset)") } + defer { for target in targets { try? fm.removeItem(at: target) } } + var settings = CameraLabelSettings() + settings.excludedAppleDoublePaths = exclusions.isEmpty ? nil : exclusions + let sentinel = Data("unrelated original; must survive".utf8) + for target in targets { + let output = SafetyValidator.resolvedDestinationRoot(source: card, destination: target, settings: settings) + try fm.createDirectory(at: output, withIntermediateDirectories: true) + try sentinel.write(to: output.appendingPathComponent("existing-original.bin")) + } + let updates = Mutex<[OperationProgress]>([]) + let operation = try await TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared, + pipelinedVerification: pipeline).performFileOperation(sourceURL: card, destinationURLs: targets, + verificationMode: mode, settings: settings, estimatedTotalBytes: nil, + progressCallback: { update in updates.withLock { $0.append(update) } }, onFileResult: nil) + let selected = before.keys.filter { !exclusions.contains($0) } + try require(operation.results.count == before.count * targets.count, "Wrong result coverage: \(label)") + try require(operation.sourceManifest?.count == before.count, "Source manifest coverage: \(label)") + try require(operation.results.filter(\.excludedAppleDouble).count == exclusions.count * targets.count, "Exclusions: \(label)") + let failed = operation.results.filter { !$0.success && !$0.excludedAppleDouble } + if !failed.isEmpty { + try require(allowCompanionRefusal, "Unexpected failed results \(label): \(failed.map { $0.error?.localizedDescription ?? "nil" })") + for row in failed { + let relative = row.sourceURL.relativePath(to: card) + let collisionRefused = row.error?.localizedDescription.contains("hidden companion") == true + var changedMetadataDetected = false + if row.verificationResult?.isValid == false { + changedMetadataDetected = try digest(row.destinationURL) != before[relative]?.sha + } + try require(recognizedCompanions.contains(relative) && (collisionRefused || changedMetadataDetected), + "Unexpected metadata failure \(label) \(relative): \(row.statusDescription), \(row.error?.localizedDescription ?? "nil")") + } + expectedMetadataFailures += failed.count + } + for row in operation.results where !row.excludedAppleDouble && row.success { + let relative = row.sourceURL.relativePath(to: card) + try require(try digest(row.destinationURL) == before[relative]?.sha, "Independent destination digest: \(label) \(relative)") + if mode == .quick { + try require(row.verificationResult?.isValid != true, "Quick claimed content verification: \(label)") + } else { + try require(row.verificationResult?.isValid == true, "Missing content verification: \(label)") + } + } + let observed = updates.withLock { $0 } + try require(!observed.isEmpty, "No progress: \(label)") + try require(zip(observed, observed.dropFirst()).allSatisfy { $0.overallProgress <= $1.overallProgress }, "Nonmonotonic progress: \(label)") + try require(observed.filter { $0.currentStage != .completed }.allSatisfy { $0.overallProgress < 1 }, "Active progress reached 100%: \(label)") + try require(observed.allSatisfy { $0.overallProgress >= 0 && $0.overallProgress <= 1 }, "Invalid progress: \(label)") + if failed.isEmpty { try require(observed.last?.overallProgress == 1, "Pipeline did not complete: \(label)") } + else { try require((observed.last?.overallProgress ?? 1) < 1, "Failed pipeline claimed complete: \(label)") } + let plan = TransferCompletion.ascmhlPlan(results: operation.results, sourceFiles: operation.sourceManifest, + destinations: targets, source: card, settings: settings) + var issues: [String] = [] + if !failed.isEmpty { + try require(plan.jobs.isEmpty && !plan.issues.isEmpty, "Failed metadata transfer planned a complete history: \(label)") + } + if mode == .quick { + try require(plan.jobs.isEmpty, "Quick planned a verified MHL: \(label)") + } else { + let mhlUpdates = Mutex<[ASCMHLGenerator.Progress]>([]) + issues = try TransferCompletion.writeASCMHL(plan.jobs, planIssues: plan.issues, + startTime: operation.startTime, source: card, toolVersion: "synthetic-drive-matrix", + progress: { update in mhlUpdates.withLock { $0.append(update) } }) + if failed.isEmpty { + try require(issues.isEmpty && plan.jobs.count == targets.count, "MHL handoff: \(label) \(issues)") + let mhlObserved = mhlUpdates.withLock { $0 } + try require(mhlObserved.last?.published == true, "MHL not published: \(label)") + try require(zip(mhlObserved, mhlObserved.dropFirst()).allSatisfy { $0.bytesProcessed <= $1.bytesProcessed }, "MHL progress regressed: \(label)") + for job in plan.jobs { + let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + let checked = try await checker.check(checker.discover(under: job.root)) { _ in } + try require(checked.isIntact && checked.recordedFileCount == selected.count, "Saved Check: \(label), count=\(checked.recordedFileCount)/\(selected.count), missing=\(checked.missingPaths), changed=\(checked.changedPaths)") + try require(checked.changedPaths.isEmpty && checked.missingPaths.isEmpty, "Saved Check changed/missing: \(label)") + savedChecks += 1 + } + } + } + let verdict = TransferCompletion.verdict(rows: TransferCompletion.rows(from: operation), sourceFiles: operation.sourceManifest, + destinations: targets, source: card, settings: settings, mode: mode, generateASCMHL: mode != .quick, + handoffIssues: issues, reportIssue: nil, project: .init(didPersist: true, locallySafe: nil)) + try require(verdict.success == (mode != .quick && failed.isEmpty && exclusions.isEmpty), "Incorrect safety verdict: \(label) \(verdict.message)") + for target in targets { + let output = SafetyValidator.resolvedDestinationRoot(source: card, destination: target, settings: settings) + try require(try Data(contentsOf: output.appendingPathComponent("existing-original.bin")) == sentinel, "Sentinel changed: \(label)") + if mode == .quick { try require(!fm.fileExists(atPath: output.appendingPathComponent("ascmhl").path), "Quick wrote MHL: \(label)") } + } + try require(try snapshot(card) == before, "Source changed: \(label)") + scenarios += 1 + resultRows += operation.results.count + print("DRIVE_MATRIX PASS n=\(scenarios) \(label) rows=\(operation.results.count) failures=\(failed.count)") + } + + private func collision(source: Volume, backup: Volume, mode: VerificationMode) async throws { + let card = source.mount.appendingPathComponent("Collision Card") + try fm.createDirectory(at: card, withIntermediateDirectories: true) + try Data("original source payload".utf8).write(to: card.appendingPathComponent("clip.bin")) + let before = try snapshot(card) + let target = backup.mount.appendingPathComponent("collision-\(scenarios)") + defer { try? fm.removeItem(at: target) } + let output = SafetyValidator.resolvedDestinationRoot(source: card, destination: target, settings: .init()) + try fm.createDirectory(at: output, withIntermediateDirectories: true) + let protected = Data("conflicting preserved destination content".utf8) + try protected.write(to: output.appendingPathComponent("clip.bin")) + let operation = try await TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared) + .performFileOperation(sourceURL: card, destinationURLs: [target], verificationMode: mode, settings: .init(), + estimatedTotalBytes: nil, progressCallback: { _ in }, onFileResult: nil) + let mediaRows = operation.results.filter { $0.sourceURL.lastPathComponent == "clip.bin" } + try require(operation.results.count == before.count && mediaRows.count == 1 && mediaRows.first?.success == false, + "Collision must fail closed: \(backup.format.name)/\(mode), rows=\(operation.results.count)/\(before.count)") + try require(try Data(contentsOf: output.appendingPathComponent("clip.bin")) == protected, "Collision overwritten") + try require(try snapshot(card) == before, "Collision changed source") + let plan = TransferCompletion.ascmhlPlan(results: operation.results, sourceFiles: operation.sourceManifest, + destinations: [target], source: card, settings: .init()) + try require(plan.jobs.isEmpty, "Collision produced complete MHL plan") + scenarios += 1 + resultRows += operation.results.count + print("DRIVE_MATRIX PASS n=\(scenarios) no_overwrite=\(backup.format.name)/\(mode)") + } + + private func makeRealAppleDouble(on mount: URL) throws -> Data { + let media = mount.appendingPathComponent("metadata-seed.bin") + try Data("metadata seed".utf8).write(to: media) + try command("/usr/bin/xattr", ["-w", "com.bitmatch.synthetic", "synthetic value", media.path]) + let data = try Data(contentsOf: mount.appendingPathComponent("._metadata-seed.bin")) + try require(data.count >= 26, "Missing generated AppleDouble") + return data + } + + private func snapshot(_ card: URL) throws -> [String: Snapshot] { + try Dictionary(uniqueKeysWithValues: CardSource.enumerateRegularFiles(base: card).map { + ($0.relativePath, Snapshot(size: $0.size, modified: $0.modificationDate ?? .distantPast, sha: try digest($0.url))) + }) + } + + private func digest(_ file: URL) throws -> String { + let handle = try FileHandle(forReadingFrom: file) + defer { try? handle.close() } + var hasher = SHA256() + while let bytes = try autoreleasepool(invoking: { try handle.read(upToCount: 1024 * 1024) }), !bytes.isEmpty { + hasher.update(data: bytes) + } + return hasher.finalize().map { String(format: "%02x", $0) }.joined() + } + + func cleanup() throws { + var failures: [String] = [] + for volume in mounted.reversed() { + do { + if try mountedPaths().contains(volume.mount.path) { + try command("/usr/bin/hdiutil", ["detach", "-quiet", volume.mount.path]) + } + mounted.removeAll { $0.mount == volume.mount } + } + catch { failures.append("\(volume.mount.path): \(error)") } + } + if !failures.isEmpty { throw NSError(domain: "MountedDriveMatrix.cleanup", code: 1, userInfo: [NSLocalizedDescriptionKey: failures.joined(separator: "; ") + "; retained " + root.path]) } + try require(try mountedPaths().allSatisfy { $0 != root.path && !$0.hasPrefix(root.path + "/") }, + "A filesystem is still mounted beneath the owned root; retained \(root.path)") + mounted.removeAll() + if fm.fileExists(atPath: root.path) { try fm.removeItem(at: root) } + print("DRIVE_MATRIX cleanup=all_images_detached_and_removed") + } + + private func mountedPaths() throws -> Set { + var entries: UnsafeMutablePointer? + let count = getmntinfo(&entries, MNT_NOWAIT) + guard count > 0, let entries else { throw posix() } + return Set((0.. NSError { NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } +} +#endif From c09ced8c6b3d2d9df5da5a28ab04e532310bf7bc Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 21:20:25 -0400 Subject: [PATCH 04/56] Keep report names and identity within page layouts --- BitMatch-iPadTests/ReportPDFLayoutTests.swift | 12 +++- BitMatchTests/ReportPDFLayoutTests.swift | 38 +++++++++++- BitMatchTests/ReportThumbnailTests.swift | 15 ++++- .../SharedReportGenerationSmokeTests.swift | 40 ++++++++++++ .../Services/ReportThumbnailService.swift | 7 ++- .../SharedReportGenerationService.swift | 61 +++++++++++++------ Shared/Views/ReportPDFLayout.swift | 21 ++++--- Shared/Views/ReportPDFRenderer.swift | 7 ++- 8 files changed, 166 insertions(+), 35 deletions(-) diff --git a/BitMatch-iPadTests/ReportPDFLayoutTests.swift b/BitMatch-iPadTests/ReportPDFLayoutTests.swift index 69b9c388..5301c320 100644 --- a/BitMatch-iPadTests/ReportPDFLayoutTests.swift +++ b/BitMatch-iPadTests/ReportPDFLayoutTests.swift @@ -66,7 +66,8 @@ struct ReportPDFLayoutTests { clientLogoData: nil, companyLogoData: nil, photographyJob: nil, safetyState: .safeToErase) let blocks = ReportView(s: summary, rows: rows).pdfBlocks let heights = ReportPDFRenderer.blockHeights(blocks) - let pages = ReportPDFLayout.pages(blockHeights: heights, contentHeight: ReportPDFRenderer.contentHeight) + let pages = ReportPDFLayout.pages(blockHeights: heights, contentHeight: ReportPDFRenderer.contentHeight, + firstPageContentHeight: ReportPDFRenderer.firstPageContentHeight) let document = try #require(PDFDocument(data: ReportPDFRenderer.renderPDF(summary: summary, results: rows))) #expect(pages.count > 1) #expect(document.pageCount == pages.count) @@ -93,4 +94,13 @@ struct ReportPDFLayoutTests { } } + + @Test func firstPageUsesSpaceThatContinuationHeadersNeedLater() { + let packed = ReportPDFLayout.pages(blockHeights: [60, 60, 20], contentHeight: 100, firstPageContentHeight: 150) + #expect(packed.map(\.blocks) == [0..<3]) + let sliced = ReportPDFLayout.pages(blockHeights: [250, 50], contentHeight: 100, firstPageContentHeight: 150) + #expect(sliced.map(\.blocks) == [0..<1, 0..<1, 1..<2]) + #expect(sliced.map(\.offset) == [0, 150, 0]) + } + } diff --git a/BitMatchTests/ReportPDFLayoutTests.swift b/BitMatchTests/ReportPDFLayoutTests.swift index 15c64202..c261b9e6 100644 --- a/BitMatchTests/ReportPDFLayoutTests.swift +++ b/BitMatchTests/ReportPDFLayoutTests.swift @@ -169,7 +169,8 @@ struct ReportPDFLayoutTests { clientLogoData: nil, companyLogoData: nil, photographyJob: nil, safetyState: .safeToErase) let blocks = ReportView(s: summary, rows: rows).pdfBlocks let heights = ReportPDFRenderer.blockHeights(blocks) - let pages = ReportPDFLayout.pages(blockHeights: heights, contentHeight: ReportPDFRenderer.contentHeight) + let pages = ReportPDFLayout.pages(blockHeights: heights, contentHeight: ReportPDFRenderer.contentHeight, + firstPageContentHeight: ReportPDFRenderer.firstPageContentHeight) let document = try #require(PDFDocument(data: ReportPDFRenderer.renderPDF(summary: summary, results: rows))) #expect(pages.count > 1) #expect(document.pageCount == pages.count) @@ -196,4 +197,39 @@ struct ReportPDFLayoutTests { } } + + @Test func cancelledSevenRowHistoryKeepsIdentityOnTheContentPage() async throws { + let root = FileManager.default.temporaryDirectory.appendingPathComponent("bm-" + UUID().uuidString) + let source = root.appendingPathComponent("Cancel Retry Card-E1061BD4-DFE4-4CC0-8053-58C5263AFFB2") + let backup = root.appendingPathComponent("destination-apfs/cancel-retry-E1061BD4-DFE4-4CC0-8053-58C5263AFFB2") + for folder in [source, backup] { try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) } + defer { try? FileManager.default.removeItem(at: root) } + var record = LocalTransferRecord(id: UUID(), createdAt: Date(), source: try LocalTransferResource(url: source), + destinations: [try LocalTransferResource(url: backup)], verificationMode: .standard, + cameraSettings: CameraLabelSettings(), reportSettings: ReportPrefs()) + record.state = .cancelled + record.startedAt = Date(); record.endedAt = Date() + record.plannedFileCounts = [7] + record.results = ["clip00.mov", "clip01.xml", "metadata.txt", "clip03.bin", "empty.raw", "clip05.mov", "clip06.txt"].map { name in + ResultRow(path: source.appendingPathComponent(name).path, status: ResultOutcome.verified.statusText, + size: 10, checksum: "fixture", destination: backup.lastPathComponent, + destinationPath: backup.appendingPathComponent(name).path) + } + let document = try #require(PDFDocument(data: await ReportExporter.historyPDF(record: record, includeThumbnails: false))) + #expect(document.pageCount == 1) + let page = try #require(document.page(at: 0)) + #expect(page.string?.contains("NOT SAFE TO ERASE") == true) + #expect(page.string?.contains(record.id.uuidString) == true) + for row in record.results { #expect(page.string?.contains(row.fileName) == true) } + } + + + @Test func firstPageUsesSpaceThatContinuationHeadersNeedLater() { + let packed = ReportPDFLayout.pages(blockHeights: [60, 60, 20], contentHeight: 100, firstPageContentHeight: 150) + #expect(packed.map(\.blocks) == [0..<3]) + let sliced = ReportPDFLayout.pages(blockHeights: [250, 50], contentHeight: 100, firstPageContentHeight: 150) + #expect(sliced.map(\.blocks) == [0..<1, 0..<1, 1..<2]) + #expect(sliced.map(\.offset) == [0, 150, 0]) + } + } diff --git a/BitMatchTests/ReportThumbnailTests.swift b/BitMatchTests/ReportThumbnailTests.swift index 56541b2a..450f50fc 100644 --- a/BitMatchTests/ReportThumbnailTests.swift +++ b/BitMatchTests/ReportThumbnailTests.swift @@ -29,6 +29,19 @@ struct ReportThumbnailTests { #expect(data.count <= 64 * 1024) } + @Test func metadataCompanionsDoNotCountAsClipsButOrdinaryHiddenClipsDo() async throws { + let (root, _, row) = try seed(); defer { try? FileManager.default.removeItem(at: root) } + let metadata = ResultRow(path: "/card/._preview.mp4", status: row.status, size: row.size, + checksum: row.checksum, destination: row.destination, destinationPath: row.destinationPath) + let hidden = ResultRow(path: "/card/.hidden.mov", status: row.status, size: row.size, + checksum: row.checksum, destination: row.destination, destinationPath: row.destinationPath) + let frames = try await ReportThumbnailService.collect(results: [row, metadata, hidden], enabled: true) { _, _ in Data([1]) } + #expect(frames.requestedCount == 2) + #expect(frames.images.count == 2) + #expect(frames.images[metadata.path] == nil) + #expect(frames.images[hidden.path] != nil) + } + @Test func disabledAndQuickNeverStartExtraction() async throws { let (_, _, row) = try seed() defer { if let path = row.destinationPath { try? FileManager.default.removeItem(at: URL(fileURLWithPath: path).deletingLastPathComponent()) } } @@ -264,7 +277,7 @@ struct ReportThumbnailTests { #expect(text.contains("Operator: Alex & Sam")) #expect(text.contains("preview.mp4")) #expect(text.contains("unsupported.braw")) - let previewCount = exclude ? "Clip previews: 1/1" : "Clip previews: 1/2" + let previewCount = "Clip previews: 1/1" #expect(text.contains(previewCount) == (mode != .quick)) let jsonURL = try #require(reports.first { $0.pathExtension == "json" }) let json = try #require(JSONSerialization.jsonObject(with: Data(contentsOf: jsonURL)) as? [String: Any]) diff --git a/BitMatchTests/SharedReportGenerationSmokeTests.swift b/BitMatchTests/SharedReportGenerationSmokeTests.swift index 7ef67d68..dc153a54 100644 --- a/BitMatchTests/SharedReportGenerationSmokeTests.swift +++ b/BitMatchTests/SharedReportGenerationSmokeTests.swift @@ -1,5 +1,6 @@ // SharedReportGenerationSmokeTests.swift import Foundation +import PDFKit import Testing @testable import BitMatch @@ -46,4 +47,43 @@ struct SharedReportGenerationSmokeTests { #expect(result.reportData.summary.formattedSize == "Empty") #expect(!String(decoding: result.jsonData, as: UTF8.self).contains("Zero KB")) } + + @Test @MainActor + func longMasterTransferNamesRemainInsideThePage() async throws { + let sourceName = String(repeating: "SourceSegment", count: 9) + " ENDSOURCE" + let backupName = String(repeating: "BackupSegment", count: 12) + " ENDBACKUP" + let source = FolderInfo(url: URL(fileURLWithPath: "/card/" + sourceName), fileCount: 7, + totalSize: 1_000_000, lastModified: Date(), isInternalDrive: false) + let backups = (0..<3).map { index in + FolderInfo(url: URL(fileURLWithPath: "/backup\(index)/" + backupName), fileCount: 7, + totalSize: 1_000_000, lastModified: Date(), isInternalDrive: false) + } + let transfer = TransferCard(source: source, destinations: backups, cameraCard: nil, metadata: nil, + progress: 1, state: .completed(.init(success: true, message: "Verified"))) + let configuration = SharedReportGenerationService.ReportConfiguration.default() + for count in [1, 12] { + let result = try await SharedReportGenerationService().generateMasterReport(transfers: Array(repeating: transfer, count: count), configuration: configuration) + if count > 1 { #expect(result.reportData.summary.totalTransfers == count) } + let document = try #require(PDFDocument(data: result.pdfData)) + if count > 1 { #expect(document.pageCount > 1) } + #expect(document.string?.contains("ENDSOURCE") == true) + #expect(document.string?.contains("ENDBACKUP") == true) + for pageIndex in 0..= configuration.margins.left - 1) + #expect(bounds.maxX <= page.bounds(for: .mediaBox).maxX - configuration.margins.right + 1) + #expect(bounds.minY >= configuration.margins.bottom - 1) + } + } + if let directory = ProcessInfo.processInfo.environment["BITMATCH_MATRIX_ARTIFACTS"] { + let folder = URL(fileURLWithPath: directory) + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + try result.pdfData.write(to: folder.appendingPathComponent("long-master-\(count).pdf")) + } + } + } + } diff --git a/Shared/Core/Services/ReportThumbnailService.swift b/Shared/Core/Services/ReportThumbnailService.swift index d75ce38a..fa49aa22 100644 --- a/Shared/Core/Services/ReportThumbnailService.swift +++ b/Shared/Core/Services/ReportThumbnailService.swift @@ -28,7 +28,12 @@ enum ReportThumbnailService { var output = ReportThumbnails() let clock = ContinuousClock() let deadline = clock.now.advanced(by: .seconds(max(0, budget))) - let eligible = results.filter { $0.isVerifiedStatus && videoExtensions.contains(URL(fileURLWithPath: $0.path).pathExtension.lowercased()) } + let eligible = results.filter { + let url = URL(fileURLWithPath: $0.path) + // Reuse clip-only presentation rules. This does not exclude any + // file from copying, verification or the recorded transfer results. + return $0.isVerifiedStatus && videoExtensions.contains(url.pathExtension.lowercased()) && ClipIntegrityCheck.supports(url) + } output.requestedCount = Set(eligible.map(\.path)).count var seen: Set = [] var attempts = 0 diff --git a/Shared/Core/Services/SharedReportGenerationService.swift b/Shared/Core/Services/SharedReportGenerationService.swift index 45ff4eff..3c04ab3c 100644 --- a/Shared/Core/Services/SharedReportGenerationService.swift +++ b/Shared/Core/Services/SharedReportGenerationService.swift @@ -198,16 +198,11 @@ class SharedReportGenerationService: ObservableObject { } for cameraGroup in reportData.cameraGroups { - if yPosition > contentRect.maxY - 150 { - // Finish current page + yPosition = renderCameraGroup(cameraGroup, in: contentRect, at: yPosition, configuration: configuration) { NSGraphicsContext.restoreGraphicsState() context.endPDFPage() - - // Start new page setupNewPage() - yPosition = contentRect.minY } - yPosition = renderCameraGroup(cameraGroup, in: contentRect, at: yPosition, configuration: configuration) } // End final page @@ -265,13 +260,9 @@ class SharedReportGenerationService: ObservableObject { // Render camera groups for cameraGroup in reportData.cameraGroups { - // Check if we need a new page - if yPosition > contentRect.maxY - 150 { + yPosition = renderCameraGroup(cameraGroup, in: contentRect, at: yPosition, configuration: configuration) { context.beginPage() - yPosition = contentRect.minY } - - yPosition = renderCameraGroup(cameraGroup, in: contentRect, at: yPosition, configuration: configuration) } } #endif @@ -306,8 +297,7 @@ class SharedReportGenerationService: ObservableObject { ].filter { !$0.hasSuffix(": ") } // Remove empty fields for line in metadata { - line.draw(at: CGPoint(x: rect.minX, y: currentY), withAttributes: metadataAttributes) - currentY += configuration.fontSize + 4 + currentY += drawWrapped(line, in: rect, at: currentY, attributes: metadataAttributes) + 2 } return currentY + 20 @@ -382,8 +372,7 @@ class SharedReportGenerationService: ObservableObject { } for line in summaryLines { - line.draw(at: CGPoint(x: rect.minX, y: currentY), withAttributes: summaryAttributes) - currentY += configuration.fontSize + 4 + currentY += drawWrapped(line, in: rect, at: currentY, attributes: summaryAttributes) + 2 } return currentY + 20 @@ -413,7 +402,7 @@ class SharedReportGenerationService: ObservableObject { return currentY + 80 } - private func renderCameraGroup(_ group: ReportData.CameraGroup, in rect: CGRect, at yPosition: CGFloat, configuration: ReportConfiguration) -> CGFloat { + private func renderCameraGroup(_ group: ReportData.CameraGroup, in rect: CGRect, at yPosition: CGFloat, configuration: ReportConfiguration, newPage: () -> Void) -> CGFloat { var currentY = yPosition // Camera group header @@ -423,8 +412,12 @@ class SharedReportGenerationService: ObservableObject { ] let groupTitle = "\(group.cameraName) (\(group.transfers.count) transfers)" - groupTitle.draw(at: CGPoint(x: rect.minX, y: currentY), withAttributes: headerAttributes) - currentY += 16 + let titleHeight = wrappedHeight(groupTitle, width: rect.width, attributes: headerAttributes) + if currentY + titleHeight + 20 > rect.maxY { + newPage() + currentY = rect.minY + } + currentY += drawWrapped(groupTitle, in: rect, at: currentY, attributes: headerAttributes) + 3 // Transfer details let detailAttributes = [ @@ -454,13 +447,41 @@ class SharedReportGenerationService: ObservableObject { } let transferLine = "\(statusSymbol) \(sourceLeaf) → \(destinationSummary) • \(transfer.fileCount) files • \(transferSize)" - transferLine.draw(at: CGPoint(x: rect.minX, y: currentY), withAttributes: detailAttributes) - currentY += configuration.fontSize + 4 + let height = wrappedHeight(transferLine, width: rect.width, attributes: detailAttributes) + if currentY + height > rect.maxY { + newPage() + currentY = rect.minY + currentY += drawWrapped(groupTitle + " (continued)", in: rect, at: currentY, attributes: headerAttributes) + 3 + } + currentY += drawWrapped(transferLine, in: rect, at: currentY, attributes: detailAttributes) + 4 } return currentY + 10 } + /// User-chosen card and backup names may be a full filename long. Measure + /// the same wrapped text that is drawn, rather than drawing past the page. + private func wrappedText(_ text: String, attributes: [NSAttributedString.Key: Any]) -> NSAttributedString { + let paragraph = NSMutableParagraphStyle() + paragraph.lineBreakMode = .byWordWrapping + var attributes = attributes + attributes[.paragraphStyle] = paragraph + return NSAttributedString(string: text, attributes: attributes) + } + + private func wrappedHeight(_ text: String, width: CGFloat, attributes: [NSAttributedString.Key: Any]) -> CGFloat { + ceil(wrappedText(text, attributes: attributes).boundingRect( + with: CGSize(width: width, height: .greatestFiniteMagnitude), + options: [.usesLineFragmentOrigin, .usesFontLeading], context: nil).height) + } + + @discardableResult + private func drawWrapped(_ text: String, in rect: CGRect, at y: CGFloat, attributes: [NSAttributedString.Key: Any]) -> CGFloat { + let height = wrappedHeight(text, width: rect.width, attributes: attributes) + wrappedText(text, attributes: attributes).draw(in: CGRect(x: rect.minX, y: y, width: rect.width, height: height)) + return height + } + // MARK: - JSON Generation private func generateJSON(transfers: [TransferCard], configuration: ReportConfiguration) throws -> Data { diff --git a/Shared/Views/ReportPDFLayout.swift b/Shared/Views/ReportPDFLayout.swift index 136f07a8..086970e1 100644 --- a/Shared/Views/ReportPDFLayout.swift +++ b/Shared/Views/ReportPDFLayout.swift @@ -7,29 +7,32 @@ enum ReportPDFLayout { let offset: CGFloat } - static func pages(blockHeights: [CGFloat], contentHeight: CGFloat) -> [Page] { + static func pages(blockHeights: [CGFloat], contentHeight: CGFloat, firstPageContentHeight: CGFloat? = nil) -> [Page] { precondition(contentHeight.isFinite && contentHeight > 0) + let firstHeight = firstPageContentHeight ?? contentHeight + precondition(firstHeight.isFinite && firstHeight > 0) precondition(blockHeights.allSatisfy { $0.isFinite && $0 >= 0 }) var pages: [Page] = [] var start = 0 var used: CGFloat = 0 + var capacity: CGFloat { pages.isEmpty ? firstHeight : contentHeight } for (index, height) in blockHeights.enumerated() { - if height > contentHeight { - if start < index { pages.append(Page(blocks: start.. capacity, start < index { + pages.append(Page(blocks: start.. capacity { var offset: CGFloat = 0 while offset < height { + let sliceHeight = capacity pages.append(Page(blocks: index..<(index + 1), offset: offset)) - offset += contentHeight + offset += sliceHeight } start = index + 1 used = 0 } else { - if used + height > contentHeight { - pages.append(Page(blocks: start.. Date: Mon, 19 Oct 2026 21:23:36 -0400 Subject: [PATCH 05/56] Preserve full filenames when wrapping Master summaries --- BitMatchTests/SharedReportGenerationSmokeTests.swift | 7 +++++++ Shared/Core/Services/SharedReportGenerationService.swift | 8 ++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/BitMatchTests/SharedReportGenerationSmokeTests.swift b/BitMatchTests/SharedReportGenerationSmokeTests.swift index dc153a54..03baa59b 100644 --- a/BitMatchTests/SharedReportGenerationSmokeTests.swift +++ b/BitMatchTests/SharedReportGenerationSmokeTests.swift @@ -68,6 +68,13 @@ struct SharedReportGenerationSmokeTests { if count > 1 { #expect(document.pageCount > 1) } #expect(document.string?.contains("ENDSOURCE") == true) #expect(document.string?.contains("ENDBACKUP") == true) + let text = try #require(document.string) + let largest = try #require(text.components(separatedBy: "Largest Transfer:").dropFirst().first) + .components(separatedBy: "Most Used Destination:")[0] + #expect(largest.contains("ENDSOURCE")) + let destination = try #require(text.components(separatedBy: "Most Used Destination:").dropFirst().first) + .components(separatedBy: "Unknown Camera")[0] + #expect(destination.contains("ENDBACKUP")) for pageIndex in 0.. NSAttributedString { let paragraph = NSMutableParagraphStyle() - paragraph.lineBreakMode = .byWordWrapping + // Filenames can be a single long token. Word wrapping can measure it + // as one line while drawing clips the rest of the name. + paragraph.lineBreakMode = .byCharWrapping var attributes = attributes attributes[.paragraphStyle] = paragraph return NSAttributedString(string: text, attributes: attributes) @@ -478,7 +480,9 @@ class SharedReportGenerationService: ObservableObject { @discardableResult private func drawWrapped(_ text: String, in rect: CGRect, at y: CGFloat, attributes: [NSAttributedString.Key: Any]) -> CGFloat { let height = wrappedHeight(text, width: rect.width, attributes: attributes) - wrappedText(text, attributes: attributes).draw(in: CGRect(x: rect.minX, y: y, width: rect.width, height: height)) + wrappedText(text, attributes: attributes).draw( + with: CGRect(x: rect.minX, y: y, width: rect.width, height: height), + options: [.usesLineFragmentOrigin, .usesFontLeading], context: nil) return height } From 38eb518e29fdce8044c9d626b7e74a572b10474d Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 21:25:24 -0400 Subject: [PATCH 06/56] Exercise mounted workflow evidence and recovery --- .../MountedWorkflowEvidenceMatrixTests.swift | 416 ++++++++++++++++++ Scripts/test-full-transfer-matrix.sh | 37 +- 2 files changed, 447 insertions(+), 6 deletions(-) create mode 100644 BitMatchTests/MountedWorkflowEvidenceMatrixTests.swift diff --git a/BitMatchTests/MountedWorkflowEvidenceMatrixTests.swift b/BitMatchTests/MountedWorkflowEvidenceMatrixTests.swift new file mode 100644 index 00000000..96697de7 --- /dev/null +++ b/BitMatchTests/MountedWorkflowEvidenceMatrixTests.swift @@ -0,0 +1,416 @@ +@preconcurrency import AVFoundation +import CoreVideo +import CryptoKit +import Darwin +import Foundation +import PDFKit +import XCTest +@testable import BitMatch +import BitMatchEngine + +/// Opt-in app-level evidence checks over the real mounted image pool used by +/// test-full-transfer-matrix.sh. No completed record or successful row is seeded. +/// The runner unmounts the owned images before deleting these unique case folders. +@MainActor +final class MountedWorkflowEvidenceMatrixTests: XCTestCase { + private let filesystems = ["apfs", "hfs", "exfat", "apfsx", "fat"] + + func testQueuedPoliciesPersistThroughReportsHistoryAndMasterReport() async throws { + let pool = try ownedPool() + let run = UUID().uuidString + let source = pool.appendingPathComponent("source-apfs/Workflow Evidence Card-\(run)") + try await seed(source) + let original = try hashes(source) + var completed = 0 + var metadataRefusals = 0 + for (index, filesystem) in filesystems.enumerated() { + for ignore in [false, true] { + let key = "evidence-\(filesystem)-ignore-\(ignore)-\(run)" + let destinationFilesystems = [filesystem, filesystems[(index + 1) % filesystems.count]] + let destinations = destinationFilesystems.map { + pool.appendingPathComponent("destination-\($0)/\(key)") + } + for destination in destinations { try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) } + let suite = "BitMatchMountedEvidence.\(UUID())" + let defaults = try XCTUnwrap(UserDefaults(suiteName: suite)) + defer { defaults.removePersistentDomain(forName: suite) } + // A queued policy must survive a conflicting current preference. + defaults.set(!ignore, forKey: "BitMatchExcludeAppleDouble") + let journalURL = pool.appendingPathComponent("\(key)-history.json") + let journal = LocalTransferJournal(fileURL: journalURL) + var settings = CameraLabelSettings(label: "A Cam café", position: .suffix, separator: .dash, groupByCamera: true) + if ignore { settings.excludedAppleDoublePaths = try AppleDoubleSelection.review(source: source) } + var prefs = ReportPrefs(notes: "Take 2, \"good\"\nKeep all sidecars", operatorName: "Alex & Sam") + prefs.includeThumbnails = true + let mode = [VerificationMode.standard, .thorough, .paranoid][index % 3] + let id = try journal.enqueue(sourceURL: source, destinationURLs: destinations, + verificationMode: mode, cameraSettings: settings, reportSettings: prefs, generateASCMHL: true) + let operations = MountedEvidenceOperations() + let coordinator = SharedAppCoordinator(platformManager: MountedEvidencePlatformManager(fileOperations: operations), + transferJournal: journal, defaults: defaults) + coordinator.startQueue() + let ended = await waitUntil(timeout: .seconds(45)) { + !coordinator.queueIsRunning && !coordinator.isOperationInProgress + } + if !ended { coordinator.cancelOperation() } + XCTAssertTrue(ended, key) + // Decode the saved bytes without opening a second exclusive journal owner. + let savedRecords = try JSONDecoder().decode([LocalTransferRecord].self, from: Data(contentsOf: journalURL)) + let record = try XCTUnwrap(savedRecords.first { $0.id == id }) + let failedRows = record.results.filter { !$0.isVerifiedStatus && ResultOutcome(statusText: $0.status) != .excludedAppleDouble } + let competingBytes = await operations.witnesses.snapshots + for row in failedRows { + XCTAssertFalse(ignore, "An excluded source metadata file must not be attempted") + XCTAssertEqual(URL(fileURLWithPath: row.path).lastPathComponent, "._clip.mov", row.status) + let path = try XCTUnwrap(row.destinationPath) + XCTAssertTrue(path.contains("/destination-fat/") || path.contains("/destination-exfat/"), row.status) + let refusal = try XCTUnwrap(competingBytes[path]) + XCTAssertEqual(refusal.code, NSFileWriteFileExistsError) + XCTAssertEqual(refusal.domain, "DestinationWriter") + XCTAssertEqual(ResultOutcome(statusText: row.status), .failed) + XCTAssertEqual(try digestFile(URL(fileURLWithPath: path)), refusal.sha256, "Competing metadata changed after refusal") + } + metadataRefusals += failedRows.count + let wholeCardPassed = !ignore && failedRows.isEmpty + XCTAssertEqual(record.state, wholeCardPassed ? .completed : .issues, "\(key): \(record.summary)") + XCTAssertEqual(record.results.count, original.count * 2, key) + let selectedCount = original.count - (ignore ? 1 : 0) + XCTAssertEqual(record.plannedFileCounts, [selectedCount, selectedCount], key) + XCTAssertEqual(record.results.filter { ResultOutcome(statusText: $0.status) == .excludedAppleDouble }.count, ignore ? 2 : 0, key) + XCTAssertEqual(record.results.filter(\.isVerifiedStatus).count, (original.count - (ignore ? 1 : 0)) * 2 - failedRows.count, key) + XCTAssertEqual(TransferLibraryPresentation.safetyState(for: record) == .safeToErase, wholeCardPassed, key) + XCTAssertEqual(try hashes(source), original, "Source changed: \(key)") + for (backup, destination) in destinations.enumerated() { + let output = try SafetyValidator.resolvedDestinationRootChecked(source: source, destination: destination, settings: settings) + XCTAssertTrue(PathContainment.isWithin(output.path, root: destination.path), key) + let backupFailed = failedRows.contains { row in + row.destinationPath.map { PathContainment.isWithin($0, root: output.path) } == true + } + for (relative, digest) in original { + let path = output.appendingPathComponent(relative) + if relative == "._clip.mov" && (ignore || backupFailed) { + // Ignoring source metadata does not suppress metadata created by + // macOS itself. Preserve a competing companion instead of replacing it. + if FileManager.default.fileExists(atPath: path.path) { + XCTAssertTrue(try AppleDoubleSelection.review(source: output).contains(relative), key) + if !ignore { XCTAssertNotEqual(try digestFile(path), digest, key) } + } + } else { XCTAssertEqual(try digestFile(path), digest, "\(key) \(relative)") } + } + XCTAssertEqual(FileManager.default.fileExists(atPath: output.appendingPathComponent("ascmhl/ascmhl_chain.xml").path), !backupFailed, "\(key) backup=\(backup) fs=\(destinationFilesystems[backup])") + } + let persisted = try JSONDecoder().decode([LocalTransferRecord].self, from: Data(contentsOf: journalURL)) + XCTAssertEqual(persisted.first?.state, record.state, key) + XCTAssertEqual(persisted.first?.results.count, record.results.count, key) + let historyJSON = try TransferHistoryDocument(record: record, asCSV: false).data + let object = try XCTUnwrap(JSONSerialization.jsonObject(with: historyJSON) as? [String: Any]) + XCTAssertEqual(object["savedSafeToErase"] as? Bool, wholeCardPassed, key) + XCTAssertEqual(object["operatorName"] as? String, "Alex & Sam", key) + XCTAssertEqual(object["notes"] as? String, prefs.notes, key) + XCTAssertEqual((object["results"] as? [[String: Any]])?.count, record.results.count, key) + XCTAssertNil(object["bookmark"], key) + let csv = String(decoding: try TransferHistoryDocument(record: record, asCSV: true).data, as: UTF8.self) + XCTAssertTrue(csv.contains("Alex & Sam"), key) + XCTAssertTrue(csv.contains("\"\"good\"\""), key) + XCTAssertTrue(csv.contains("savedSafeToErase"), key) + let historyPDF = try await ReportExporter.historyPDF(record: record, includeThumbnails: true) + let historyText = try pdfText(historyPDF) + XCTAssertTrue(historyText.contains("Regenerated from saved history"), key) + XCTAssertTrue(historyText.contains("Operator: Alex & Sam"), key) + let previewCandidates = 2 // Real movie plus the corrupt movie; metadata is not a clip preview. + XCTAssertTrue(historyText.contains("Clip previews: 1/\(previewCandidates)"), "\(key): \(historyText)") + let reports = try FileManager.default.contentsOfDirectory(at: destinations[0].appendingPathComponent("Reports"), includingPropertiesForKeys: nil) + let pdf = try XCTUnwrap(reports.first { $0.pathExtension == "pdf" }) + let automaticText = try pdfText(Data(contentsOf: pdf)) + XCTAssertTrue(automaticText.contains("Operator: Alex & Sam"), key) + XCTAssertTrue(automaticText.contains("Clip previews: 1/\(previewCandidates)"), key) + XCTAssertEqual(automaticText.contains("intentionally excluded"), ignore, key) + let json = try XCTUnwrap(reports.first { $0.pathExtension == "json" }) + let automaticJSON = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(contentsOf: json)) as? [String: Any]) + XCTAssertEqual(automaticJSON["safeToErase"] as? Bool, wholeCardPassed, key) + let scan = await ReportScanner.scanReports(at: destinations[0]) + XCTAssertTrue(scan.skipped.isEmpty, key) + XCTAssertEqual(scan.cards.count, 1, key) + let master = try await SharedReportGenerationService().generateMasterReport(transfers: scan.cards, configuration: .default()) + XCTAssertEqual(master.reportData.summary.totalTransfers, 1, key) + XCTAssertEqual(master.reportData.summary.totalFiles, original.count, key) + let masterText = try pdfText(master.pdfData) + XCTAssertTrue(masterText.contains("MASTER TRANSFER REPORT"), key) + XCTAssertFalse(masterText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, key) + if completed < 2, let artifacts = ProcessInfo.processInfo.environment["BITMATCH_MATRIX_ARTIFACTS"] { + let folder = URL(fileURLWithPath: artifacts) + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + let label = ignore ? "selected" : "complete" + try Data(contentsOf: pdf).write(to: folder.appendingPathComponent("\(label)-automatic-with-preview.pdf")) + try historyPDF.write(to: folder.appendingPathComponent("\(label)-saved-history-with-preview.pdf")) + try master.pdfData.write(to: folder.appendingPathComponent("\(label)-master.pdf")) + } + completed += 1 + print("MOUNTED_WORKFLOW_CASE \(key) mode=\(mode.rawValue) files=\(original.count) backups=2 preview=1/\(previewCandidates) metadataRefusals=\(failedRows.count) history=real") + } + } + XCTAssertEqual(completed, 10) + print("MOUNTED_WORKFLOW_SUMMARY cases=\(completed) transfers=\(completed) historyExports=30 automaticReports=10 masterReports=10 previews=20 metadataRefusals=\(metadataRefusals)") + } + + func testCancelBeforeMHLAndRetryRetainsBothAttemptsAndExistingHistories() async throws { + let pool = try ownedPool() + let run = UUID().uuidString + let source = pool.appendingPathComponent("source-apfs/Cancel Retry Card-\(run)") + try await seed(source) + try FileManager.default.removeItem(at: source.appendingPathComponent("._clip.mov")) + let original = try hashes(source) + var completed = 0 + for filesystem in filesystems { + let destination = pool.appendingPathComponent("destination-\(filesystem)/cancel-retry-\(run)") + try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) + let suite = "BitMatchMountedCancel.\(UUID())" + let defaults = try XCTUnwrap(UserDefaults(suiteName: suite)) + defer { defaults.removePersistentDomain(forName: suite) } + let journal = LocalTransferJournal(fileURL: pool.appendingPathComponent("cancel-\(filesystem)-\(run)-history.json")) + let id = try journal.enqueue(sourceURL: source, destinationURLs: [destination], verificationMode: .standard, + cameraSettings: CameraLabelSettings(), reportSettings: ReportPrefs(), generateASCMHL: true) + let operations = MountedBeforeHandoffGate() + let coordinator = SharedAppCoordinator(platformManager: MountedEvidencePlatformManager(fileOperations: operations), transferJournal: journal, defaults: defaults) + coordinator.startQueue() + let reached = await waitUntil(timeout: .seconds(30)) { await operations.gate.reached } + XCTAssertTrue(reached, filesystem) + coordinator.cancelOperation() + let ended = await waitUntil(timeout: .seconds(10)) { !coordinator.isOperationInProgress } + XCTAssertTrue(ended, filesystem) + let cancelled = try XCTUnwrap(journal.records.first { $0.id == id }) + XCTAssertEqual(cancelled.state, .cancelled, filesystem) + XCTAssertEqual(cancelled.results.count, original.count, filesystem) + XCTAssertNotEqual(TransferLibraryPresentation.safetyState(for: cancelled), .safeToErase, filesystem) + let cancelledJSON = try TransferHistoryDocument(record: cancelled, asCSV: false).data + let cancelledObject = try XCTUnwrap(JSONSerialization.jsonObject(with: cancelledJSON) as? [String: Any]) + XCTAssertEqual(cancelledObject["savedSafeToErase"] as? Bool, false, filesystem) + let cancelledPDF = try await ReportExporter.historyPDF(record: cancelled, includeThumbnails: false) + XCTAssertTrue(try pdfText(cancelledPDF).contains("NOT SAFE TO ERASE"), filesystem) + if filesystem == "apfs", let artifacts = ProcessInfo.processInfo.environment["BITMATCH_MATRIX_ARTIFACTS"] { + let folder = URL(fileURLWithPath: artifacts) + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + try cancelledPDF.write(to: folder.appendingPathComponent("cancelled-saved-history.pdf")) + } + XCTAssertFalse(FileManager.default.fileExists(atPath: destination.appendingPathComponent("Reports").path), filesystem) + let output = SafetyValidator.resolvedDestinationRoot(source: source, destination: destination, settings: CameraLabelSettings()) + XCTAssertFalse(FileManager.default.fileExists(atPath: output.appendingPathComponent("ascmhl").path), filesystem) + let live = SharedAppCoordinator(platformManager: MountedEvidencePlatformManager(fileOperations: TransferPipeline(fileSystem: MacOSFileSystemService.shared, checksum: ChecksumEngine.shared)), transferJournal: journal, defaults: defaults) + live.retryTransfer(id) + let retry = try XCTUnwrap(journal.records.first { $0.retryOf == id }?.id) + let settled = await waitUntil(timeout: .seconds(40)) { !live.queueIsRunning && !live.isOperationInProgress } + XCTAssertTrue(settled, filesystem) + let finished = try XCTUnwrap(journal.records.first { $0.id == retry }) + XCTAssertEqual(finished.state, .completed, "\(filesystem): \(finished.summary)") + XCTAssertEqual(finished.retryOf, id, filesystem) + XCTAssertEqual(finished.results.count, original.count, filesystem) + XCTAssertTrue(finished.results.allSatisfy { $0.isVerifiedStatus && $0.wasReused }, filesystem) + let originalAttempt = try XCTUnwrap(journal.records.first { $0.id == id }) + XCTAssertEqual(originalAttempt.state, cancelled.state, "Retry changed original evidence") + XCTAssertEqual(originalAttempt.summary, cancelled.summary) + XCTAssertEqual(originalAttempt.results.map(\.id), cancelled.results.map(\.id)) + XCTAssertEqual(originalAttempt.endedAt, cancelled.endedAt) + let history = try hashes(output.appendingPathComponent("ascmhl")) + XCTAssertFalse(history.isEmpty, filesystem) + // Requesting MHL again must refuse an existing history, preserve its + // bytes and keep the overall verdict amber despite verified media. + let repeatedID = try live.enqueue(source: source, destinations: [destination], verificationMode: .standard, + cameraSettings: CameraLabelSettings(), generateASCMHL: true, reportSettings: ReportPrefs()) + live.startQueue() + let repeatedSettled = await waitUntil(timeout: .seconds(40)) { !live.queueIsRunning && !live.isOperationInProgress } + XCTAssertTrue(repeatedSettled, filesystem) + let repeated = try XCTUnwrap(journal.records.first { $0.id == repeatedID }) + XCTAssertEqual(repeated.state, .issues, repeated.summary) + XCTAssertTrue(repeated.results.allSatisfy { $0.isVerifiedStatus && $0.wasReused }, filesystem) + XCTAssertNotEqual(TransferLibraryPresentation.safetyState(for: repeated), .safeToErase, filesystem) + XCTAssertTrue(repeated.summary.contains("ASC MHL"), repeated.summary) + XCTAssertEqual(try hashes(output.appendingPathComponent("ascmhl")), history, filesystem) + XCTAssertEqual(try hashes(source), original, filesystem) + completed += 1 + print("MOUNTED_CANCEL_RETRY_CASE \(filesystem) original=cancelled retry=verified reused=\(finished.results.count) existingMHLRequest=issues historyPreserved=true") + } + XCTAssertEqual(completed, 5) + print("MOUNTED_CANCEL_RETRY_SUMMARY filesystems=\(completed) transfers=15") + } + + func testMountedOverlapConfigurationRejectedWithoutWrites() async throws { + let pool = try ownedPool() + var refused = 0 + for filesystem in filesystems { + let source = pool.appendingPathComponent("source-\(filesystem)/Invalid Overlap Card-\(UUID())") + try FileManager.default.createDirectory(at: source, withIntermediateDirectories: true) + try Data("Source must stay unchanged".utf8).write(to: source.appendingPathComponent("clip.bin")) + let original = try hashes(source) + let suite = "BitMatchMountedInvalid.\(UUID())" + let defaults = try XCTUnwrap(UserDefaults(suiteName: suite)) + defer { defaults.removePersistentDomain(forName: suite) } + let journal = LocalTransferJournal(fileURL: pool.appendingPathComponent("invalid-\(filesystem).json")) + let coordinator = SharedAppCoordinator(platformManager: MountedEvidencePlatformManager(fileOperations: TransferPipeline(fileSystem: MacOSFileSystemService.shared, checksum: ChecksumEngine.shared)), transferJournal: journal, defaults: defaults) + XCTAssertThrowsError(try coordinator.enqueue(source: source, destinations: [source])) + XCTAssertTrue(journal.records.isEmpty, filesystem) + XCTAssertEqual(try hashes(source), original, filesystem) + refused += 1 + } + XCTAssertEqual(refused, 5) + print("MOUNTED_INVALID_CONFIG_SUMMARY refused=\(refused) destinationWrites=0") + } + + private func ownedPool() throws -> URL { + guard let path = ProcessInfo.processInfo.environment["BITMATCH_MATRIX_ROOT"] else { throw XCTSkip("Run MATRIX_FOCUS=evidence Scripts/test-full-transfer-matrix.sh") } + let root = URL(fileURLWithPath: path) + guard FileManager.default.fileExists(atPath: root.appendingPathComponent(".bitmatch-matrix-owned").path) else { throw NSError(domain: "MountedWorkflowOwnership", code: 1) } + var devices = Set() + for role in ["source", "destination"] { + for (name, expected) in [("apfs", "apfs"), ("hfs", "hfs"), ("exfat", "exfat"), ("apfsx", "apfs"), ("fat", "msdos")] { + let mount = root.appendingPathComponent("\(role)-\(name)") + var filesystem = statfs() + guard statfs(mount.path, &filesystem) == 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } + let actual = withUnsafeBytes(of: filesystem.f_fstypename) { String(decoding: $0.prefix { $0 != 0 }, as: UTF8.self) } + XCTAssertEqual(actual, expected) + var identity = stat() + guard stat(mount.path, &identity) == 0, devices.insert(identity.st_dev).inserted else { throw NSError(domain: "MountedWorkflowSeparateImages", code: 1) } + } + } + return root + } + + private func seed(_ source: URL) async throws { + try FileManager.default.createDirectory(at: source.appendingPathComponent("DCIM/Sidecars"), withIntermediateDirectories: true) + try Data(repeating: 0x47, count: 1024 * 1024).write(to: source.appendingPathComponent("DCIM/clip.bin")) + try Data("日本語 café".utf8).write(to: source.appendingPathComponent("DCIM/Sidecars/clip.xml")) + try Data().write(to: source.appendingPathComponent("empty.raw")) + try Data("legitimate filename, not AppleDouble".utf8).write(to: source.appendingPathComponent("._notes.txt")) + try Data("camera metadata".utf8).write(to: source.appendingPathComponent(".camera-metadata")) + try Data("not a playable movie".utf8).write(to: source.appendingPathComponent("broken.mov")) + var bytes = [UInt8](repeating: 0, count: 42) + func put(_ value: UInt32, _ offset: Int) { for index in 0..<4 { bytes[offset + index] = UInt8(truncatingIfNeeded: value >> (24 - 8 * index)) } } + put(0x00051607, 0); put(0x00020000, 4); bytes[25] = 1 + put(2, 26); put(38, 30); put(4, 34) + try Data(bytes).write(to: source.appendingPathComponent("._clip.mov")) + try await makeMovie(source.appendingPathComponent("clip.mov")) + } + + private func makeMovie(_ url: URL) async throws { + let writer = try AVAssetWriter(outputURL: url, fileType: .mov) + let input = AVAssetWriterInput(mediaType: .video, outputSettings: [AVVideoCodecKey: AVVideoCodecType.h264, AVVideoWidthKey: 64, AVVideoHeightKey: 64]) + let adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input, sourcePixelBufferAttributes: [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32ARGB, kCVPixelBufferWidthKey as String: 64, kCVPixelBufferHeightKey as String: 64]) + writer.add(input) + XCTAssertTrue(writer.startWriting()) + writer.startSession(atSourceTime: .zero) + for index in 0..<2 { + var pixel: CVPixelBuffer? + XCTAssertEqual(CVPixelBufferCreate(kCFAllocatorDefault, 64, 64, kCVPixelFormatType_32ARGB, nil, &pixel), kCVReturnSuccess) + let buffer = try XCTUnwrap(pixel) + CVPixelBufferLockBaseAddress(buffer, []) + let address = try XCTUnwrap(CVPixelBufferGetBaseAddress(buffer)) + let pixels = address.assumingMemoryBound(to: UInt8.self) + let stride = CVPixelBufferGetBytesPerRow(buffer) + for y in 0..<64 { + for x in 0..<64 { + let offset = y * stride + x * 4 + pixels[offset] = 255 + pixels[offset + 1] = x < 32 ? 224 : 32 + pixels[offset + 2] = y < 32 ? 224 : 32 + pixels[offset + 3] = index == 0 ? 64 : 192 + } + } + CVPixelBufferUnlockBaseAddress(buffer, []) + let ready = await waitUntil(timeout: .seconds(10)) { input.isReadyForMoreMediaData } + XCTAssertTrue(ready) + XCTAssertTrue(adaptor.append(buffer, withPresentationTime: CMTime(value: Int64(index), timescale: 30))) + } + input.markAsFinished() + await writer.finishWriting() + XCTAssertEqual(writer.status, .completed, String(describing: writer.error)) + } + + private func hashes(_ root: URL) throws -> [String: String] { + try Dictionary(uniqueKeysWithValues: CardSource.enumerateRegularFiles(base: root).map { ($0.relativePath, try digestFile($0.url)) }) + } + private func digestFile(_ url: URL) throws -> String { SHA256.hash(data: try Data(contentsOf: url)).map { String(format: "%02x", $0) }.joined() } + private func pdfText(_ data: Data) throws -> String { + let document = try XCTUnwrap(PDFDocument(data: data)) + XCTAssertGreaterThan(document.pageCount, 0) + return try XCTUnwrap(document.string) + } +} + +private actor MountedHandoffGate { + private(set) var reached = false + private var waiter: CheckedContinuation? + private var cancelled = false + func hold() async throws { + reached = true + if !cancelled { await withCheckedContinuation { waiter = $0 } } + throw CancellationError() + } + func cancel() { cancelled = true; waiter?.resume(); waiter = nil } +} + +/// The real copy+verification completes, then the test holds the handoff until an +/// explicit coordinator cancellation. No operation results or digest are faked. +private final class MountedBeforeHandoffGate: FileOperationsService, @unchecked Sendable { + let gate = MountedHandoffGate() + private let pipeline = TransferPipeline(fileSystem: MacOSFileSystemService.shared, checksum: ChecksumEngine.shared) + func performFileOperation(sourceURL: URL, destinationURLs: [URL], verificationMode: VerificationMode, settings: CameraLabelSettings, + estimatedTotalBytes: Int64?, progressCallback: @escaping ProgressCallback, onFileResult: FileResultCallback?) async throws -> FileOperation { + _ = try await pipeline.performFileOperation(sourceURL: sourceURL, destinationURLs: destinationURLs, verificationMode: verificationMode, + settings: settings, estimatedTotalBytes: estimatedTotalBytes, progressCallback: progressCallback, onFileResult: onFileResult) + try await gate.hold() + throw CancellationError() + } + func cancelOperation() { pipeline.cancelOperation(); Task { await gate.cancel() } } + func pauseOperation() async { await pipeline.pauseOperation() } + func resumeOperation() async { await pipeline.resumeOperation() } +} + +/// Real macOS services and pipeline, without a modal error dialog in a test. +private final class MountedEvidencePlatformManager: PlatformManager { + private let real = MacOSPlatformManager.shared + nonisolated let fileOperations: FileOperationsService + init(fileOperations: FileOperationsService) { self.fileOperations = fileOperations } + nonisolated var fileSystem: FileSystemService { real.fileSystem } + nonisolated var checksum: ChecksumService { real.checksum } + nonisolated var cameraDetection: CameraDetectionService { real.cameraDetection } + nonisolated var supportsDragAndDrop: Bool { real.supportsDragAndDrop } + func presentAlert(title: String, message: String) async {} + func presentError(_ error: Error) async {} + func openURL(_ url: URL) async -> Bool { false } +} + +private struct MountedMetadataRefusal: Sendable { + let domain: String + let code: Int + let sha256: String +} + +private actor MountedMetadataWitnesses { + private(set) var snapshots: [String: MountedMetadataRefusal] = [:] + func record(_ result: FileOperationResult) throws { + guard !result.success, result.sourceURL.lastPathComponent == "._clip.mov" else { return } + let data = try Data(contentsOf: result.destinationURL) + let error = result.error as NSError? + snapshots[result.destinationURL.path] = MountedMetadataRefusal(domain: error?.domain ?? "missing", code: error?.code ?? 0, + sha256: SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined()) + } +} + +/// Retains competing bytes at the actual refusal, before MHL/report work, then +/// forwards every unmodified engine result to the real coordinator. +private final class MountedEvidenceOperations: FileOperationsService, @unchecked Sendable { + let witnesses = MountedMetadataWitnesses() + private let pipeline = TransferPipeline(fileSystem: MacOSFileSystemService.shared, checksum: ChecksumEngine.shared) + func performFileOperation(sourceURL: URL, destinationURLs: [URL], verificationMode: VerificationMode, settings: CameraLabelSettings, + estimatedTotalBytes: Int64?, progressCallback: @escaping ProgressCallback, onFileResult: FileResultCallback?) async throws -> FileOperation { + let witnesses = self.witnesses + return try await pipeline.performFileOperation(sourceURL: sourceURL, destinationURLs: destinationURLs, + verificationMode: verificationMode, settings: settings, estimatedTotalBytes: estimatedTotalBytes, + progressCallback: progressCallback, onFileResult: { result in + try? await witnesses.record(result) + await onFileResult?(result) + }) + } + func cancelOperation() { pipeline.cancelOperation() } + func pauseOperation() async { await pipeline.pauseOperation() } + func resumeOperation() async { await pipeline.resumeOperation() } +} diff --git a/Scripts/test-full-transfer-matrix.sh b/Scripts/test-full-transfer-matrix.sh index e49a0599..0274c118 100755 --- a/Scripts/test-full-transfer-matrix.sh +++ b/Scripts/test-full-transfer-matrix.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash # 800 option/filesystem combinations, each followed by a repeat/reuse run. +# MATRIX_FOCUS=all also checks naming, projects, evidence, collisions and the +# FAT32 size limit in the same image pool. Use evidence for app/history only. set -euo pipefail ROOT=$(cd "$(dirname "$0")/.." && pwd) TEST_NAME=testFullSyntheticFilesystemMatrix @@ -8,14 +10,24 @@ if [[ "${MATRIX_FOCUS:-full}" == collision ]]; then TEST_NAME=testCaseCollisions if [[ "${MATRIX_FOCUS:-full}" == features ]]; then TEST_NAME=testProjectAndReportFilesystemMatrix; fi if [[ "${MATRIX_FOCUS:-full}" == limits ]]; then TEST_NAME=testFAT32LargeFileLimitKeepsOtherBackupVerified; fi IMAGE_SIZE=3g -if [[ "${MATRIX_FOCUS:-full}" == limits ]]; then IMAGE_SIZE=6g; fi +if [[ "${MATRIX_FOCUS:-full}" == limits || "${MATRIX_FOCUS:-full}" == all ]]; then IMAGE_SIZE=6g; fi PAIR=$(mktemp -d /tmp/bitmatch-full-matrix.XXXXXX) touch "$PAIR/.bitmatch-matrix-owned" MOUNTS=() cleanup() { + local status=$? local busy=0 - for mount in "${MOUNTS[@]}"; do hdiutil detach -quiet "$mount" || busy=1; done - if [[ "$busy" == 0 ]]; then rm -rf "$PAIR"; else echo "Busy disposable images retained: $PAIR" >&2; fi + for mount in "${MOUNTS[@]-}"; do + [[ -n "$mount" ]] || continue + hdiutil detach -quiet "$mount" || busy=1 + done + if [[ "$busy" == 0 && -f "$PAIR/.bitmatch-matrix-owned" ]]; then + rm -rf "$PAIR" + else + echo "Disposable images retained for inspection: $PAIR" >&2 + if [[ "$status" == 0 ]]; then status=1; fi + fi + exit "$status" } trap cleanup EXIT for role in source destination; do @@ -23,14 +35,27 @@ for role in source destination; do name="$role-${item%%:*}"; fs=${item#*:} mkdir "$PAIR/$name" hdiutil create -size "$IMAGE_SIZE" -type SPARSE -fs "$fs" -volname "BM_${item%%:*}" -o "$PAIR/$name.sparseimage" - hdiutil attach -quiet -nobrowse -mountpoint "$PAIR/$name" "$PAIR/$name.sparseimage" MOUNTS+=("$PAIR/$name") + hdiutil attach -quiet -nobrowse -mountpoint "$PAIR/$name" "$PAIR/$name.sparseimage" done done +TESTS=("-only-testing:BitMatchTests/CopyVerifyExecutorIntegrityTests/$TEST_NAME") +if [[ "${MATRIX_FOCUS:-full}" == evidence ]]; then + TESTS=("-only-testing:BitMatchTests/MountedWorkflowEvidenceMatrixTests") +elif [[ "${MATRIX_FOCUS:-full}" == all ]]; then + TESTS=( + "-only-testing:BitMatchTests/CopyVerifyExecutorIntegrityTests/testFullSyntheticFilesystemMatrix" + "-only-testing:BitMatchTests/CopyVerifyExecutorIntegrityTests/testCameraRenameFilesystemMatrix" + "-only-testing:BitMatchTests/CopyVerifyExecutorIntegrityTests/testProjectAndReportFilesystemMatrix" + "-only-testing:BitMatchTests/CopyVerifyExecutorIntegrityTests/testCaseCollisionsAcrossRealFilesystems" + "-only-testing:BitMatchTests/CopyVerifyExecutorIntegrityTests/testFAT32LargeFileLimitKeepsOtherBackupVerified" + "-only-testing:BitMatchTests/MountedWorkflowEvidenceMatrixTests" + ) +fi if [[ -n "${MATRIX_KEY:-}" ]]; then export TEST_RUNNER_BITMATCH_MATRIX_KEY="$MATRIX_KEY"; fi TEST_RUNNER_BITMATCH_MATRIX_TRACE="${MATRIX_TRACE:-0}" \ TEST_RUNNER_BITMATCH_MATRIX_ROOT="$PAIR" \ xcodebuild -project "$ROOT/BitMatch.xcodeproj" -derivedDataPath "${MATRIX_DERIVED_DATA:-$ROOT/.derived-data/mac-test}" \ CODE_SIGNING_ALLOWED=NO test -scheme BitMatch -destination 'platform=macOS' \ - -parallel-testing-enabled NO \ - -only-testing:"BitMatchTests/CopyVerifyExecutorIntegrityTests/$TEST_NAME" + -jobs 4 -parallel-testing-enabled NO \ + "${TESTS[@]}" From 30888a540b9bfeadd807ab4a500931dae73cf018 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 21:20:38 -0400 Subject: [PATCH 07/56] Add an opt-in physical-drive validation runner --- Scripts/test-physical-drives.py | 415 ++++++++++++++++++++++++++++++++ 1 file changed, 415 insertions(+) create mode 100755 Scripts/test-physical-drives.py diff --git a/Scripts/test-physical-drives.py b/Scripts/test-physical-drives.py new file mode 100755 index 00000000..ab1f73fd --- /dev/null +++ b/Scripts/test-physical-drives.py @@ -0,0 +1,415 @@ +#!/usr/bin/env python3 +"""Run the engine's opt-in physical-drive tests in new, marked folders. + +Never formats or ejects a physical drive. Logs/config stay in /private/tmp. +The large case writes 119.6 GB and performs several full rereads. At an +illustrative (not measured) 40 MB/s, 478.4 GB of destination I/O takes 3.3 h. +A fast exFAT image keeps the file sizes but shares its selected host SSD. +On the large source SSD it is not an independent backup. An image never +adds a separate drive, USB bridge, or power-loss test. +Requires Python with descriptor-relative filesystem APIs (e.g. Homebrew +Python); older Framework Python builds are rejected before touching media. +""" +import argparse +import datetime as dt +import json +import os +from pathlib import Path +import platform +import plistlib +import re +import shutil +import signal +import stat +import subprocess +import sys +import tempfile +import uuid + + +class UnsafeFixture(RuntimeError): + pass + + +def plist_command(*args): + result = subprocess.run(list(args), check=True, capture_output=True, timeout=30) + return plistlib.loads(result.stdout) + + +def disk_info(path): + return plist_command('/usr/sbin/diskutil', 'info', '-plist', str(path)) + + +def direct_mount(value): + path = Path(value).absolute() + if path.parent != Path('/Volumes') or path.resolve(strict=True) != path or path.is_symlink(): + raise UnsafeFixture('Use a direct /Volumes/ mount, without symlink aliases.') + return path + + +def physical_volume(value): + path = direct_mount(value) + info = disk_info(path) + roles = info.get('APFSVolumeRoles', info.get('APFSVolumeRole', [])) + roles = [roles] if isinstance(roles, str) else roles + if (not info.get('Mounted') or info.get('MountPoint') != str(path) + or not info.get('VolumeUUID') or info.get('Internal') is not False + or info.get('ReadOnlyMedia') or info.get('ReadOnlyVolume') + or info.get('Writable') is False or info.get('BootVolume') or info.get('SystemImage') + or info.get('Bootable') is True + or any(str(role).lower() not in ('none', '') for role in roles) + or not os.access(path, os.W_OK | os.X_OK)): + raise UnsafeFixture('The selected mount must be writable, external data media.') + parent = info.get('ParentWholeDisk') + container = info.get('APFSContainerReference') + if container: + containers = plist_command('/usr/sbin/diskutil', 'apfs', 'list', '-plist').get('Containers', []) + stores = [store for item in containers if item.get('ContainerReference') == container + for store in item.get('PhysicalStores', [])] + if len(stores) != 1: + raise UnsafeFixture('An APFS role must resolve to exactly one physical store.') + parent = disk_info(stores[0]['DeviceIdentifier']).get('ParentWholeDisk') + if not parent: + raise UnsafeFixture('Cannot establish the physical ParentWholeDisk.') + whole = disk_info(parent) + if (not whole.get('Whole') or whole.get('Internal') is not False + or str(whole.get('VirtualOrPhysical', '')).lower() != 'physical' + or whole.get('DiskImage') or whole.get('BootVolume') or whole.get('SystemImage') + or whole.get('Bootable') is True): + raise UnsafeFixture('Virtual, internal, or ambiguous physical media is refused.') + return {'path': path, 'uuid': info['VolumeUUID'], 'parent': parent, + 'filesystem': info.get('FilesystemType', info.get('FileSystemPersonality', ''))} + + +def mounted_paths(): + output = subprocess.check_output(['/sbin/mount'], text=True, timeout=30) + paths = [] + for line in output.splitlines(): + if ' on ' not in line or ' (' not in line: + raise UnsafeFixture('Cannot parse the mount table; retaining fixtures.') + paths.append(Path(line.split(' on ', 1)[1].rsplit(' (', 1)[0])) + if Path('/') not in paths: + raise UnsafeFixture('The mount table is incomplete; retaining fixtures.') + return paths + + +def below(path, root): + return path == root or root in path.parents + + +def require_execution(log, method): + text = log.read_text(errors='replace') + rows = re.findall(r'Executed (\d+) tests?, with (?:(\d+) tests? skipped and )?(\d+) failures', text) + if (not rows or tuple(int(x or 0) for x in rows[-1]) != (1, 0, 0) + or not re.search(r"Test Case '-\[[^\]]+ " + re.escape(method) + r"\]' passed", text)): + raise UnsafeFixture('The requested XCTest did not execute and pass exactly once: ' + str(log)) + + +def remove_tree_at(parent_fd, name, identity, device): + """Descriptor-relative removal; never follows links or crosses a mount.""" + fd = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent_fd) + try: + current = os.fstat(fd) + if (current.st_dev, current.st_ino) != identity or current.st_dev != device: + raise UnsafeFixture('Fixture directory identity changed; retaining it.') + with os.scandir(fd) as entries: + ordered = sorted(entries, key=lambda item: item.name == '.bitmatch-validation-owned.json') + for entry in ordered: + item = entry.stat(follow_symlinks=False) + if stat.S_ISDIR(item.st_mode): + remove_tree_at(fd, entry.name, (item.st_dev, item.st_ino), device) + else: + os.unlink(entry.name, dir_fd=fd) + named = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + if (named.st_dev, named.st_ino) != identity: + raise UnsafeFixture('Fixture name was replaced; retaining replacement.') + finally: + os.close(fd) + os.rmdir(name, dir_fd=parent_fd) + + +class Run: + def __init__(self, volumes, large, image_host): + self.volumes, self.large, self.image_host = volumes, large, image_host + self.nonce = str(uuid.uuid4()) + self.workspace = Path(tempfile.mkdtemp(prefix='bitmatch-physical-', dir='/private/tmp')) + self.root_name = 'BitMatch-validation-' + dt.datetime.now().strftime('%Y%m%d-') + self.nonce + self.config = {'nonce': self.nonce, 'volumes': {}} + self.roots, self.identities = {}, {} + self.removed = set() + self.image = self.image_mount = self.child = None + self.image_identity = None + self.attach_started = self.attach_confirmed = False + self.warnings = [] + self.passed = [] + self.project = Path(__file__).resolve().parent.parent + self.build = self.workspace / 'build' + self.swift = ['/usr/bin/xcrun', 'swift'] + apple = subprocess.run(['/usr/sbin/sysctl', '-n', 'hw.optional.arm64'], capture_output=True, text=True) + if apple.returncode == 0 and apple.stdout.strip() == '1': + self.swift = ['/usr/bin/arch', '-arm64'] + self.swift + + def command(self, args, log, env=None, timeout=None): + print('Running ' + log.name + ' — receipts: ' + str(self.workspace), flush=True) + with log.open('wb') as output: + self.child = subprocess.Popen(args, stdout=output, stderr=subprocess.STDOUT, + env=env, start_new_session=True) + try: + status = self.child.wait(timeout=timeout) + finally: + if self.child.poll() is None: + for sig in (signal.SIGINT, signal.SIGTERM): + os.killpg(self.child.pid, sig) + try: + self.child.wait(timeout=10) + break + except subprocess.TimeoutExpired: + pass + if self.child.poll() is not None: + self.child = None + if status != 0: + raise UnsafeFixture('Command failed; inspect ' + str(log)) + + def make_root(self, key, mount, volume_uuid): + root = mount / self.root_name + root.mkdir(mode=0o700) # No existing directory is ever adopted. + self.roots[key] = root + item = root.lstat() + self.identities[key] = (item.st_dev, item.st_ino) + record = {'nonce': self.nonce, 'root': str(root), 'mount': str(mount), 'volumeUUID': volume_uuid} + marker = root / '.bitmatch-validation-owned.json' + fd = os.open(marker, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, 'w') as stream: + json.dump(record, stream) + stream.flush() + os.fsync(stream.fileno()) + directory_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + os.fsync(directory_fd) + item = os.fstat(directory_fd) + named = root.lstat() + if (item.st_dev, item.st_ino) != (named.st_dev, named.st_ino): + raise UnsafeFixture('Fixture root changed while saving its ownership marker.') + self.identities[key] = (item.st_dev, item.st_ino) + finally: + os.close(directory_fd) + self.config['volumes'][key] = record + + def validate_root(self, key): + root, record = self.roots[key], self.config['volumes'].get(key) + if not record or root.parent != Path(record['mount']): + raise UnsafeFixture('An unmarked fixture root is retained.') + current = disk_info(root.parent) + if current.get('VolumeUUID', '').casefold() != record['volumeUUID'].casefold(): + raise UnsafeFixture('Fixture volume UUID changed; retaining it.') + fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + item = os.fstat(fd) + if (item.st_dev, item.st_ino) != self.identities[key]: + raise UnsafeFixture('Fixture root was replaced; retaining it.') + marker_fd = os.open('.bitmatch-validation-owned.json', os.O_RDONLY | os.O_NOFOLLOW, dir_fd=fd) + with os.fdopen(marker_fd, 'rb') as stream: + if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode) or os.fstat(stream.fileno()).st_size > 4096: + raise UnsafeFixture('Fixture marker is not a bounded regular file.') + raw = stream.read(4097) + if len(raw) > 4096 or json.loads(raw) != record: + raise UnsafeFixture('Fixture marker changed; retaining it.') + finally: + os.close(fd) + + def image_attachments(self): + images = plist_command('/usr/bin/hdiutil', 'info', '-plist').get('images', []) + return [entry for entry in images if entry.get('image-path') == str(self.image)] + + def prepare_image(self): + key = self.image_host + self.validate_root(key) + self.image = self.roots[key] / 'LargeDestination.sparseimage' + self.image_mount = self.workspace / 'large-mount' + self.image_mount.mkdir(mode=0o700) + self.command(['/usr/bin/hdiutil', 'create', '-quiet', '-size', '160g', '-type', 'SPARSE', + '-fs', 'ExFAT', '-volname', 'BMLARGE', '-o', str(self.image)], self.workspace / 'image-create.log', timeout=120) + self.validate_root(key) + item = self.image.lstat() + if not stat.S_ISREG(item.st_mode) or item.st_dev != self.identities[key][0]: + raise UnsafeFixture('The image must be a new regular file on its selected physical volume.') + self.image_identity = (item.st_dev, item.st_ino) + self.attach_started = True + self.command(['/usr/bin/hdiutil', 'attach', '-quiet', '-nobrowse', '-mountpoint', + str(self.image_mount), str(self.image)], self.workspace / 'image-attach.log', timeout=120) + attachments = self.image_attachments() + points = [Path(entity['mount-point']) for image in attachments + for entity in image.get('system-entities', []) if entity.get('mount-point')] + if points != [self.image_mount] or self.image_mount not in mounted_paths(): + raise UnsafeFixture('Unexpected image attachment; retaining its host root.') + info = disk_info(self.image_mount) + if not info.get('VolumeUUID') or info.get('MountPoint') != str(self.image_mount): + raise UnsafeFixture('Cannot identify the owned image volume.') + self.attach_confirmed = True + self.make_root('largeDestination', self.image_mount, info['VolumeUUID']) + + def write_config(self): + path = self.workspace / 'config.json' + path.write_text(json.dumps(self.config, indent=2) + '\n') + path.chmod(0o600) + return path + + def execute(self): + test = self.project / 'Packages/BitMatchEngine/Tests/BitMatchEngineTests/PhysicalDriveTransferTests.swift' + if not test.is_file(): + raise UnsafeFixture('This checkout does not contain PhysicalDriveTransferTests.swift.') + self.command(self.swift + ['build', '--build-tests', '--jobs', '4', '--package-path', + str(self.project / 'Packages/BitMatchEngine'), '--scratch-path', str(self.build)], self.workspace / 'build.log') + for key, volume in self.volumes.items(): + refreshed = physical_volume(str(volume['path'])) + if refreshed['uuid'] != volume['uuid'] or refreshed['parent'] != volume['parent']: + raise UnsafeFixture('A selected physical volume changed before provisioning.') + self.make_root(key, volume['path'], volume['uuid']) + methods = ['testCardToTwoPhysicalBackupsAcrossModesAndReuse'] + if self.large: + methods.append('testEightLargeFilesThroughPhysicalCopyVerificationAndMHL') + for index, method in enumerate(methods): + if index == 1 and self.image_host: + self.prepare_image() + env = dict(os.environ, BITMATCH_HARDWARE_CONFIG=str(self.write_config())) + if index == 1: + env['BITMATCH_HARDWARE_LARGE'] = '1' + else: + env.pop('BITMATCH_HARDWARE_LARGE', None) + log = self.workspace / ('large.log' if index else 'card.log') + self.command(self.swift + ['test', '--skip-build', '--disable-swift-testing', '--no-parallel', '--jobs', '4', + '--package-path', str(self.project / 'Packages/BitMatchEngine'), '--scratch-path', str(self.build), + '--filter', 'PhysicalDriveTransferTests/' + method], log, env) + require_execution(log, method) + self.passed.append(method) + + def cleanup(self): + # An unresponsive test must never race recursive deletion of its files. + if self.child is not None and self.child.poll() is None: + self.warnings.append('Test process is still active; all media fixtures retained.') + return + image_safe = True + if self.image is not None: + try: + self.validate_root(self.image_host) + item = self.image.lstat() + if not stat.S_ISREG(item.st_mode) or (item.st_dev, item.st_ino) != self.image_identity: + raise UnsafeFixture('Image backing file identity changed.') + attachments = self.image_attachments() + if self.attach_started and not self.attach_confirmed and not attachments: + raise UnsafeFixture('Image attachment did not settle; retaining backing data.') + points = [Path(entity['mount-point']) for image in attachments + for entity in image.get('system-entities', []) if entity.get('mount-point')] + if any(point != self.image_mount for point in points): + raise UnsafeFixture('The owned image has unexpected mountpoints.') + if 'largeDestination' in self.roots: + if points != [self.image_mount]: + raise UnsafeFixture('Cannot revalidate the marked virtual root; retaining its image.') + self.validate_root('largeDestination') + if points: + if self.image_mount not in mounted_paths(): + raise UnsafeFixture('Image and mount-table evidence disagree.') + self.command(['/usr/bin/hdiutil', 'detach', '-quiet', str(self.image_mount)], self.workspace / 'image-detach.log', timeout=120) + if self.image_attachments() or any(below(point, self.image_mount) for point in mounted_paths()): + raise UnsafeFixture('The owned image is still attached; retaining backing data.') + except Exception as error: + image_safe = False + self.warnings.append(str(error)) + for key in ('sd', 'ssdA', 'ssdB'): + if key not in self.roots or (key == self.image_host and not image_safe): + continue + try: + self.validate_root(key) + root = self.roots[key] + if any(below(point, root) for point in mounted_paths()): + raise UnsafeFixture('A child mount remains inside a fixture; retaining it.') + parent_fd = os.open(root.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + remove_tree_at(parent_fd, root.name, self.identities[key], self.identities[key][0]) + self.removed.add(key) + finally: + os.close(parent_fd) + except Exception as error: + self.warnings.append(str(error)) + if image_safe and not any(below(point, self.build) for point in mounted_paths()): + shutil.rmtree(self.build, ignore_errors=False) if self.build.exists() else None + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--source-volume', required=True, metavar='/Volumes/CARD') + parser.add_argument('--backup-volume', required=True, action='append', metavar='/Volumes/SSD', + help='Repeat twice. First seeds the large source; second is the default large destination. Three distinct external physical disks are required.') + parser.add_argument('--large', action='store_true', help='Also run the 119.6 GB / eight-file case.') + parser.add_argument('--fast-large-image-on', metavar='/Volumes/SSD', + help='With --large, put a 160 GiB sparse exFAT image on one authorized backup.') + args = parser.parse_args() + if len(args.backup_volume) != 2 or (args.fast_large_image_on and not args.large): + parser.error('Provide exactly two --backup-volume arguments; the image option also requires --large.') + if platform.system() != 'Darwin': + parser.error('Physical validation requires macOS.') + if (not all(function in os.supports_dir_fd for function in (os.open, os.stat, os.unlink, os.rmdir)) + or os.scandir not in os.supports_fd or not shutil.rmtree.avoids_symlink_attacks): + parser.error('Use a Python build with descriptor-relative filesystem APIs, such as Homebrew python3.') + run = None + status = 1 + try: + volumes = dict(zip(('sd', 'ssdA', 'ssdB'), map(physical_volume, [args.source_volume] + args.backup_volume))) + if len({entry['parent'].casefold() for entry in volumes.values()}) != 3: + raise UnsafeFixture('Source and backups must be on three distinct physical ParentWholeDisks.') + image_host = None + if args.fast_large_image_on: + chosen = direct_mount(args.fast_large_image_on) + image_host = next((key for key in ('ssdA', 'ssdB') if volumes[key]['path'] == chosen), None) + if image_host is None: + raise UnsafeFixture('The image host must be one of the two explicitly authorized backups.') + required = {'sd': 4 * 1024**3, 'ssdA': 6 * 1024**3, 'ssdB': 6 * 1024**3} + if args.large: + required['ssdA'] = 119_600_000_000 + 8 * 1024**3 + required[image_host or 'ssdB'] += (160 * 1024**3 if image_host else 119_600_000_000) + for key in {'ssdA', image_host or 'ssdB'}: + if 'fat32' in str(volumes[key]['filesystem']).lower() or str(volumes[key]['filesystem']).lower() == 'msdos': + raise UnsafeFixture('The large case requires support for individual 45 GB files.') + for key, volume in volumes.items(): + if shutil.disk_usage(volume['path']).free < required[key]: + raise UnsafeFixture('Insufficient free space for ' + key + '; no fixture folders created.') + run = Run(volumes, args.large, image_host) + def interrupt(signum, frame): + raise KeyboardInterrupt('Interrupted by signal ' + str(signum)) + signal.signal(signal.SIGINT, interrupt) + signal.signal(signal.SIGTERM, interrupt) + print('Fixtures are generated test data only. Existing footage is never selected or modified.', flush=True) + if args.large: + print('Large case: 119.6 GB; up to 478.4 GB destination I/O. Illustrative 40 MB/s: 3.3 hours; throughput is not measured yet.', flush=True) + if image_host: + print('Large exFAT image shares its selected SSD hardware; ' + + ('also the large source SSD, so this is not an independent backup.' if image_host == 'ssdA' + else 'this adds filesystem coverage, not another physical drive.'), flush=True) + run.execute() + status = 0 + except (Exception, KeyboardInterrupt) as error: + print('Physical validation stopped: ' + str(error), file=sys.stderr) + finally: + if run: + # A second interrupt must not abandon a normal detach attempt. + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + try: + run.cleanup() + except Exception as error: + run.warnings.append(str(error)) + if run.warnings: + status = 1 + summary = {'exitStatus': status, 'largeRequested': args.large, 'imageSharesHostSSD': bool(image_host), + 'imageSharesLargeSourceSSD': image_host == 'ssdA', + 'passedXCTests': run.passed, 'cleanupWarnings': run.warnings, + 'retainedFixtureRoles': sorted(set(run.roots) - run.removed + - ({'largeDestination'} if run.image_host in run.removed else set())), + 'logsRetained': str(run.workspace)} + (run.workspace / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n') + print(json.dumps(summary, indent=2)) + return status + + +if __name__ == '__main__': + sys.exit(main()) From dbf4b68b86306e8ad55c8c280205cfecd726f87a Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 21:27:36 -0400 Subject: [PATCH 08/56] Accept selected removable SD readers in drive validation --- Scripts/test-physical-drives.py | 65 +++++++++++++------ Scripts/test_physical_drive_runner.py | 89 +++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 20 deletions(-) create mode 100644 Scripts/test_physical_drive_runner.py diff --git a/Scripts/test-physical-drives.py b/Scripts/test-physical-drives.py index ab1f73fd..121eaf48 100755 --- a/Scripts/test-physical-drives.py +++ b/Scripts/test-physical-drives.py @@ -47,36 +47,58 @@ def direct_mount(value): return path -def physical_volume(value): - path = direct_mount(value) - info = disk_info(path) - roles = info.get('APFSVolumeRoles', info.get('APFSVolumeRole', [])) - roles = [roles] if isinstance(roles, str) else roles - if (not info.get('Mounted') or info.get('MountPoint') != str(path) - or not info.get('VolumeUUID') or info.get('Internal') is not False - or info.get('ReadOnlyMedia') or info.get('ReadOnlyVolume') - or info.get('Writable') is False or info.get('BootVolume') or info.get('SystemImage') - or info.get('Bootable') is True - or any(str(role).lower() not in ('none', '') for role in roles) - or not os.access(path, os.W_OK | os.X_OK)): - raise UnsafeFixture('The selected mount must be writable, external data media.') +def parent_disks(info): parent = info.get('ParentWholeDisk') container = info.get('APFSContainerReference') if container: containers = plist_command('/usr/sbin/diskutil', 'apfs', 'list', '-plist').get('Containers', []) stores = [store for item in containers if item.get('ContainerReference') == container for store in item.get('PhysicalStores', [])] - if len(stores) != 1: - raise UnsafeFixture('An APFS role must resolve to exactly one physical store.') - parent = disk_info(stores[0]['DeviceIdentifier']).get('ParentWholeDisk') + parents = {disk_info(store['DeviceIdentifier']).get('ParentWholeDisk') for store in stores} + if not parents or None in parents: + raise UnsafeFixture('Cannot establish the APFS physical stores.') + return parents if not parent: raise UnsafeFixture('Cannot establish the physical ParentWholeDisk.') + return {parent} + + +def removable_sd_reader(info): + return (info.get('Internal') is True and info.get('RemovableMedia') is True + and info.get('BusProtocol') == 'Secure Digital' and info.get('OSInternalMedia') is False + and not info.get('SystemImage')) + + +def physical_volume(value): + path = direct_mount(value) + info = disk_info(path) + roles = info.get('APFSVolumeRoles', info.get('APFSVolumeRole', [])) + roles = [roles] if isinstance(roles, str) else roles + reader = removable_sd_reader(info) + if (info.get('Mounted') is False or info.get('MountPoint') != str(path) + or not info.get('VolumeUUID') or (info.get('Internal') is not False and not reader) + or info.get('ReadOnlyMedia') or info.get('ReadOnlyVolume') + or any(info.get(key) is False for key in ('Writable', 'WritableMedia', 'WritableVolume')) + or info.get('BootVolume') or info.get('SystemImage') or info.get('OSInternalMedia') + or any(str(role).lower() not in ('none', '') for role in roles) + or not os.access(path, os.W_OK | os.X_OK)): + raise UnsafeFixture('Use writable external data media or an explicitly selected removable SD card.') + parents = parent_disks(info) + if len(parents) != 1: + raise UnsafeFixture('A selected volume must resolve to exactly one physical store.') + parent = next(iter(parents)) whole = disk_info(parent) - if (not whole.get('Whole') or whole.get('Internal') is not False + if (not whole.get('WholeDisk', whole.get('Whole')) + or (whole.get('Internal') is not False and not (reader and removable_sd_reader(whole))) or str(whole.get('VirtualOrPhysical', '')).lower() != 'physical' or whole.get('DiskImage') or whole.get('BootVolume') or whole.get('SystemImage') - or whole.get('Bootable') is True): + or whole.get('OSInternalMedia') or whole.get('ReadOnlyMedia') + or any(whole.get(key) is False for key in ('Writable', 'WritableMedia'))): raise UnsafeFixture('Virtual, internal, or ambiguous physical media is refused.') + # Bootable is a capability flag, including on some ExFAT data SSDs. + # Reject the actual startup device instead of inferring it from that flag. + if parent in parent_disks(disk_info('/')): + raise UnsafeFixture('The active startup physical disk cannot be used for validation.') return {'path': path, 'uuid': info['VolumeUUID'], 'parent': parent, 'filesystem': info.get('FilesystemType', info.get('FileSystemPersonality', ''))} @@ -159,7 +181,10 @@ def command(self, args, log, env=None, timeout=None): finally: if self.child.poll() is None: for sig in (signal.SIGINT, signal.SIGTERM): - os.killpg(self.child.pid, sig) + try: + os.killpg(self.child.pid, sig) + except ProcessLookupError: + pass # The child exited between poll and signal. try: self.child.wait(timeout=10) break @@ -339,7 +364,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('--source-volume', required=True, metavar='/Volumes/CARD') parser.add_argument('--backup-volume', required=True, action='append', metavar='/Volumes/SSD', - help='Repeat twice. First seeds the large source; second is the default large destination. Three distinct external physical disks are required.') + help='Repeat twice. First seeds the large source; second is the default large destination. Three distinct physical disks are required; an internal removable SD reader is allowed.') parser.add_argument('--large', action='store_true', help='Also run the 119.6 GB / eight-file case.') parser.add_argument('--fast-large-image-on', metavar='/Volumes/SSD', help='With --large, put a 160 GiB sparse exFAT image on one authorized backup.') diff --git a/Scripts/test_physical_drive_runner.py b/Scripts/test_physical_drive_runner.py new file mode 100644 index 00000000..1cf80b55 --- /dev/null +++ b/Scripts/test_physical_drive_runner.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Mock-only regressions: no drive queries, subprocesses, or media writes.""" +import importlib.util +import io +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +sys.dont_write_bytecode = True +spec = importlib.util.spec_from_file_location('physical_runner', Path(__file__).with_name('test-physical-drives.py')) +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) + + +class PhysicalDrivePreflightTests(unittest.TestCase): + def setUp(self): + self.volume = {'MountPoint': '/Volumes/Fixture', 'VolumeUUID': 'FIXTURE', 'Internal': False, + 'ParentWholeDisk': 'disk8', 'FilesystemType': 'exfat', 'OSInternalMedia': False, + 'Writable': True, 'WritableMedia': True, 'WritableVolume': True} + self.whole = {'WholeDisk': True, 'Internal': False, 'VirtualOrPhysical': 'Physical', + 'OSInternalMedia': False, 'WritableMedia': True} + self.boot = {'ParentWholeDisk': 'disk0'} + self.path = patch.object(runner, 'direct_mount', return_value=Path('/Volumes/Fixture')) + self.access = patch.object(runner.os, 'access', return_value=True) + self.path.start(); self.access.start() + self.addCleanup(self.path.stop); self.addCleanup(self.access.stop) + + def query(self, volume=None, whole=None, boot=None): + with patch.object(runner, 'disk_info', side_effect=[volume or self.volume, whole or self.whole, boot or self.boot]): + return runner.physical_volume('explicit selection') + + def test_current_diskutil_keys_without_mounted_or_whole_are_supported(self): + self.assertEqual(self.query()['parent'], 'disk8') + + def test_boot_capability_does_not_misidentify_a_data_ssd_as_the_startup_disk(self): + self.assertEqual(self.query(dict(self.volume, Bootable=True))['parent'], 'disk8') + + def test_explicit_removable_secure_digital_media_in_an_internal_reader_is_allowed(self): + self.assertEqual(self.query( + dict(self.volume, Internal=True, RemovableMedia=True, BusProtocol='Secure Digital'), + dict(self.whole, Internal=True, RemovableMedia=True, BusProtocol='Secure Digital'))['parent'], 'disk8') + + def test_internal_fixed_or_non_sd_media_is_refused(self): + for flags in [dict(Internal=True), dict(Internal=True, RemovableMedia=False, BusProtocol='Secure Digital'), + dict(Internal=True, RemovableMedia=True, BusProtocol='PCI-Express'), + dict(Internal=True, RemovableMedia=True, BusProtocol='USB')]: + with self.subTest(flags=flags), self.assertRaises(runner.UnsafeFixture): + self.query(dict(self.volume, **flags)) + + def test_partition_and_whole_device_must_both_prove_internal_reader_exception(self): + volume = dict(self.volume, Internal=True, RemovableMedia=True, BusProtocol='Secure Digital') + with self.assertRaises(runner.UnsafeFixture): + self.query(volume, dict(self.whole, Internal=True, RemovableMedia=False, BusProtocol='Secure Digital')) + + def test_read_only_system_virtual_and_active_boot_devices_are_refused(self): + for flags in [dict(Writable=False), dict(WritableMedia=False), dict(WritableVolume=False), + dict(ReadOnlyMedia=True), dict(SystemImage=True), dict(OSInternalMedia=True), + dict(BootVolume=True), dict(APFSVolumeRoles=['System'])]: + with self.subTest(flags=flags), self.assertRaises(runner.UnsafeFixture): + self.query(dict(self.volume, **flags)) + for flags in [dict(VirtualOrPhysical='Virtual'), dict(SystemImage=True), + dict(OSInternalMedia=True), dict(Internal=True), dict(WritableMedia=False)]: + with self.subTest(flags=flags), self.assertRaises(runner.UnsafeFixture): + self.query(whole=dict(self.whole, **flags)) + with self.assertRaises(runner.UnsafeFixture): + self.query(boot={'ParentWholeDisk': 'disk8'}) + + def test_interruption_and_timeout_are_propagated_after_graceful_child_exit(self): + for exception in [KeyboardInterrupt(), subprocess.TimeoutExpired('mock', 1)]: + with self.subTest(exception=type(exception).__name__), tempfile.TemporaryDirectory() as temporary: + child = MagicMock() + child.pid = 123456 + child.wait.side_effect = [exception, 0] + child.poll.side_effect = [None, 0] + run = runner.Run.__new__(runner.Run) + run.workspace = Path(temporary) + run.child = None + with patch.object(runner.subprocess, 'Popen', return_value=child), \ + patch.object(runner.os, 'killpg', side_effect=ProcessLookupError()), patch('sys.stdout', io.StringIO()): + with self.assertRaises(type(exception)): + run.command(['never executed'], run.workspace / 'mock.log') + self.assertIsNone(run.child) + + +if __name__ == '__main__': + unittest.main() From 3e03d6238a029180d11ba51721ae7c8ba1399bd2 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 21:34:44 -0400 Subject: [PATCH 09/56] Generate mounted workflow clip fixtures off MainActor --- BitMatchTests/MountedWorkflowEvidenceMatrixTests.swift | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/BitMatchTests/MountedWorkflowEvidenceMatrixTests.swift b/BitMatchTests/MountedWorkflowEvidenceMatrixTests.swift index 96697de7..392fb834 100644 --- a/BitMatchTests/MountedWorkflowEvidenceMatrixTests.swift +++ b/BitMatchTests/MountedWorkflowEvidenceMatrixTests.swift @@ -291,6 +291,14 @@ final class MountedWorkflowEvidenceMatrixTests: XCTestCase { } private func makeMovie(_ url: URL) async throws { + // Synthetic codec setup performs synchronous Security/encoder work. + // Keep fixture preparation off the actor used to drive the app. + try await Task.detached { + try await Self.writeMovie(url) + }.value + } + + nonisolated private static func writeMovie(_ url: URL) async throws { let writer = try AVAssetWriter(outputURL: url, fileType: .mov) let input = AVAssetWriterInput(mediaType: .video, outputSettings: [AVVideoCodecKey: AVVideoCodecType.h264, AVVideoWidthKey: 64, AVVideoHeightKey: 64]) let adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input, sourcePixelBufferAttributes: [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32ARGB, kCVPixelBufferWidthKey as String: 64, kCVPixelBufferHeightKey as String: 64]) From ef7cd051ed59ae5d12d1792242562c716d0be9c8 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 21:55:44 -0400 Subject: [PATCH 10/56] Add portable transfer settings with an import preview --- BitMatch-iPad/Views/ModularContentView.swift | 2 + .../PortableTransferSettingsIOSTests.swift | 141 ++++++++++ BitMatch/Views/PreferencesWindow.swift | 2 + ...ortableTransferSettingsSnapshotTests.swift | 68 +++++ .../PortableTransferSettingsTests.swift | 251 ++++++++++++++++++ CHANGELOG.md | 1 + .../Models/PortableTransferSettings.swift | 160 +++++++++++ .../Services/TransferSettingsImport.swift | 105 ++++++++ Shared/Views/TransferSettingsSection.swift | 144 ++++++++++ docs/GUIDE.md | 6 + docs/TODO.md | 2 +- 11 files changed, 881 insertions(+), 1 deletion(-) create mode 100644 BitMatch-iPadTests/PortableTransferSettingsIOSTests.swift create mode 100644 BitMatchTests/PortableTransferSettingsSnapshotTests.swift create mode 100644 BitMatchTests/PortableTransferSettingsTests.swift create mode 100644 Shared/Core/Models/PortableTransferSettings.swift create mode 100644 Shared/Core/Services/TransferSettingsImport.swift create mode 100644 Shared/Views/TransferSettingsSection.swift diff --git a/BitMatch-iPad/Views/ModularContentView.swift b/BitMatch-iPad/Views/ModularContentView.swift index 29dc7d55..10d6e7b0 100644 --- a/BitMatch-iPad/Views/ModularContentView.swift +++ b/BitMatch-iPad/Views/ModularContentView.swift @@ -311,6 +311,8 @@ struct SettingsSheetView: View { .disabled(coordinator.isOperationInProgress || coordinator.isReplayingQueuedTransfer) } + TransferSettingsSection(coordinator: coordinator) + Section("Diagnostics") { LocalCrashEvidenceSettingView(settings: generalSettings) } diff --git a/BitMatch-iPadTests/PortableTransferSettingsIOSTests.swift b/BitMatch-iPadTests/PortableTransferSettingsIOSTests.swift new file mode 100644 index 00000000..dc3c7979 --- /dev/null +++ b/BitMatch-iPadTests/PortableTransferSettingsIOSTests.swift @@ -0,0 +1,141 @@ +import Foundation +import Testing +import XCTest +import SwiftUI +import UIKit +import Vision +@testable import BitMatch_iPad +import BitMatchEngine + +@MainActor +struct PortableTransferSettingsIOSTests { + @Test func mobileSettingsRoundTripPreviewCancelApplyAndPersistence() throws { + let name = "BitMatch-iPadTests.PortableSettings.\(UUID())" + let defaults = try #require(UserDefaults(suiteName: name)) + let root = FileManager.default.temporaryDirectory.appendingPathComponent("mobile-settings-\(UUID())") + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + defer { defaults.removePersistentDomain(forName: name); try? FileManager.default.removeItem(at: root) } + func coordinator() -> SharedAppCoordinator { + SharedAppCoordinator(platformManager: IOSPlatformManager.shared, + transferJournal: LocalTransferJournal(fileURL: root.appendingPathComponent("journal-\(UUID()).json")), + projectStore: UserDefaultsPhotographerJobStore(defaults: defaults), defaults: defaults) + } + let app = coordinator() + app.reportSettings.operatorName = "Private operator" + app.reportSettings.notes = "Private notes" + app.cameraLabels.settings.label = "Private camera" + let original = app.portableTransferSettings + let proposed = mobileSettingsProposal() + let model = TransferSettingsImportModel() + try model.preview(proposed.encoded(), coordinator: app) + #expect(app.portableTransferSettings == original) + model.cancel() + #expect(app.portableTransferSettings == original) + #expect(model.review == nil) + try model.preview(proposed.encoded(), coordinator: app) + app.isOperationInProgress = true + #expect(throws: TransferSettingsImportError.busy) { try model.apply(to: app) } + app.isOperationInProgress = false + try model.apply(to: app) + #expect(app.portableTransferSettings == proposed) + #expect(app.reportSettings.operatorName == "Private operator") + #expect(app.reportSettings.notes == "Private notes") + #expect(app.cameraLabels.settings.label == "Private camera") + #expect(coordinator().portableTransferSettings == proposed) + let text = String(decoding: try proposed.encoded(), as: UTF8.self) + #expect(!text.contains("Private")) + } + + @Test func mobileDecoderRejectsUnsupportedAndOversizedFiles() throws { + let valid = try mobileSettingsProposal().encoded() + var object = try #require(JSONSerialization.jsonObject(with: valid) as? [String: Any]) + object["schemaVersion"] = 2 + #expect(throws: TransferSettingsImportError.unsupportedVersion) { + _ = try PortableTransferSettings.decode(JSONSerialization.data(withJSONObject: object)) + } + #expect(throws: TransferSettingsImportError.tooLarge) { + _ = try PortableTransferSettings.decode(Data(repeating: 32, count: PortableTransferSettings.maximumBytes + 1)) + } + #expect(throws: TransferSettingsImportError.invalidFile) { _ = try PortableTransferSettings.decode(Data("no".utf8)) } + } +} + +private func mobileSettingsProposal() -> PortableTransferSettings { + .init(verificationMode: .quick, generateASCMHL: false, ignoreAppleDouble: false, + reports: .init(makeReport: false, includeThumbnails: true), + naming: .init(position: .suffix, separator: .dash, autoNumber: false, groupByCamera: true)) +} + +@MainActor +final class PortableTransferSettingsSnapshotTests: XCTestCase { + func testRenderNarrowPhoneAndPadImportPreview() throws { + let environment = ProcessInfo.processInfo.environment + guard environment["BITMATCH_CAPTURE_WORKFLOW_SNAPSHOTS"] == "1" + || environment["TEST_RUNNER_BITMATCH_CAPTURE_WORKFLOW_SNAPSHOTS"] == "1" else { + throw XCTSkip("Enable workflow snapshots for settings preview captures") + } + let current = PortableTransferSettings(verificationMode: .standard, generateASCMHL: true, ignoreAppleDouble: true, + reports: .init(makeReport: true, includeThumbnails: false), + naming: .init(position: .prefix, separator: .underscore, autoNumber: true, groupByCamera: false)) + let review = TransferSettingsReview(current: current, proposed: mobileSettingsProposal()) + XCTAssertEqual(review.changes.count, 9) + for (name, size) in [("smallphone-transfer-settings-preview", CGSize(width: 320, height: 568)), + ("phone-transfer-settings-preview", CGSize(width: 390, height: 844)), + ("narrowpad-transfer-settings-preview", CGSize(width: 500, height: 700))] { + let host = UIHostingController(rootView: TransferSettingsReviewView(review: review, apply: {}, cancel: {}) + .preferredColorScheme(.dark) + .frame(width: size.width, height: size.height) + .transaction { $0.disablesAnimations = true }) + let window = UIWindow(frame: CGRect(origin: .zero, size: size)) + window.rootViewController = host + window.makeKeyAndVisible() + defer { + window.isHidden = true + // Let UIKit finish the window's disappearance before removing + // the hosting controller; the fixture must not strand a + // NavigationStack appearance transition between captures. + RunLoop.main.run(until: Date(timeIntervalSinceNow: 0.1)) + window.rootViewController = nil + } + host.view.frame = CGRect(origin: .zero, size: size) + host.view.setNeedsLayout() + host.view.layoutIfNeeded() + RunLoop.main.run(until: Date(timeIntervalSinceNow: 0.3)) + host.view.layoutIfNeeded() + try capture(host.view, size: size, name: name, expected: ["verification", "apply", "cancel"]) + // A short screen must keep the disclosure and actions reachable; + // a tall render alone would conceal missing scrolling behavior. + let scroll = try XCTUnwrap(findScrollView(in: host.view)) + // Form measures its later rows lazily. Re-settle after scrolling + // rather than mistaking its initial estimated height for the end. + for _ in 0..<4 { + scroll.setContentOffset(CGPoint(x: 0, + y: max(-scroll.adjustedContentInset.top, + scroll.contentSize.height - scroll.bounds.height + scroll.adjustedContentInset.bottom)), animated: false) + RunLoop.main.run(until: Date(timeIntervalSinceNow: 0.1)) + host.view.layoutIfNeeded() + } + try capture(host.view, size: size, name: name + "-scrolled", expected: ["quick checks file sizes", "future transfers", "apply", "cancel"]) + } + } + + private func findScrollView(in view: UIView) -> UIScrollView? { + if let scroll = view as? UIScrollView, scroll.contentSize.height > 0 { return scroll } + return view.subviews.lazy.compactMap { self.findScrollView(in: $0) }.first + } + + private func capture(_ view: UIView, size: CGSize, name: String, expected: [String]) throws { + let image = UIGraphicsImageRenderer(size: size).image { _ in + view.drawHierarchy(in: view.bounds, afterScreenUpdates: true) + } + let request = VNRecognizeTextRequest() + request.recognitionLevel = .accurate + try VNImageRequestHandler(cgImage: try XCTUnwrap(image.cgImage)).perform([request]) + let text = (request.results ?? []).compactMap { $0.topCandidates(1).first?.string }.joined(separator: " ").lowercased() + for value in expected { XCTAssertTrue(text.contains(value), "\(name) missing \(value): \(text)") } + let attachment = XCTAttachment(data: try XCTUnwrap(image.pngData()), uniformTypeIdentifier: "public.png") + attachment.name = name + attachment.lifetime = .keepAlways + add(attachment) + } +} diff --git a/BitMatch/Views/PreferencesWindow.swift b/BitMatch/Views/PreferencesWindow.swift index 42667b45..867f010d 100644 --- a/BitMatch/Views/PreferencesWindow.swift +++ b/BitMatch/Views/PreferencesWindow.swift @@ -152,6 +152,8 @@ struct PreferencesWindow: View { LocalCrashEvidenceSettingView(settings: generalSettings) } + TransferSettingsSection(coordinator: coordinator) + Section("Updates") { Toggle( "Check for updates automatically", diff --git a/BitMatchTests/PortableTransferSettingsSnapshotTests.swift b/BitMatchTests/PortableTransferSettingsSnapshotTests.swift new file mode 100644 index 00000000..38a0bfaa --- /dev/null +++ b/BitMatchTests/PortableTransferSettingsSnapshotTests.swift @@ -0,0 +1,68 @@ +import XCTest +import SwiftUI +import AppKit +import Vision +@testable import BitMatch +import BitMatchEngine + +@MainActor +final class PortableTransferSettingsSnapshotTests: XCTestCase { + func testCaptureMacTransferSettings() throws { + let environment = ProcessInfo.processInfo.environment + guard environment["BITMATCH_CAPTURE_WORKFLOW_SNAPSHOTS"] == "1" + || environment["TEST_RUNNER_BITMATCH_CAPTURE_WORKFLOW_SNAPSHOTS"] == "1" else { + throw XCTSkip("Enable workflow snapshots for settings captures") + } + let folders = try CoordinatorFolders() + defer { folders.cleanup() } + let app = SharedAppCoordinator(platformManager: RecordingPlatformManager(fileOperations: RecordingFileOperations()), + transferJournal: LocalTransferJournal(fileURL: folders.journalURL), + projectStore: InMemoryPhotographerJobStore(), defaults: folders.defaults) + let companions = MacAppEnvironment.makeForTesting(coordinator: app) + let updater = UpdaterController(coordinator: app) + let previousPane = UserDefaults.standard.object(forKey: "BitMatchSelectedSettingsPane") + UserDefaults.standard.set("General", forKey: "BitMatchSelectedSettingsPane") + defer { + if let previousPane { UserDefaults.standard.set(previousPane, forKey: "BitMatchSelectedSettingsPane") } + else { UserDefaults.standard.removeObject(forKey: "BitMatchSelectedSettingsPane") } + } + try capture(PreferencesWindow(coordinator: app, cameraAutoSource: companions.cameraAutoSource, + remoteBackups: companions.remoteBackups, updater: updater), size: CGSize(width: 680, height: 720), + name: "mac-transfer-settings", expected: ["transfer settings", "import settings", "export settings"]) + let current = app.portableTransferSettings + let proposed = PortableTransferSettings(verificationMode: .quick, generateASCMHL: !current.generateASCMHL, + ignoreAppleDouble: !current.ignoreAppleDouble, + reports: .init(makeReport: !current.reports.makeReport, includeThumbnails: !current.reports.includeThumbnails), + naming: .init(position: .suffix, separator: .dash, autoNumber: false, groupByCamera: true)) + try capture(TransferSettingsReviewView(review: .init(current: current, proposed: proposed), apply: {}, cancel: {}), + size: CGSize(width: 480, height: 600), name: "mac-transfer-settings-preview", expected: ["verification", "quick"]) + } + + private func capture(_ view: V, size: CGSize, name: String, expected: [String]) throws { + let host = NSHostingView(rootView: view.frame(width: size.width, height: size.height) + .preferredColorScheme(.dark).transaction { $0.disablesAnimations = true }) + let window = NSWindow(contentRect: CGRect(origin: .zero, size: size), styleMask: [.titled], backing: .buffered, defer: true) + window.isReleasedWhenClosed = false + window.appearance = NSAppearance(named: .darkAqua) + window.contentView = host + window.orderFront(nil) + defer { window.orderOut(nil); window.close() } + host.frame = CGRect(origin: .zero, size: size) + RunLoop.main.run(until: Date().addingTimeInterval(0.3)) + window.layoutIfNeeded() + host.layoutSubtreeIfNeeded() + host.displayIfNeeded() + let bitmap = try XCTUnwrap(host.bitmapImageRepForCachingDisplay(in: host.bounds)) + host.cacheDisplay(in: host.bounds, to: bitmap) + let data = try XCTUnwrap(bitmap.representation(using: .png, properties: [:])) + let request = VNRecognizeTextRequest() + request.recognitionLevel = .accurate + try VNImageRequestHandler(cgImage: try XCTUnwrap(bitmap.cgImage)).perform([request]) + let text = (request.results ?? []).compactMap { $0.topCandidates(1).first?.string }.joined(separator: " ").lowercased() + for value in expected { XCTAssertTrue(text.contains(value), "\(name) missing \(value): \(text)") } + let attachment = XCTAttachment(data: data, uniformTypeIdentifier: "public.png") + attachment.name = name + attachment.lifetime = .keepAlways + add(attachment) + } +} diff --git a/BitMatchTests/PortableTransferSettingsTests.swift b/BitMatchTests/PortableTransferSettingsTests.swift new file mode 100644 index 00000000..9509b39f --- /dev/null +++ b/BitMatchTests/PortableTransferSettingsTests.swift @@ -0,0 +1,251 @@ +import Foundation +import Darwin +import Testing +@testable import BitMatch +import BitMatchEngine + +@MainActor +@Suite(.serialized) +struct PortableTransferSettingsTests { + private final class Fixture { + let name = "BitMatchTests.PortableSettings.\(UUID())" + let defaults: UserDefaults + let folders: CoordinatorFolders + let coordinator: SharedAppCoordinator + @MainActor init() throws { + defaults = try #require(UserDefaults(suiteName: name)) + folders = try CoordinatorFolders() + coordinator = SharedAppCoordinator(platformManager: RecordingPlatformManager(fileOperations: RecordingFileOperations()), + transferJournal: LocalTransferJournal(fileURL: folders.journalURL), + projectStore: InMemoryPhotographerJobStore(), defaults: defaults) + } + deinit { defaults.removePersistentDomain(forName: name); folders.cleanup() } + } + + private func changed(_ current: PortableTransferSettings) -> PortableTransferSettings { + .init(verificationMode: .quick, generateASCMHL: !current.generateASCMHL, + ignoreAppleDouble: !current.ignoreAppleDouble, + reports: .init(makeReport: !current.reports.makeReport, includeThumbnails: !current.reports.includeThumbnails), + naming: .init(position: .suffix, separator: .dash, autoNumber: !current.naming.autoNumber, + groupByCamera: !current.naming.groupByCamera)) + } + + @Test func roundTripHasOnlyTheApprovedFieldsAndNoPrivateValues() throws { + let fixture = try Fixture() + let app = fixture.coordinator + app.reportSettings.clientName = "PRIVATE_CLIENT" + app.reportSettings.projectName = "PRIVATE_PROJECT" + app.reportSettings.production = "PRIVATE_PRODUCTION" + app.reportSettings.company = "PRIVATE_COMPANY" + app.reportSettings.operatorName = "PRIVATE_OPERATOR" + app.reportSettings.notes = "PRIVATE_NOTES" + app.cameraLabels.settings.label = "PRIVATE_CAMERA" + app.cameraLabels.settings.destinationPathComponents = ["PRIVATE_ROUTE"] + fixture.defaults.set("PRIVATE_TOKEN", forKey: "unrelatedCredential") + app.sourceURL = fixture.folders.source + app.destinationURLs = [fixture.folders.primary] + let original = app.portableTransferSettings + let data = try original.encoded() + #expect(try PortableTransferSettings.decode(data) == original) + let text = String(decoding: data, as: UTF8.self) + #expect(!text.contains("PRIVATE_")) + #expect(!text.contains(fixture.folders.root.path)) + let object = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) + #expect(Set(object.keys) == ["format", "schemaVersion", "verificationMode", "generateASCMHL", "ignoreAppleDouble", "reports", "naming"]) + let report = try #require(object["reports"] as? [String: Any]) + #expect(Set(report.keys) == ["makeReport", "includeThumbnails"]) + let naming = try #require(object["naming"] as? [String: Any]) + #expect(Set(naming.keys) == ["position", "separator", "autoNumber", "groupByCamera"]) + #expect(data.count < PortableTransferSettings.maximumBytes) + } + + @Test(arguments: ["malformed", "format", "future", "version-type", "missing", "mode", "bool", "unknown", "private-nested", "private-naming", "naming", "null"]) + func invalidFilesNeverChangeSettings(kind: String) throws { + let fixture = try Fixture() + let original = fixture.coordinator.portableTransferSettings + var object = try #require(JSONSerialization.jsonObject(with: changed(original).encoded()) as? [String: Any]) + switch kind { + case "format": object["format"] = "Another settings file" + case "future": object["schemaVersion"] = 999 + case "version-type": object["schemaVersion"] = "1" + case "missing": object.removeValue(forKey: "ignoreAppleDouble") + case "mode": object["verificationMode"] = "SUPER FAST" + case "bool": object["generateASCMHL"] = 1 + case "unknown": object["destinations"] = ["/Volumes/private"] + case "private-nested": object["reports"] = ["makeReport": true, "includeThumbnails": false, "notes": "private"] + case "private-naming": + var naming = try #require(object["naming"] as? [String: Any]) + naming["label"] = "private camera" + object["naming"] = naming + case "naming": object["naming"] = ["position": "Sideways", "separator": "_", "autoNumber": true, "groupByCamera": false] + case "null": object["reports"] = NSNull() + default: break + } + let data = kind == "malformed" ? Data("{\"broken\":".utf8) : try JSONSerialization.data(withJSONObject: object) + let model = TransferSettingsImportModel() + #expect(throws: (any Error).self) { try model.preview(data, coordinator: fixture.coordinator) } + #expect(model.review == nil) + #expect(fixture.coordinator.portableTransferSettings == original) + #expect(ReportPrefsStore(defaults: fixture.defaults).load().makeReport == original.reports.makeReport) + } + + @Test func oversizedAndSpecialFilesAreRejectedBeforeUnboundedReads() throws { + let fixture = try Fixture() + let file = fixture.folders.root.appendingPathComponent("settings.json") + #expect(throws: TransferSettingsImportError.invalidFile) { + _ = try TransferSettingsFileReader.read(URL(string: "https://example.com/settings.json")!) + } + #expect(throws: TransferSettingsImportError.tooLarge) { + _ = try PortableTransferSettings.decode(Data(repeating: 32, count: PortableTransferSettings.maximumBytes + 1)) + } + try Data().write(to: file) + let handle = try FileHandle(forWritingTo: file) + try handle.truncate(atOffset: 2 * 1_024 * 1_024 * 1_024) + try handle.close() + #expect(throws: TransferSettingsImportError.tooLarge) { _ = try TransferSettingsFileReader.read(file) } + let pipe = fixture.folders.root.appendingPathComponent("pipe.json") + #expect(mkfifo(pipe.path, 0o600) == 0) + #expect(throws: TransferSettingsImportError.invalidFile) { _ = try TransferSettingsFileReader.read(pipe) } + let link = fixture.folders.root.appendingPathComponent("link.json") + try FileManager.default.createSymbolicLink(at: link, withDestinationURL: file) + #expect(throws: (any Error).self) { _ = try TransferSettingsFileReader.read(link) } + } + + @Test func boundedReaderAcceptsAValidFileAtItsSizeLimit() throws { + let fixture = try Fixture() + let proposal = changed(fixture.coordinator.portableTransferSettings) + var data = try proposal.encoded() + data.append(Data(repeating: 32, count: PortableTransferSettings.maximumBytes - data.count)) + let file = fixture.folders.root.appendingPathComponent("settings.json") + try data.write(to: file) + let loaded = try TransferSettingsFileReader.read(file) + #expect(loaded.count == PortableTransferSettings.maximumBytes) + #expect(try PortableTransferSettings.decode(loaded) == proposal) + #expect(try Data(contentsOf: file) == data) + } + + @Test(arguments: VerificationMode.allCases, CameraLabelSettings.Separator.allCases) + func allModesAndSeparatorsRoundTrip(mode: VerificationMode, separator: CameraLabelSettings.Separator) throws { + let settings = PortableTransferSettings(verificationMode: mode, generateASCMHL: false, ignoreAppleDouble: false, + reports: .init(makeReport: true, includeThumbnails: false), + naming: .init(position: .suffix, separator: separator, autoNumber: true, groupByCamera: true)) + #expect(try PortableTransferSettings.decode(settings.encoded()) == settings) + } + + @Test func previewAndCancelDoNotApplyAnything() throws { + let fixture = try Fixture() + let original = fixture.coordinator.portableTransferSettings + let model = TransferSettingsImportModel() + try model.preview(changed(original).encoded(), coordinator: fixture.coordinator) + #expect(model.review?.changes.count == 9) + #expect(fixture.coordinator.portableTransferSettings == original) + model.cancel() + #expect(model.review == nil) + #expect(fixture.coordinator.portableTransferSettings == original) + #expect(throws: TransferSettingsImportError.invalidFile) { try model.apply(to: fixture.coordinator) } + } + + @Test func applyPersistsOnlyApprovedFieldsAndRetainsPrivateSettings() throws { + let fixture = try Fixture() + let app = fixture.coordinator + app.reportSettings.clientName = "Keep client" + app.reportSettings.operatorName = "Keep operator" + app.reportSettings.notes = "Keep notes" + app.reportSettings.enableAutoCameraDetection = false + app.cameraLabels.settings.label = "Keep camera" + app.cameraLabels.settings.destinationPathComponents = ["Keep", "route"] + let destinations = app.destinationURLs + let general = app.generalSettings.collectLocalCrashEvidence + let original = app.portableTransferSettings + let proposed = changed(original) + let model = TransferSettingsImportModel() + try model.preview(proposed.encoded(), coordinator: app) + try model.apply(to: app) + #expect(model.review == nil) + #expect(app.portableTransferSettings == proposed) + #expect(app.reportSettings.clientName == "Keep client") + #expect(app.reportSettings.operatorName == "Keep operator") + #expect(app.reportSettings.notes == "Keep notes") + #expect(!app.reportSettings.enableAutoCameraDetection) + #expect(app.cameraLabels.settings.label == "Keep camera") + #expect(app.cameraLabels.settings.destinationPathComponents == ["Keep", "route"]) + #expect(app.destinationURLs == destinations) + #expect(app.sourceURL == nil) + #expect(app.generalSettings.collectLocalCrashEvidence == general) + let saved = SharedAppCoordinator(platformManager: RecordingPlatformManager(fileOperations: RecordingFileOperations()), + transferJournal: LocalTransferJournal(fileURL: fixture.folders.root.appendingPathComponent("relaunch.json")), + projectStore: InMemoryPhotographerJobStore(), defaults: fixture.defaults) + #expect(saved.portableTransferSettings == proposed) + #expect(saved.reportSettings.notes == "Keep notes") + #expect(saved.cameraLabels.settings.label == "Keep camera") + } + + @Test func applyRechecksActiveWorkAndQueueWithoutChangingRecordedCards() throws { + let fixture = try Fixture() + let app = fixture.coordinator + let original = app.portableTransferSettings + let model = TransferSettingsImportModel() + try model.preview(changed(original).encoded(), coordinator: app) + app.isOperationInProgress = true + #expect(throws: TransferSettingsImportError.busy) { try model.apply(to: app) } + #expect(app.portableTransferSettings == original) + app.isOperationInProgress = false + let id = try app.transferJournal.enqueue(sourceURL: fixture.folders.source, destinationURLs: [fixture.folders.primary], + verificationMode: .thorough, cameraSettings: CameraLabelSettings(), reportSettings: ReportPrefs(), generateASCMHL: true) + let encoder = JSONEncoder() + encoder.outputFormatting = .sortedKeys + let before = try encoder.encode(app.transferJournal.records.first { $0.id == id }) + // A waiting journal card is a queue record even before a session starts. + #expect(throws: TransferSettingsImportError.busy) { try model.apply(to: app) } + #expect(app.portableTransferSettings == original) + #expect(try encoder.encode(app.transferJournal.records.first { $0.id == id }) == before) + } + + @Test func changedPreferencesRequireANewPreview() throws { + let fixture = try Fixture() + let app = fixture.coordinator + let model = TransferSettingsImportModel() + try model.preview(changed(app.portableTransferSettings).encoded(), coordinator: app) + app.reportSettings.includeThumbnails.toggle() + let current = app.portableTransferSettings + #expect(throws: TransferSettingsImportError.settingsChanged) { try model.apply(to: app) } + #expect(app.portableTransferSettings == current) + } + + @Test func preparedProjectCardBlocksApplyAndStaysUntouched() throws { + let fixture = try Fixture() + let app = fixture.coordinator + let model = TransferSettingsImportModel() + try model.preview(changed(app.portableTransferSettings).encoded(), coordinator: app) + let jobs = app.photographerJobViewModel + jobs.createWeddingJob(clientName: "Keep client", jobName: "Keep job", eventDate: Date(timeIntervalSince1970: 0)) + try jobs.prepareCard(photographerName: "Keep operator", cameraName: "Keep camera", + analysis: CardAnalysis(fingerprint: "owned-fixture", fileCount: 1, totalBytes: 4, + companionGroups: [], sourcePaths: [fixture.folders.source.appendingPathComponent("A.ARW").path])) + #expect(jobs.hasPreparedIngestAwaitingStart) + let encoder = JSONEncoder() + encoder.outputFormatting = .sortedKeys + let before = try encoder.encode(jobs.activeJob) + let settings = app.portableTransferSettings + #expect(throws: TransferSettingsImportError.busy) { try model.apply(to: app) } + #expect(app.portableTransferSettings == settings) + #expect(try encoder.encode(jobs.activeJob) == before) + } + + @Test func selectedSourceIsRetainedAndAppleDoubleReviewStartsAgain() throws { + let fixture = try Fixture() + let app = fixture.coordinator + app.appleDoubleSelection.enabled = false + app.sourceURL = fixture.folders.source + let original = app.portableTransferSettings + let proposed = changed(original) + let model = TransferSettingsImportModel() + try model.preview(proposed.encoded(), coordinator: app) + try model.apply(to: app) + #expect(app.sourceURL == fixture.folders.source) + #expect(app.appleDoubleSelection.enabled) + #expect(app.appleDoubleSelection.paths == nil) + #expect(app.appleDoubleSelection.isReviewing) + #expect(!app.canStartOperation) + } +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 872f7d8e..ba4933cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +- Move reusable transfer settings between Mac, iPad and iPhone from Settings. Imports show the changes before Apply and leave backup locations, camera labels, private report details and saved transfers alone. - Older History entries without a saved selected-file plan now stay visible but do not claim the whole card is safe to erase, even if their saved rows say verified. They also no longer trigger the “Backed up before…” hint. Quick entries without that plan show Needs attention instead of a clean size-only copy. History exports keep the original message and add the saved safety verdict. - If a new transfer cannot save its selected-file plan, BitMatch stops before creating a completion report or marking a project card safe. The final record keeps the source unsafe. Finish, queue, menu bar, Dock and notifications follow the saved verdict. A later History-save failure also shows an interruption; any separately saved report or project evidence remains available for review. - Mac: The menu bar shows the current transfer and its destinations, or the most recent result when idle. The idle icon is a small copy-and-check mark. Closing the main window during a transfer hides it while the work continues; Open BitMatch brings the same window back. Quit still asks before stopping active work. diff --git a/Shared/Core/Models/PortableTransferSettings.swift b/Shared/Core/Models/PortableTransferSettings.swift new file mode 100644 index 00000000..abac6ff3 --- /dev/null +++ b/Shared/Core/Models/PortableTransferSettings.swift @@ -0,0 +1,160 @@ +import Foundation +import BitMatchEngine + +/// A whitelist of reusable choices, separate from stored reports and jobs. +/// Names, paths, permissions, identities and transfer evidence never enter it. +struct PortableTransferSettings: Codable, Equatable, Sendable { + static let format = "BitMatch transfer settings" + static let schemaVersion = 1 + static let maximumBytes = 16 * 1_024 + + let verificationMode: VerificationMode + let generateASCMHL: Bool + let ignoreAppleDouble: Bool + let reports: Reports + let naming: Naming + + struct Reports: Codable, Equatable, Sendable { + let makeReport: Bool + let includeThumbnails: Bool + init(makeReport: Bool, includeThumbnails: Bool) { + self.makeReport = makeReport + self.includeThumbnails = includeThumbnails + } + private enum CodingKeys: String, CodingKey, CaseIterable { case makeReport, includeThumbnails } + init(from decoder: Decoder) throws { + try rejectUnknownSettingsKeys(decoder, allowed: CodingKeys.allCases.map(\.rawValue)) + let values = try decoder.container(keyedBy: CodingKeys.self) + makeReport = try values.decode(Bool.self, forKey: .makeReport) + includeThumbnails = try values.decode(Bool.self, forKey: .includeThumbnails) + } + } + + struct Naming: Codable, Equatable, Sendable { + let position: CameraLabelSettings.LabelPosition + let separator: CameraLabelSettings.Separator + let autoNumber: Bool + let groupByCamera: Bool + init(position: CameraLabelSettings.LabelPosition, separator: CameraLabelSettings.Separator, + autoNumber: Bool, groupByCamera: Bool) { + self.position = position + self.separator = separator + self.autoNumber = autoNumber + self.groupByCamera = groupByCamera + } + private enum CodingKeys: String, CodingKey, CaseIterable { case position, separator, autoNumber, groupByCamera } + init(from decoder: Decoder) throws { + try rejectUnknownSettingsKeys(decoder, allowed: CodingKeys.allCases.map(\.rawValue)) + let values = try decoder.container(keyedBy: CodingKeys.self) + position = try values.decode(CameraLabelSettings.LabelPosition.self, forKey: .position) + separator = try values.decode(CameraLabelSettings.Separator.self, forKey: .separator) + autoNumber = try values.decode(Bool.self, forKey: .autoNumber) + groupByCamera = try values.decode(Bool.self, forKey: .groupByCamera) + } + } + + init(verificationMode: VerificationMode, generateASCMHL: Bool, ignoreAppleDouble: Bool, + reports: Reports, naming: Naming) { + self.verificationMode = verificationMode + self.generateASCMHL = generateASCMHL + self.ignoreAppleDouble = ignoreAppleDouble + self.reports = reports + self.naming = naming + } + + private enum CodingKeys: String, CodingKey, CaseIterable { + case format, schemaVersion, verificationMode, generateASCMHL, ignoreAppleDouble, reports, naming + } + + init(from decoder: Decoder) throws { + let values = try decoder.container(keyedBy: CodingKeys.self) + guard try values.decode(String.self, forKey: .format) == Self.format else { + throw TransferSettingsImportError.invalidFile + } + guard try values.decode(Int.self, forKey: .schemaVersion) == Self.schemaVersion else { + throw TransferSettingsImportError.unsupportedVersion + } + try rejectUnknownSettingsKeys(decoder, allowed: CodingKeys.allCases.map(\.rawValue)) + verificationMode = try values.decode(VerificationMode.self, forKey: .verificationMode) + generateASCMHL = try values.decode(Bool.self, forKey: .generateASCMHL) + ignoreAppleDouble = try values.decode(Bool.self, forKey: .ignoreAppleDouble) + reports = try values.decode(Reports.self, forKey: .reports) + naming = try values.decode(Naming.self, forKey: .naming) + } + + func encode(to encoder: Encoder) throws { + var values = encoder.container(keyedBy: CodingKeys.self) + try values.encode(Self.format, forKey: .format) + try values.encode(Self.schemaVersion, forKey: .schemaVersion) + try values.encode(verificationMode, forKey: .verificationMode) + try values.encode(generateASCMHL, forKey: .generateASCMHL) + try values.encode(ignoreAppleDouble, forKey: .ignoreAppleDouble) + try values.encode(reports, forKey: .reports) + try values.encode(naming, forKey: .naming) + } + + func encoded() throws -> Data { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + return try encoder.encode(self) + } + + static func decode(_ data: Data) throws -> Self { + guard data.count <= maximumBytes else { throw TransferSettingsImportError.tooLarge } + do { return try JSONDecoder().decode(Self.self, from: data) } + catch let error as TransferSettingsImportError { throw error } + catch { throw TransferSettingsImportError.invalidFile } + } + + struct Change: Identifiable, Equatable, Sendable { + let title: String + let before: String + let after: String + var id: String { title } + } + + func changes(from current: Self) -> [Change] { + var result: [Change] = [] + func add(_ title: String, _ before: String, _ after: String) { + if before != after { result.append(Change(title: title, before: before, after: after)) } + } + func onOff(_ value: Bool) -> String { value ? "On" : "Off" } + add("Verification", current.verificationMode.rawValue, verificationMode.rawValue) + add("ASC MHL", onOff(current.generateASCMHL), onOff(generateASCMHL)) + add("Ignore AppleDouble metadata", onOff(current.ignoreAppleDouble), onOff(ignoreAppleDouble)) + add("Reports", onOff(current.reports.makeReport), onOff(reports.makeReport)) + add("PDF clip thumbnails", onOff(current.reports.includeThumbnails), onOff(reports.includeThumbnails)) + add("Label position", current.naming.position.rawValue, naming.position.rawValue) + add("Separator", current.naming.separator.displayName, naming.separator.displayName) + add("Auto-number folders", onOff(current.naming.autoNumber), onOff(naming.autoNumber)) + add("Group by camera", onOff(current.naming.groupByCamera), onOff(naming.groupByCamera)) + return result + } +} + +enum TransferSettingsImportError: LocalizedError, Equatable { + case invalidFile, unsupportedVersion, tooLarge, busy, settingsChanged + var errorDescription: String? { + switch self { + case .invalidFile: "Choose a transfer settings JSON file exported by BitMatch." + case .unsupportedVersion: "This settings file needs a different version of BitMatch." + case .tooLarge: "This file is too large to be a transfer settings file." + case .busy: "Finish current or queued work before importing settings." + case .settingsChanged: "Settings changed while this preview was open. Import the file again to review the changes." + } + } +} + +private struct SettingsCodingKey: CodingKey { + let stringValue: String + let intValue: Int? = nil + init?(stringValue: String) { self.stringValue = stringValue } + init?(intValue: Int) { return nil } +} + +private func rejectUnknownSettingsKeys(_ decoder: Decoder, allowed: [String]) throws { + let values = try decoder.container(keyedBy: SettingsCodingKey.self) + guard Set(values.allKeys.map(\.stringValue)).isSubset(of: Set(allowed)) else { + throw TransferSettingsImportError.invalidFile + } +} diff --git a/Shared/Core/Services/TransferSettingsImport.swift b/Shared/Core/Services/TransferSettingsImport.swift new file mode 100644 index 00000000..548b481b --- /dev/null +++ b/Shared/Core/Services/TransferSettingsImport.swift @@ -0,0 +1,105 @@ +import Foundation +import Combine +import Darwin +import BitMatchEngine + +/// Bound reads before decoding: choosing a large report or media file must not +/// load it into memory just to find out it is not a settings file. +enum TransferSettingsFileReader { + static func read(_ url: URL) throws -> Data { + guard url.isFileURL else { throw TransferSettingsImportError.invalidFile } + let accessed = url.startAccessingSecurityScopedResource() + defer { if accessed { url.stopAccessingSecurityScopedResource() } } + try Task.checkCancellation() + let descriptor = open(url.path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC | O_NONBLOCK) + guard descriptor >= 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } + let handle = FileHandle(fileDescriptor: descriptor, closeOnDealloc: true) + defer { try? handle.close() } + var info = stat() + guard fstat(descriptor, &info) == 0, (info.st_mode & S_IFMT) == S_IFREG else { + throw TransferSettingsImportError.invalidFile + } + guard info.st_size <= PortableTransferSettings.maximumBytes else { throw TransferSettingsImportError.tooLarge } + var data = Data() + while data.count <= PortableTransferSettings.maximumBytes { + try Task.checkCancellation() + let next = try handle.read(upToCount: PortableTransferSettings.maximumBytes + 1 - data.count) ?? Data() + if next.isEmpty { break } + data.append(next) + } + try Task.checkCancellation() + guard data.count <= PortableTransferSettings.maximumBytes else { throw TransferSettingsImportError.tooLarge } + return data + } +} + +struct TransferSettingsReview: Identifiable, Sendable { + let id = UUID() + let current: PortableTransferSettings + let proposed: PortableTransferSettings + var changes: [PortableTransferSettings.Change] { proposed.changes(from: current) } +} + +@MainActor +final class TransferSettingsImportModel: ObservableObject { + @Published private(set) var review: TransferSettingsReview? + + func preview(_ data: Data, coordinator: SharedAppCoordinator) throws { + review = nil + guard coordinator.canImportTransferSettings else { throw TransferSettingsImportError.busy } + let decoded = try PortableTransferSettings.decode(data) + review = TransferSettingsReview(current: coordinator.portableTransferSettings, proposed: decoded) + } + + func cancel() { review = nil } + + func apply(to coordinator: SharedAppCoordinator) throws { + guard let review else { throw TransferSettingsImportError.invalidFile } + try coordinator.applyTransferSettings(review) + self.review = nil + } +} + +@MainActor +extension SharedAppCoordinator { + var canImportTransferSettings: Bool { + !isOperationInProgress && !queueIsRunning && !isReplayingQueuedTransfer + && !hasUnresolvedQueueRecords && editingSetupTransferID == nil && reviewedQueueRecordID == nil + && !transferJournal.records.contains(where: { $0.state.showsInQueue }) + && !photographerJobViewModel.isPreparing && !photographerJobViewModel.hasPreparedIngestAwaitingStart + } + + var portableTransferSettings: PortableTransferSettings { + let naming = cameraLabels.settings + return PortableTransferSettings(verificationMode: verificationMode, generateASCMHL: generateASCMHL, + ignoreAppleDouble: appleDoubleSelection.enabled, + reports: .init(makeReport: reportSettings.makeReport, includeThumbnails: reportSettings.includeThumbnails), + naming: .init(position: naming.position, separator: naming.separator, + autoNumber: naming.autoNumber, groupByCamera: naming.groupByCamera)) + } + + func applyTransferSettings(_ review: TransferSettingsReview) throws { + // A transfer or queue can start while the preview is open. Recheck on + // the main actor immediately before changing any preference. + guard canImportTransferSettings else { throw TransferSettingsImportError.busy } + guard portableTransferSettings == review.current else { throw TransferSettingsImportError.settingsChanged } + let proposed = review.proposed + verificationMode = proposed.verificationMode + generateASCMHL = proposed.generateASCMHL + var reports = reportSettings + reports.makeReport = proposed.reports.makeReport + reports.includeThumbnails = proposed.reports.includeThumbnails + reportSettings = reports + if proposed.naming != portableTransferSettings.naming { + var naming = cameraLabels.settings + naming.position = proposed.naming.position + naming.separator = proposed.naming.separator + naming.autoNumber = proposed.naming.autoNumber + naming.groupByCamera = proposed.naming.groupByCamera + // The aggregate cameraLabelSettings setter also changes selection. + // Preserve its private label/path and use the naming model directly. + cameraLabels.settings = naming + } + appleDoubleSelection.enabled = proposed.ignoreAppleDouble + } +} diff --git a/Shared/Views/TransferSettingsSection.swift b/Shared/Views/TransferSettingsSection.swift new file mode 100644 index 00000000..d63cba58 --- /dev/null +++ b/Shared/Views/TransferSettingsSection.swift @@ -0,0 +1,144 @@ +import SwiftUI +import UniformTypeIdentifiers +import BitMatchEngine + +/// The same quiet Settings section and review flow on Mac, iPhone and iPad. +struct TransferSettingsSection: View { + @ObservedObject var coordinator: SharedAppCoordinator + @StateObject private var importer = TransferSettingsImportModel() + @State private var choosingFile = false + @State private var exporting = false + @State private var exportDocument: TransferSettingsDocument? + @State private var errorMessage: String? + @State private var readTask: Task? + @State private var isReading = false + + var body: some View { + Section { + Button("Export settings…") { + do { + exportDocument = TransferSettingsDocument(data: try coordinator.portableTransferSettings.encoded()) + exporting = true + } catch { errorMessage = error.localizedDescription } + } + .disabled(!coordinator.canImportTransferSettings) + Button("Import settings…") { choosingFile = true } + .disabled(!coordinator.canImportTransferSettings || isReading) + } header: { + Text("Transfer settings") + } footer: { + Text(coordinator.canImportTransferSettings + ? "Move transfer options between devices. Backups and project details stay on this device." + : "Finish current or queued work before importing settings.") + } + .fileImporter(isPresented: $choosingFile, allowedContentTypes: [.json]) { result in + switch result { + case .success(let url): + readTask?.cancel() + isReading = true + readTask = Task { + let reader = Task.detached { try TransferSettingsFileReader.read(url) } + do { + let data = try await withTaskCancellationHandler { + try await reader.value + } onCancel: { reader.cancel() } + try Task.checkCancellation() + try importer.preview(data, coordinator: coordinator) + } catch is CancellationError { + // Dismissing Settings cancels this read, not a transfer. + } catch { errorMessage = error.localizedDescription } + isReading = false + } + case .failure(let error): + if (error as NSError).code != CocoaError.userCancelled.rawValue { errorMessage = error.localizedDescription } + } + } + .fileExporter(isPresented: $exporting, document: exportDocument, contentType: .json, + defaultFilename: "BitMatch-transfer-settings") { result in + if case .failure(let error) = result, + (error as NSError).code != CocoaError.userCancelled.rawValue { errorMessage = error.localizedDescription } + } + .sheet(isPresented: Binding(get: { importer.review != nil }, set: { if !$0 { importer.cancel(); errorMessage = nil } })) { + if let review = importer.review { + TransferSettingsReviewView(review: review, errorMessage: errorMessage, + apply: { + do { try importer.apply(to: coordinator); errorMessage = nil } + catch { errorMessage = error.localizedDescription } + }, cancel: { importer.cancel(); errorMessage = nil }) + } + } + .alert("Couldn’t use settings file", isPresented: Binding( + get: { errorMessage != nil && importer.review == nil }, set: { if !$0 { errorMessage = nil } }) + ) { Button("OK") { errorMessage = nil } } message: { Text(errorMessage ?? "") } + .onDisappear { readTask?.cancel() } + } +} + +struct TransferSettingsReviewView: View { + let review: TransferSettingsReview + var errorMessage: String? = nil + let apply: () -> Void + let cancel: () -> Void + + var body: some View { + NavigationStack { + Form { + if review.changes.isEmpty { + Section { Text("These transfer settings already match.") } + } else { + Section("Changes") { + ForEach(review.changes) { change in + VStack(alignment: .leading, spacing: 4) { + Text(change.title) + Text("\(change.before) → \(change.after)") + .font(.subheadline).foregroundStyle(.secondary) + } + .padding(.vertical, 2) + .accessibilityElement(children: .combine) + } + } + } + if review.proposed.verificationMode == .quick { + Section { + Label("Quick checks file sizes. It does not verify contents.", systemImage: "exclamationmark.triangle") + .foregroundStyle(.orange) + } + } + if let errorMessage { + Section { Text(errorMessage).foregroundStyle(.red) } + } + Section { + Text("Changes apply to future transfers. Backups, project details and saved transfers stay as they are.") + .font(.footnote).foregroundStyle(.secondary) + } + } + #if os(iOS) + .navigationTitle("Import") + .navigationBarTitleDisplayMode(.inline) + #else + .navigationTitle("Import settings") + #endif + .toolbar { + ToolbarItem(placement: .cancellationAction) { Button("Cancel", action: cancel) } + ToolbarItem(placement: .confirmationAction) { + Button("Apply", action: apply).disabled(review.changes.isEmpty) + } + } + } + #if os(macOS) + .frame(width: 480, height: 600) + #endif + } +} + +private struct TransferSettingsDocument: FileDocument { + static var readableContentTypes: [UTType] { [.json] } + let data: Data + init(data: Data) { self.data = data } + init(configuration: ReadConfiguration) throws { + // Import uses the bounded URL reader, rather than loading an arbitrary + // FileWrapper into memory through this export-only document. + throw TransferSettingsImportError.invalidFile + } + func fileWrapper(configuration: WriteConfiguration) throws -> FileWrapper { FileWrapper(regularFileWithContents: data) } +} diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 09297aae..ae769fda 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -39,6 +39,12 @@ BitMatch checks the file structure, shows a quiet exclusion count, and keeps the The option follows the transfer into the queue and history. Reports list the exclusions, and ASC MHL describes the selected destination files. **Keep the source:** “selected files verified” does not mean the whole card is backed up or safe to erase. Existing destination metadata is left alone, including companions macOS creates itself. This option doesn't repair conflicting files or filter an existing checksum inventory during Check. +## Move Transfer Settings Between Devices + +Use **Settings → Transfer settings → Export settings…** to save your verification, ASC MHL, AppleDouble, report and basic folder-naming choices. On another Mac, iPad or iPhone, choose **Import settings…**, review the changes, then Apply. Cancel leaves your settings alone. + +This moves options, not a job: it leaves out backup locations, camera labels, folder paths, operator and notes, project details, credentials and saved transfers. Import is unavailable while transfers or queued work need attention. Changing the AppleDouble preference reviews the current source again before Start. Folder recipes and saved destinations aren't included. + ## Check Existing Backups Choose **Check**, then pick the folder or drive you want to check. Use **Its saved checksums** when BitMatch finds supported records, such as a BitMatch JSON report or an ASC MHL inventory. It reads the files again and compares them with those records; the original card does not need to be connected. diff --git a/docs/TODO.md b/docs/TODO.md index 3b4ce1d9..3dff0cb3 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -158,7 +158,7 @@ Remote scope excludes screen sharing, arbitrary filesystem browsing, remote shel ## P4 — Convenience after reliability -- [ ] **Portable settings/workflow export and import.** Build on existing project recipes and saved settings. Include safe preferences, verification/report choices, naming recipes and reusable destination configurations; preview changes before import and revalidate destinations on the receiving machine. **Acceptance:** omit credentials, security-scoped bookmarks, private history and machine-specific identities. No facility configuration-management system. +- [ ] **Portable settings/workflow export and import.** The first bounded slice moves verification, ASC MHL, the global AppleDouble preference, report/thumbnail switches and basic naming options between Mac, iPad and iPhone. It requires a changes preview, rejects unsupported or oversized files, and blocks Apply during active, queued or prepared work. It omits labels, paths, per-source exclusions, private report fields, credentials, bookmarks, history and identities. **Remaining:** explicit portable folder-recipe and reusable destination configuration designs; never serialize existing job/preset records or activate imported destinations automatically. Preview changes and revalidate destinations on the receiving machine. No facility configuration-management system. - [ ] **Simplified operator mode, only if users ask.** Investigate preconfigured destinations, naming and verification with an everyday source → start → review flow. Warnings stay visible and validation still runs; no facility administration or permission system. - [ ] **Organize and rename.** Extend existing project naming and folder recipes with remaining naming-pattern and organize-by-date workflows. Preserve original card contents and test destination collisions before adding new options. - [ ] **Honest byte-weighted progress.** Implemented in #56. Integration passed full engine/Mac tests, the universal iOS build and iPhone/iPad simulator suites. The shared implementation budgets selected-file bytes for source reads, writes/reuse checks, required verification and optional ASC MHL work; live chunk callbacks advance large clips, and finishing reports never reset the bar or show active 100%. Destination bars use their own byte work. Queue cards use persisted size estimates, destination counts and verification mode; a queue with unknown sizes stays indeterminate. Estimates stay separate from copy evidence and remain stable per attempt. Synthetic regressions cover tiny-plus-large files, uneven destinations, reuse, cancellation/failure, MHL mapping and 10,000 small files. **Remaining:** signed-app/physical-drive walkthroughs and finer progress through extra source-hashing passes on reused files. The current retry budget remains deliberately logical rather than an exact tally of every physical read. Reserve 100% for the authoritative result. From 1d72c5366a038b724d90272903c21c06a5b7990f Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 22:03:24 -0400 Subject: [PATCH 11/56] Show reported USB 2 connections in Advanced --- .../DriveConnectionDetailsTests.swift | 98 ++++++++++++++ .../File/DriveConnectionInfo.swift | 123 ++++++++++++++++++ .../DriveConnectionInfoTests.swift | 102 +++++++++++++++ .../Models/DriveConnectionDetailsModel.swift | 68 ++++++++++ .../Views/Setup/CoordinatorSetupScreen.swift | 3 + .../Setup/DriveConnectionDetailsSection.swift | 26 ++++ 6 files changed, 420 insertions(+) create mode 100644 BitMatchTests/DriveConnectionDetailsTests.swift create mode 100644 Packages/BitMatchEngine/Sources/BitMatchEngine/File/DriveConnectionInfo.swift create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/DriveConnectionInfoTests.swift create mode 100644 Shared/Core/Models/DriveConnectionDetailsModel.swift create mode 100644 Shared/Views/Setup/DriveConnectionDetailsSection.swift diff --git a/BitMatchTests/DriveConnectionDetailsTests.swift b/BitMatchTests/DriveConnectionDetailsTests.swift new file mode 100644 index 00000000..2e7aa728 --- /dev/null +++ b/BitMatchTests/DriveConnectionDetailsTests.swift @@ -0,0 +1,98 @@ +import XCTest +import BitMatchEngine +@testable import BitMatch + +@MainActor +final class DriveConnectionDetailsTests: XCTestCase { + private final class Provider: DriveConnectionInfoProviding, @unchecked Sendable { + private let lock = NSLock() + private var requested: [URL] = [] + var calls: [URL] { lock.withLock { requested } } + let blocked: URL? + let started = XCTestExpectation(description: "Lookup entered") + let release = DispatchSemaphore(value: 0) + + init(blocked: URL? = nil) { self.blocked = blocked } + + func connectionInfo(for url: URL) -> DriveConnectionInfo? { + lock.withLock { requested.append(url) } + if url == blocked { + started.fulfill() + // A test failure cannot leave the test process blocked. + guard release.wait(timeout: .now() + 10) == .success else { return nil } + } + return url.lastPathComponent == "Unknown" ? nil : .usb2HighSpeed + } + } + + func testOnlySelectedKnownConnectionsAppearInSelectionOrder() async { + let provider = Provider() + let model = DriveConnectionDetailsModel(provider: provider) + let first = URL(fileURLWithPath: "/Volumes/Backup A/Card") + let unknown = URL(fileURLWithPath: "/Volumes/Unknown") + let second = URL(fileURLWithPath: "/Volumes/Backup B") + await model.refresh(destinations: [first, unknown, second]) + XCTAssertEqual(provider.calls, [first, unknown, second]) + XCTAssertEqual(model.rows.map(\.url), [first, second]) + XCTAssertEqual(model.rows.map(\.title), ["Backup A", "Backup B"]) + XCTAssertTrue(model.rows.allSatisfy { $0.connection == .usb2HighSpeed }) + XCTAssertEqual(model.rows.first?.connectionLabel, "USB 2 connection (480 Mb/s)") + } + + func testSupersededLookupCannotPublishOverNewSelection() async { + let old = URL(fileURLWithPath: "/Volumes/Old") + let new = URL(fileURLWithPath: "/Volumes/New") + let provider = Provider(blocked: old) + let model = DriveConnectionDetailsModel(provider: provider) + let oldTask = Task { await model.refresh(destinations: [old]) } + defer { provider.release.signal(); oldTask.cancel(); model.clear() } + await fulfillment(of: [provider.started], timeout: 5) + await model.refresh(destinations: [new]) + XCTAssertEqual(model.rows.map(\.url), [new]) + provider.release.signal() + await oldTask.value + XCTAssertEqual(model.rows.map(\.url), [new]) + XCTAssertEqual(provider.calls, [old, new]) + } + + func testClearDropsPendingResultAndStopsLaterSelectedLookups() async { + let old = URL(fileURLWithPath: "/Volumes/Old") + let later = URL(fileURLWithPath: "/Volumes/Later") + let provider = Provider(blocked: old) + let model = DriveConnectionDetailsModel(provider: provider) + let task = Task { await model.refresh(destinations: [old, later]) } + defer { provider.release.signal(); task.cancel(); model.clear() } + await fulfillment(of: [provider.started], timeout: 5) + model.clear() + provider.release.signal() + await task.value + XCTAssertTrue(model.rows.isEmpty) + XCTAssertEqual(provider.calls, [old]) + } + + func testParentTaskCancellationDropsPendingResultAndStopsLaterLookups() async { + let old = URL(fileURLWithPath: "/Volumes/Old") + let later = URL(fileURLWithPath: "/Volumes/Later") + let provider = Provider(blocked: old) + let model = DriveConnectionDetailsModel(provider: provider) + let task = Task { await model.refresh(destinations: [old, later]) } + defer { provider.release.signal(); task.cancel(); model.clear() } + await fulfillment(of: [provider.started], timeout: 5) + task.cancel() + provider.release.signal() + await task.value + XCTAssertTrue(model.rows.isEmpty) + XCTAssertEqual(provider.calls, [old]) + } + + func testEmptySelectionClearsKnownHintWithoutAnotherLookup() async { + let provider = Provider() + let model = DriveConnectionDetailsModel(provider: provider) + let destination = URL(fileURLWithPath: "/Volumes/Backup") + await model.refresh(destinations: [destination]) + XCTAssertEqual(model.rows.count, 1) + await model.refresh(destinations: []) + XCTAssertTrue(model.rows.isEmpty) + XCTAssertEqual(provider.calls, [destination]) + } +} diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DriveConnectionInfo.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DriveConnectionInfo.swift new file mode 100644 index 00000000..2003838d --- /dev/null +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DriveConnectionInfo.swift @@ -0,0 +1,123 @@ +import Foundation + +#if os(macOS) +import DiskArbitration +import IOKit +import IOUSBHost +#endif + +/// A reported connection fact, not a drive benchmark or a safety verdict. +public enum DriveConnectionInfo: Equatable, Sendable { + case usb2HighSpeed + + public var bitsPerSecond: UInt64 { 480_000_000 } +} + +public protocol DriveConnectionInfoProviding: Sendable { + func connectionInfo(for url: URL) -> DriveConnectionInfo? +} + +/// Reads the selected drive's registry ancestry. It never opens a USB user +/// client, claims the device, or probes transfer speed. +public struct SystemDriveConnectionInfoProvider: DriveConnectionInfoProviding { + public init() {} + + public func connectionInfo(for url: URL) -> DriveConnectionInfo? { +#if os(macOS) + guard url.isFileURL, !Task.isCancelled else { return nil } + let scoped = url.startAccessingSecurityScopedResource() + defer { if scoped { url.stopAccessingSecurityScopedResource() } } + guard let service = Self.mediaService(for: url) else { return nil } + var initialID: UInt64 = 0 + guard IORegistryEntryGetRegistryEntryID(service, &initialID) == KERN_SUCCESS else { + IOObjectRelease(service) + return nil + } + + let fact = DriveConnectionRegistryWalk.find( + startingAt: service, + isCancelled: { Task.isCancelled }, + isUSBDevice: { IOObjectConformsTo($0, "IOUSBHostDevice") != 0 }, + speed: { entry in + guard let value = IORegistryEntryCreateCFProperty( + entry, IOUSBHostMatchingPropertyKey.speed.rawValue as CFString, + kCFAllocatorDefault, 0 + )?.takeRetainedValue() as? NSNumber else { return nil } + // Do not truncate a fractional or unrelated property into a + // known speed. USBSpeed uses the IOUSBHost enum, not the older + // USBDeviceSpeed enum, whose high-speed value is different. + let number = value.doubleValue + guard number.isFinite, number.rounded() == number, + (0...7).contains(number) else { return nil } + return Int(number) + }, + parent: { entry in + var parent: io_registry_entry_t = 0 + guard IORegistryEntryGetParentEntry(entry, kIOServicePlane, &parent) == KERN_SUCCESS, + parent != 0 else { return nil } + return parent + }, + release: { IOObjectRelease($0) } + ) + guard fact != nil, !Task.isCancelled, + let current = Self.mediaService(for: url) else { return nil } + defer { IOObjectRelease(current) } + var currentID: UInt64 = 0 + guard IORegistryEntryGetRegistryEntryID(current, ¤tID) == KERN_SUCCESS, + currentID == initialID, !Task.isCancelled else { return nil } + return fact +#else + return nil +#endif + } + +#if os(macOS) + /// The returned registry handle is owned by the caller. + private static func mediaService(for url: URL) -> io_service_t? { + guard let session = DASessionCreate(kCFAllocatorDefault), + let disk = DADiskCreateFromVolumePath(kCFAllocatorDefault, session, url as CFURL) else { + return nil + } + let whole = DADiskCopyWholeDisk(disk) ?? disk + guard let description = DADiskCopyDescription(whole) as? [String: Any], + let name = description[kDADiskDescriptionMediaBSDNameKey as String] as? String, + let matching = name.withCString({ IOBSDNameMatching(kIOMainPortDefault, 0, $0) }) else { + return nil + } + let service = IOServiceGetMatchingService(kIOMainPortDefault, matching) + return service == 0 ? nil : service + } +#endif +} + +/// Consumes its initial handle and every retained parent. Stop at the nearest +/// USB device: a hub's speed must never stand in for its attached drive. +enum DriveConnectionRegistryWalk { + static func find( + startingAt initial: Handle, + limit: Int = 64, + isCancelled: () -> Bool, + isUSBDevice: (Handle) -> Bool, + speed: (Handle) -> Int?, + parent: (Handle) -> Handle?, + release: (Handle) -> Void + ) -> DriveConnectionInfo? { + var current = initial + defer { release(current) } + guard limit > 0 else { return nil } + for depth in 0..? + private var generation: UInt64 = 0 + + init(provider: any DriveConnectionInfoProviding = SystemDriveConnectionInfoProvider()) { + self.provider = provider + } + + deinit { worker?.cancel() } + + func refresh(destinations: [URL]) async { + clear() + guard !destinations.isEmpty, !Task.isCancelled else { return } + let currentGeneration = generation + let task = Task.detached(priority: .utility) { [provider] in + var details: [DriveConnectionDetail] = [] + for url in destinations { + guard !Task.isCancelled else { break } + guard let connection = provider.connectionInfo(for: url), !Task.isCancelled else { continue } + details.append(DriveConnectionDetail( + url: url, + title: DestinationIdentityPresentation.title(for: url), + connection: connection + )) + } + return details + } + worker = task + let details = await withTaskCancellationHandler { + await task.value + } onCancel: { + task.cancel() + } + guard generation == currentGeneration else { return } + worker = nil + guard !Task.isCancelled, !task.isCancelled else { return } + rows = details + } + + func clear() { + generation &+= 1 + worker?.cancel() + worker = nil + rows = [] + } +} diff --git a/Shared/Views/Setup/CoordinatorSetupScreen.swift b/Shared/Views/Setup/CoordinatorSetupScreen.swift index 0a8f66fe..67a9138b 100644 --- a/Shared/Views/Setup/CoordinatorSetupScreen.swift +++ b/Shared/Views/Setup/CoordinatorSetupScreen.swift @@ -129,6 +129,9 @@ struct CoordinatorSetupScreen Date: Mon, 19 Oct 2026 22:09:38 -0400 Subject: [PATCH 12/56] Resolve selected backup folders before reading USB connection facts --- .../File/DriveConnectionInfo.swift | 23 ++++++++- .../DriveConnectionInfoTests.swift | 50 +++++++++++++++++++ .../Models/DriveConnectionDetailsModel.swift | 3 ++ .../Setup/DriveConnectionDetailsSection.swift | 1 + 4 files changed, 76 insertions(+), 1 deletion(-) diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DriveConnectionInfo.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DriveConnectionInfo.swift index 2003838d..679b0dd2 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DriveConnectionInfo.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DriveConnectionInfo.swift @@ -75,7 +75,11 @@ public struct SystemDriveConnectionInfoProvider: DriveConnectionInfoProviding { /// The returned registry handle is owned by the caller. private static func mediaService(for url: URL) -> io_service_t? { guard let session = DASessionCreate(kCFAllocatorDefault), - let disk = DADiskCreateFromVolumePath(kCFAllocatorDefault, session, url as CFURL) else { + let disk = DriveConnectionVolumeResolver.disk( + for: url, + readVolumeRoot: { try? $0.resourceValues(forKeys: [.volumeURLKey]).volume }, + makeDisk: { DADiskCreateFromVolumePath(kCFAllocatorDefault, session, $0 as CFURL) } + ) else { return nil } let whole = DADiskCopyWholeDisk(disk) ?? disk @@ -90,6 +94,23 @@ public struct SystemDriveConnectionInfoProvider: DriveConnectionInfoProviding { #endif } +/// Disk Arbitration needs a mount root, even when the selected backup is a +/// folder several levels below it. Read only that selection's volume fact; +/// never discover or enumerate other mounted volumes. +enum DriveConnectionVolumeResolver { + static func disk( + for selectedURL: URL, + readVolumeRoot: (URL) -> URL?, + makeDisk: (URL) -> Disk?, + isCancelled: () -> Bool = { Task.isCancelled } + ) -> Disk? { + guard selectedURL.isFileURL, !isCancelled(), + let volumeRoot = readVolumeRoot(selectedURL), volumeRoot.isFileURL, + !isCancelled() else { return nil } + return makeDisk(volumeRoot) + } +} + /// Consumes its initial handle and every retained parent. Stop at the nearest /// USB device: a hub's speed must never stand in for its attached drive. enum DriveConnectionRegistryWalk { diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DriveConnectionInfoTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DriveConnectionInfoTests.swift index f5c4b60a..1ff6fcc4 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DriveConnectionInfoTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DriveConnectionInfoTests.swift @@ -2,6 +2,56 @@ import XCTest @testable import BitMatchEngine final class DriveConnectionInfoTests: XCTestCase { + func testSelectedFolderResolvesItsOwnVolumeBeforeDiskLookup() { + let root = URL(fileURLWithPath: "/Volumes/Backup") + for selected in [root, root.appendingPathComponent("Card"), root.appendingPathComponent("Job/Day/Card")] { + var reads: [URL] = [] + var lookups: [URL] = [] + let value = DriveConnectionVolumeResolver.disk( + for: selected, + readVolumeRoot: { reads.append($0); return root }, + makeDisk: { lookups.append($0); return "selected-disk" } + ) + XCTAssertEqual(value, "selected-disk") + XCTAssertEqual(reads, [selected]) + XCTAssertEqual(lookups, [root], "Disk Arbitration must receive the mount root") + } + } + + func testMissingOrNonFileVolumeCannotTriggerDiskLookup() { + let selected = URL(fileURLWithPath: "/Volumes/Backup/Card") + let remote = URL(string: "sftp://example.invalid/backup")! + for volume in [nil, remote] as [URL?] { + let value: String? = DriveConnectionVolumeResolver.disk( + for: selected, readVolumeRoot: { _ in volume }, + makeDisk: { _ in XCTFail("No reliable selected volume"); return "unexpected" } + ) + XCTAssertNil(value) + } + let value: String? = DriveConnectionVolumeResolver.disk( + for: remote, readVolumeRoot: { _ in XCTFail("Not a local selection"); return selected }, + makeDisk: { _ in XCTFail("Not a local selection"); return "unexpected" } + ) + XCTAssertNil(value) + } + + func testCancelledVolumeResolutionCannotStartDiskLookup() { + for cancelDuringRead in [false, true] { + var cancelled = !cancelDuringRead + let value: String? = DriveConnectionVolumeResolver.disk( + for: URL(fileURLWithPath: "/Volumes/Backup/Card"), + readVolumeRoot: { _ in + XCTAssertTrue(cancelDuringRead) + cancelled = true + return URL(fileURLWithPath: "/Volumes/Backup") + }, + makeDisk: { _ in XCTFail("Cancelled lookup"); return "unexpected" }, + isCancelled: { cancelled } + ) + XCTAssertNil(value) + } + } + func testOnlyUSBHostHighSpeedProducesHint() { // IOUSBHost high-speed is 3. Legacy USBDeviceSpeed uses 2; it must // not accidentally label low/full-speed or unknown links USB 2. diff --git a/Shared/Core/Models/DriveConnectionDetailsModel.swift b/Shared/Core/Models/DriveConnectionDetailsModel.swift index 70793804..6cadb60a 100644 --- a/Shared/Core/Models/DriveConnectionDetailsModel.swift +++ b/Shared/Core/Models/DriveConnectionDetailsModel.swift @@ -17,6 +17,9 @@ nonisolated struct DriveConnectionDetail: Identifiable, Equatable, Sendable { /// Owns only the expanded Setup hint. A removed or replaced selection cannot /// publish a late registry lookup into the current setup. +/// This is a snapshot refreshed when Advanced opens or selected backups change, +/// not a live cable monitor. Reopening Advanced refreshes an unchanged URL after +/// a reconnect; the provider's media-ID check protects only an active lookup. @MainActor final class DriveConnectionDetailsModel: ObservableObject { @Published private(set) var rows: [DriveConnectionDetail] = [] diff --git a/Shared/Views/Setup/DriveConnectionDetailsSection.swift b/Shared/Views/Setup/DriveConnectionDetailsSection.swift index d46bbd3a..1f9eeb04 100644 --- a/Shared/Views/Setup/DriveConnectionDetailsSection.swift +++ b/Shared/Views/Setup/DriveConnectionDetailsSection.swift @@ -14,6 +14,7 @@ struct DriveConnectionDetailsSection: View { Text("\(row.title): \(row.connectionLabel)") .font(.caption) .foregroundStyle(.secondary) + .help("Checked when Advanced opens or selected backups change. Reopen Advanced after reconnecting a drive.") } Text("A cable, hub or port may be limiting this connection.") .font(.caption) From 0a18806f169d684a5eea0731df4354e1332c2578 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 22:04:02 -0400 Subject: [PATCH 13/56] Add opt-in physical workflow evidence checks --- .../PhysicalWorkflowEvidenceTests.swift | 619 ++++++++++++++++++ docs/HARDWARE_TESTING.md | 10 + 2 files changed, 629 insertions(+) create mode 100644 BitMatchTests/PhysicalWorkflowEvidenceTests.swift diff --git a/BitMatchTests/PhysicalWorkflowEvidenceTests.swift b/BitMatchTests/PhysicalWorkflowEvidenceTests.swift new file mode 100644 index 00000000..209d3a9d --- /dev/null +++ b/BitMatchTests/PhysicalWorkflowEvidenceTests.swift @@ -0,0 +1,619 @@ +#if os(macOS) +@preconcurrency import AVFoundation +import CoreVideo +import CryptoKit +import Darwin +import Foundation +import ImageIO +import PDFKit +import Synchronization +import XCTest +@testable import BitMatch +@testable import BitMatchEngine + +/// Explicit opt-in. Reuses already authorized/marked sd, ssdA and ssdB roots; +/// never formats, ejects, adopts an existing child or changes existing media. +/// +/// BITMATCH_HARDWARE_APP_WORKFLOW=1 BITMATCH_HARDWARE_CONFIG= +/// BITMATCH_HARDWARE_APP_ARTIFACTS= +/// Optional BITMATCH_HARDWARE_APP_BENCHMARK=1 runs three paired Standard/Thorough +/// trials (alternating order). It is a steady-state test, not a cable-pull test. +@MainActor +final class PhysicalWorkflowEvidenceTests: XCTestCase { + func testHostEvidenceHarnessSmoke() async throws { + guard ProcessInfo.processInfo.environment["BITMATCH_PHYSICAL_APP_HOST_SMOKE"] == "1" else { + throw XCTSkip("Set BITMATCH_PHYSICAL_APP_HOST_SMOKE=1 to validate this harness on host scratch folders") + } + let root = FileManager.default.temporaryDirectory.appendingPathComponent("bitmatch-physical-app-smoke-\(UUID())") + let source = root.appendingPathComponent("Card") + let targets = [root.appendingPathComponent("Backup A"), root.appendingPathComponent("Backup B")] + let evidence = root.appendingPathComponent("evidence") + for url in [source, evidence] + targets { try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) } + defer { try? FileManager.default.removeItem(at: root) } + try await seed(source, mibOverride: 32) + let original = try await Task.detached { try Self.snapshot(source) }.value + for mode in [VerificationMode.standard, .thorough] { + let destinations = try targets.map { target in + let destination = target.appendingPathComponent(mode.rawValue) + try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: false) + return destination + } + _ = try await run(source: source, original: original, targets: destinations, mode: mode, pair: 1, evidence: evidence) + } + } + + func testPhysicalConfigRefusesUnmarkedHostDirectoriesWithoutWrites() throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent("bitmatch-physical-config-\(UUID())") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let sentinel = directory.appendingPathComponent("keep.txt") + let original = Data("unrelated data".utf8) + try original.write(to: sentinel) + let nonce = UUID().uuidString + let roles = ["sd", "ssdA", "ssdB"].reduce(into: [String: PhysicalAppFixture.Volume]()) { + $0[$1] = .init(nonce: nonce, root: directory.path, mount: directory.deletingLastPathComponent().path, + volumeUUID: UUID().uuidString, filesystem: "apfs", initialFreeBytes: 0) + } + let config = directory.appendingPathComponent("config.json") + try JSONEncoder().encode(PhysicalAppFixture.Config(nonce: nonce, volumes: roles)).write(to: config) + XCTAssertThrowsError(try PhysicalAppFixture.load(environment: ["BITMATCH_HARDWARE_APP_WORKFLOW": "1", "BITMATCH_HARDWARE_CONFIG": config.path])) + XCTAssertEqual(try Data(contentsOf: sentinel), original) + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: directory.path).sorted(), ["config.json", "keep.txt"]) + } + + func testDescriptorCleanupDoesNotFollowLinksAndRefusesReplacement() throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent("bitmatch-physical-cleanup-\(UUID())") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let outside = directory.appendingPathComponent("outside") + let child = directory.appendingPathComponent("owned") + for url in [outside, child] { try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) } + let sentinel = outside.appendingPathComponent("keep.txt") + try Data("keep".utf8).write(to: sentinel) + try FileManager.default.createSymbolicLink(at: child.appendingPathComponent("link"), withDestinationURL: outside) + var identity = stat(); XCTAssertEqual(lstat(child.path, &identity), 0) + let parent = Darwin.open(directory.path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW) + XCTAssertGreaterThanOrEqual(parent, 0); defer { Darwin.close(parent) } + try PhysicalAppFixture.removeTree(parent: parent, name: "owned", device: identity.st_dev, inode: identity.st_ino) + XCTAssertEqual(try Data(contentsOf: sentinel), Data("keep".utf8)) + XCTAssertFalse(FileManager.default.fileExists(atPath: child.path)) + try FileManager.default.createDirectory(at: child, withIntermediateDirectories: false) + XCTAssertEqual(lstat(child.path, &identity), 0) + try FileManager.default.moveItem(at: child, to: directory.appendingPathComponent("old-owned")) + try FileManager.default.createDirectory(at: child, withIntermediateDirectories: false) + try Data("replacement".utf8).write(to: child.appendingPathComponent("keep.txt")) + XCTAssertThrowsError(try PhysicalAppFixture.removeTree(parent: parent, name: "owned", device: identity.st_dev, inode: identity.st_ino)) + XCTAssertEqual(try Data(contentsOf: child.appendingPathComponent("keep.txt")), Data("replacement".utf8)) + } + + func testPhysicalCardToTwoBackupsAndSavedEvidence() async throws { + let fixture = try PhysicalAppFixture.load() + let evidence = try evidenceDirectory() + let source = try fixture.child(role: "sd", name: "app-card-\(UUID())") + do { + try await seed(source) + let original = try await Task.detached { try Self.snapshot(source) }.value + let benchmark = ProcessInfo.processInfo.environment["BITMATCH_HARDWARE_APP_BENCHMARK"] == "1" + let trials: [[VerificationMode]] = benchmark + ? [[.standard, .thorough], [.thorough, .standard], [.standard, .thorough]] : [[.standard]] + var receipts: [PhysicalAppReceipt] = [] + for (pair, modes) in trials.enumerated() { + for mode in modes { + let label = "pair-\(pair + 1)-\(mode.rawValue)-\(UUID())" + let targets = try [fixture.child(role: "ssdA", name: "app-\(label)"), + fixture.child(role: "ssdB", name: "app-\(label)")] + let receipt = try await run(source: source, original: original, targets: targets, + mode: mode, pair: pair + 1, evidence: evidence) + receipts.append(receipt) + try JSONEncoder().encode(receipts).write(to: evidence.appendingPathComponent("trials.json"), options: .atomic) + try fixture.removeChild(targets[0]); try fixture.removeChild(targets[1]) + } + } + XCTAssertEqual(receipts.count, benchmark ? 6 : 1) + let after = try await Task.detached { try Self.snapshot(source) }.value + XCTAssertEqual(after, original, "Physical source changed") + try fixture.removeChild(source) + print("PHYSICAL_APP_SUMMARY trials=\(receipts.count) pairedBenchmark=\(benchmark) backups=2 realJournal=true independentHashes=true previewsEmbedded=true cleanup=true") + } catch { + // Keep owned children on failure for review; never delete a replaced + // or disconnected fixture by falling back to a pathname operation. + print("PHYSICAL_APP_FAILURE ownedChildrenRetained=true") + throw error + } + } + + private func run(source: URL, original: [String: PhysicalAppFile], targets: [URL], + mode: VerificationMode, pair: Int, evidence: URL) async throws -> PhysicalAppReceipt { + let suite = "BitMatchPhysicalApp.\(UUID())" + let defaults = try XCTUnwrap(UserDefaults(suiteName: suite)) + defer { defaults.removePersistentDomain(forName: suite) } + let prefix = "pair-\(pair)-\(mode.rawValue)" + let journalURL = evidence.appendingPathComponent("\(prefix)-journal.json") + let journal = LocalTransferJournal(fileURL: journalURL) + var settings = CameraLabelSettings() + settings.excludedAppleDoublePaths = try AppleDoubleSelection.review(source: source) + let excluded = Set(settings.excludedAppleDoublePaths ?? []) + let selected = original.filter { !excluded.contains($0.key) } + let prefs = ReportPrefs(verificationMode: mode, includeThumbnails: true, + notes: "Synthetic card; steady-state validation", operatorName: "BitMatch validation") + let samples = PhysicalAppMeasurements() + let operations = PhysicalAppOperations(samples: samples) + let coordinator = SharedAppCoordinator(platformManager: PhysicalAppPlatform(fileOperations: operations), + transferJournal: journal, defaults: defaults) + let id = try coordinator.enqueue(source: source, destinations: targets, verificationMode: mode, + cameraSettings: settings, generateASCMHL: true, reportSettings: prefs) + samples.state.withLock { $0.sample() } + let cpuStart = try Self.cpuSeconds() + let started = ProcessInfo.processInfo.systemUptime + let monitor = Task.detached { + while !Task.isCancelled { + samples.state.withLock { $0.sample() } + do { try await Task.sleep(for: .milliseconds(25)) } catch { break } + } + } + defer { monitor.cancel() } + coordinator.startQueue() + let ended = await waitUntil(timeout: .seconds(900)) { + !coordinator.queueIsRunning && !coordinator.isOperationInProgress + } + if !ended { coordinator.cancelOperation() } + let queueSettled = ProcessInfo.processInfo.systemUptime - started + let cpu = try Self.cpuSeconds() - cpuStart + monitor.cancel(); await monitor.value + XCTAssertTrue(ended, "Physical app workflow timed out") + guard ended else { throw PhysicalAppError.workflowFailed } + let records = try JSONDecoder().decode([LocalTransferRecord].self, from: Data(contentsOf: journalURL)) + let record = try XCTUnwrap(records.first { $0.id == id }) + XCTAssertEqual(record.state, excluded.isEmpty ? .completed : .issues, record.summary) + XCTAssertEqual(record.plannedFileCounts, [selected.count, selected.count]) + XCTAssertEqual(record.results.count, original.count * 2) + XCTAssertEqual(record.results.filter(\.isVerifiedStatus).count, selected.count * 2) + XCTAssertEqual(record.results.filter { ResultOutcome(statusText: $0.status) == .excludedAppleDouble }.count, excluded.count * 2) + XCTAssertEqual(TransferLibraryPresentation.safetyState(for: record) == .safeToErase, excluded.isEmpty) + guard record.results.filter(\.isVerifiedStatus).count == selected.count * 2 else { throw PhysicalAppError.workflowFailed } + let pipelineSamples = samples.state.withLock { $0 } + XCTAssertNotNil(pipelineSamples.pipelineReturnedUptime) + XCTAssertGreaterThanOrEqual(cpu, 0) + XCTAssertGreaterThan(pipelineSamples.peakMiB, 0) + XCTAssertLessThan(pipelineSamples.peakMiB, 1_024, "Small streaming app workflow exceeded 1 GiB physical footprint") + XCTAssertLessThan(pipelineSamples.peakMiB - pipelineSamples.initialMiB, 256, "Streaming app footprint grew unexpectedly") + let io = try XCTUnwrap(pipelineSamples.io) + let totalBytes = selected.values.reduce(Int64(0)) { $0 + $1.size } + XCTAssertEqual(io.destinations.count, 2) + for destination in io.destinations { + XCTAssertEqual(destination.writtenBytes, totalBytes) + XCTAssertGreaterThanOrEqual(destination.readbackBytes, totalBytes) + } + let telemetry = try XCTUnwrap(record.performanceTelemetry) + XCTAssertNotNil(telemetry.copyDurationSeconds) + XCTAssertNotNil(telemetry.verifyDurationSeconds) + XCTAssertNotNil(telemetry.mhlDurationSeconds) + XCTAssertGreaterThan(telemetry.mhlBytes ?? 0, 0) + for target in targets { + let output = try SafetyValidator.resolvedDestinationRootChecked(source: source, destination: target, settings: settings) + let actual = try await Task.detached { try Self.snapshot(output, ignoreEvidence: true) }.value + for (path, value) in selected { XCTAssertEqual(actual[path], value, "Independent destination hash mismatch") } + let generated = Set(try AppleDoubleSelection.review(source: output)) + XCTAssertTrue(Set(actual.keys).subtracting(selected.keys).isSubset(of: generated), "Unexpected destination media") + let history = output.appendingPathComponent("ascmhl") + XCTAssertTrue(FileManager.default.fileExists(atPath: history.appendingPathComponent("ascmhl_chain.xml").path)) + let checker = SavedChecksumCheck(fileAccess: MacOSFileSystemService.shared, checksum: ChecksumEngine.shared) + let discovery = try await checker.discover(under: output) + XCTAssertEqual(discovery.expectedFiles.count, selected.count) + let check = try await checker.check(discovery) { _ in } + XCTAssertTrue(check.isIntact) + XCTAssertEqual(check.matchingPaths.count, selected.count) + } + let reportFiles = try FileManager.default.contentsOfDirectory(at: targets[0].appendingPathComponent("Reports"), includingPropertiesForKeys: nil) + let automaticPDF = try XCTUnwrap(reportFiles.first { $0.pathExtension == "pdf" }) + let automaticData = try Data(contentsOf: automaticPDF) + let automaticText = try Self.pdfText(automaticData) + XCTAssertTrue(automaticText.contains("Clip previews: 1/1")) + XCTAssertTrue(automaticText.contains("Operator: BitMatch validation")) + let embedded = try Self.previewJPEGs(automaticData) + XCTAssertFalse(embedded.isEmpty, "Automatic PDF has no real embedded preview") + try automaticData.write(to: evidence.appendingPathComponent("\(prefix)-automatic.pdf")) + try XCTUnwrap(embedded.first).write(to: evidence.appendingPathComponent("\(prefix)-preview.jpg")) + for ext in ["json", "csv"] { + let report = try XCTUnwrap(reportFiles.first { $0.pathExtension == ext }) + let data = try Data(contentsOf: report) + XCTAssertFalse(data.isEmpty) + if ext == "json" { + let object = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any]) + XCTAssertEqual(object["safeToErase"] as? Bool, excluded.isEmpty) + XCTAssertEqual((object["results"] as? [[String: Any]])?.count, record.results.count) + } else { XCTAssertTrue(String(decoding: data, as: UTF8.self).contains("BitMatch validation")) } + try data.write(to: evidence.appendingPathComponent("\(prefix)-automatic.\(ext)")) + } + let historyPDF = try await ReportExporter.historyPDF(record: record, includeThumbnails: true) + XCTAssertTrue(try Self.pdfText(historyPDF).contains("Regenerated from saved history")) + XCTAssertTrue(try Self.pdfText(historyPDF).contains("Clip previews: 1/1")) + XCTAssertFalse(try Self.previewJPEGs(historyPDF).isEmpty) + try historyPDF.write(to: evidence.appendingPathComponent("\(prefix)-saved-history.pdf")) + for csv in [false, true] { + let data = try TransferHistoryDocument(record: record, asCSV: csv).data + if !csv { + let object = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any]) + XCTAssertEqual(object["savedSafeToErase"] as? Bool, excluded.isEmpty) + XCTAssertEqual((object["results"] as? [[String: Any]])?.count, record.results.count) + } + try data.write(to: evidence.appendingPathComponent("\(prefix)-saved-history.\(csv ? "csv" : "json")")) + } + let scanned = await ReportScanner.scanReports(at: targets[0]) + XCTAssertTrue(scanned.skipped.isEmpty) + XCTAssertEqual(scanned.cards.count, 1) + let master = try await SharedReportGenerationService().generateMasterReport(transfers: scanned.cards, configuration: .default()) + XCTAssertTrue(try Self.pdfText(master.pdfData).contains("MASTER TRANSFER REPORT")) + XCTAssertEqual(master.reportData.summary.totalTransfers, 1) + try master.pdfData.write(to: evidence.appendingPathComponent("\(prefix)-master.pdf")) + let after = try await Task.detached { try Self.snapshot(source) }.value + XCTAssertEqual(after, original, "Source changed after reports/check") + let receipt = PhysicalAppReceipt(pair: pair, mode: mode.rawValue, sourceFiles: original.count, + selectedFiles: selected.count, sourceBytes: totalBytes, queueSettledSeconds: queueSettled, + pipelineReturnedSeconds: pipelineSamples.pipelineReturnedUptime.map { $0 - started }, + cpuSeconds: cpu, initialMiB: pipelineSamples.initialMiB, peakMiB: pipelineSamples.peakMiB, + telemetry: telemetry, destinations: io.destinations.enumerated().map { + .init(index: $0.offset, pipelineWrittenBytes: $0.element.writtenBytes, pipelineReadbackBytes: $0.element.readbackBytes) + }) + print("PHYSICAL_APP_TRIAL pair=\(pair) mode=\(mode.rawValue) sourceFiles=\(original.count) selectedFiles=\(selected.count) bytes=\(totalBytes) queueSettledSeconds=\(queueSettled) cpuSeconds=\(cpu) initialMiB=\(receipt.initialMiB) peakMiB=\(receipt.peakMiB) previews=embedded history=real") + return receipt + } + + private func evidenceDirectory() throws -> URL { + guard let path = ProcessInfo.processInfo.environment["BITMATCH_HARDWARE_APP_ARTIFACTS"], path.hasPrefix("/") else { throw PhysicalAppError.unsafeFixture } + let base = URL(fileURLWithPath: path).standardizedFileURL.resolvingSymlinksInPath() + // Evidence/logs may not be placed on the tested removable media. + guard !base.path.hasPrefix("/Volumes/") else { throw PhysicalAppError.unsafeFixture } + let folder = base.appendingPathComponent("app-workflow-\(UUID())") + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + return folder + } + + private func seed(_ source: URL, mibOverride: Int? = nil) async throws { + let environment = ProcessInfo.processInfo.environment + let mib = mibOverride ?? Int(environment["BITMATCH_HARDWARE_APP_MIB"] ?? "256") ?? -1 + guard (32...4_096).contains(mib) else { throw PhysicalAppError.unsafeFixture } + var capacity = statfs() + guard statfs(source.path, &capacity) == 0, + UInt64(capacity.f_bavail) * UInt64(capacity.f_bsize) >= UInt64(mib + 1_024) * 1_048_576 else { throw PhysicalAppError.unsafeFixture } + try await Task.detached { + let file = source.appendingPathComponent("streaming-payload.bin") + let fd = Darwin.open(file.path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0o600) + guard fd >= 0 else { throw PhysicalAppError.posix(errno) } + let handle = FileHandle(fileDescriptor: fd, closeOnDealloc: true) + defer { try? handle.close() } + guard fcntl(fd, F_NOCACHE, 1) == 0 else { throw PhysicalAppError.posix(errno) } + for chunk in 0..café 日本語".utf8).write(to: source.appendingPathComponent("camera-sidecar.xml"), options: .withoutOverwriting) + try Data("legitimate hidden file".utf8).write(to: source.appendingPathComponent("._notes.txt"), options: .withoutOverwriting) + try await Self.movie(source.appendingPathComponent("preview.mov")) + }.value + } + + nonisolated private static func movie(_ url: URL) async throws { + let writer = try AVAssetWriter(outputURL: url, fileType: .mov) + let input = AVAssetWriterInput(mediaType: .video, outputSettings: [AVVideoCodecKey: AVVideoCodecType.h264, AVVideoWidthKey: 64, AVVideoHeightKey: 64]) + let adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input, + sourcePixelBufferAttributes: [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32ARGB]) + writer.add(input) + guard writer.startWriting() else { throw writer.error ?? PhysicalAppError.fixtureFailed } + writer.startSession(atSourceTime: .zero) + for index in 0..<2 { + var pixel: CVPixelBuffer? + guard CVPixelBufferCreate(kCFAllocatorDefault, 64, 64, kCVPixelFormatType_32ARGB, nil, &pixel) == kCVReturnSuccess, + let buffer = pixel else { throw PhysicalAppError.fixtureFailed } + CVPixelBufferLockBaseAddress(buffer, []) + let bytes = CVPixelBufferGetBaseAddress(buffer)!.assumingMemoryBound(to: UInt8.self) + let stride = CVPixelBufferGetBytesPerRow(buffer) + for y in 0..<64 { for x in 0..<64 { + let offset = y * stride + x * 4 + bytes[offset] = 255; bytes[offset + 1] = x < 32 ? 224 : 32 + bytes[offset + 2] = y < 32 ? 224 : 32; bytes[offset + 3] = index == 0 ? 64 : 192 + } } + CVPixelBufferUnlockBaseAddress(buffer, []) + guard await waitUntil(timeout: .seconds(10), { input.isReadyForMoreMediaData }), + adaptor.append(buffer, withPresentationTime: CMTime(value: Int64(index), timescale: 30)) else { throw PhysicalAppError.fixtureFailed } + } + input.markAsFinished(); await writer.finishWriting() + guard writer.status == .completed else { throw writer.error ?? PhysicalAppError.fixtureFailed } + } + + nonisolated private static func snapshot(_ root: URL, ignoreEvidence: Bool = false) throws -> [String: PhysicalAppFile] { + var result: [String: PhysicalAppFile] = [:] + for item in try CardSource.enumerateRegularFiles(base: root) { + if ignoreEvidence && item.relativePath.hasPrefix("ascmhl/") { continue } + if ignoreEvidence && TransferStatusMarker.hasExactName(item.relativePath) { + guard TransferStatusMarker.validatedStatus(at: item.url, relativePath: item.relativePath) == .filesVerified else { throw PhysicalAppError.workflowFailed } + continue + } + try Task.checkCancellation() + let fd = Darwin.open(item.url.path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC) + guard fd >= 0 else { throw PhysicalAppError.posix(errno) } + let handle = FileHandle(fileDescriptor: fd, closeOnDealloc: true) + defer { try? handle.close() } + guard fcntl(fd, F_NOCACHE, 1) == 0 else { throw PhysicalAppError.posix(errno) } + var sha = SHA256() + while try autoreleasepool(invoking: { + let bytes = try handle.read(upToCount: 1_048_576) ?? Data() + guard !bytes.isEmpty else { return false } + sha.update(data: bytes); return true + }) {} + result[item.relativePath] = .init(size: item.size, sha256: sha.finalize().map { String(format: "%02x", $0) }.joined()) + } + return result + } + nonisolated private static func cpuSeconds() throws -> Double { + var usage = rusage() + guard getrusage(RUSAGE_SELF, &usage) == 0 else { throw PhysicalAppError.posix(errno) } + return Double(usage.ru_utime.tv_sec + usage.ru_stime.tv_sec) + Double(usage.ru_utime.tv_usec + usage.ru_stime.tv_usec) / 1_000_000 + } + nonisolated private static func pdfText(_ data: Data) throws -> String { + guard let document = PDFDocument(data: data), document.pageCount > 0, let text = document.string, !text.isEmpty else { throw PhysicalAppError.invalidPDF } + return text + } + nonisolated private static func previewJPEGs(_ data: Data) throws -> [Data] { + guard let provider = CGDataProvider(data: data as CFData), let document = CGPDFDocument(provider) else { throw PhysicalAppError.invalidPDF } + let collector = PhysicalPreviewCollector() + for index in 1...document.numberOfPages { + guard let page = document.page(at: index), let pageDictionary = page.dictionary else { continue } + var resources: CGPDFDictionaryRef?; var images: CGPDFDictionaryRef? + guard CGPDFDictionaryGetDictionary(pageDictionary, "Resources", &resources), let resources, + CGPDFDictionaryGetDictionary(resources, "XObject", &images), let images else { continue } + CGPDFDictionaryApplyFunction(images, { _, object, context in + guard let context else { return } + let collector = Unmanaged.fromOpaque(context).takeUnretainedValue() + var stream: CGPDFStreamRef? + guard CGPDFObjectGetValue(object, .stream, &stream), let stream else { return } + guard let dictionary = CGPDFStreamGetDictionary(stream) else { return } + var subtype: UnsafePointer?; var width = 0; var height = 0 + guard CGPDFDictionaryGetName(dictionary, "Subtype", &subtype), let subtype, String(cString: subtype) == "Image", + CGPDFDictionaryGetInteger(dictionary, "Width", &width), CGPDFDictionaryGetInteger(dictionary, "Height", &height), + width == 64, height == 64 else { return } + var format = CGPDFDataFormat.raw + guard let bytes = CGPDFStreamCopyData(stream, &format), format == .jpegEncoded, + let source = CGImageSourceCreateWithData(bytes, nil), let image = CGImageSourceCreateImageAtIndex(source, 0, nil), + image.width == 64, image.height == 64 else { return } + // The generated clip has distinct red/green quadrants. Verify + // the decoded PDF image retains them, rather than accepting a + // same-sized placeholder or a blank image. + var pixels = [UInt8](repeating: 0, count: 64 * 64 * 4) + let matches = pixels.withUnsafeMutableBytes { storage -> Bool in + guard let address = storage.baseAddress, + let drawing = CGContext(data: address, width: 64, height: 64, bitsPerComponent: 8, + bytesPerRow: 64 * 4, space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return false } + drawing.draw(image, in: CGRect(x: 0, y: 0, width: 64, height: 64)) + let values = storage.bindMemory(to: UInt8.self) + let leftRed = Int(values[(16 * 64 + 16) * 4]) + let rightRed = Int(values[(16 * 64 + 48) * 4]) + let topGreen = Int(values[(16 * 64 + 16) * 4 + 1]) + let bottomGreen = Int(values[(48 * 64 + 16) * 4 + 1]) + return abs(leftRed - rightRed) > 100 && abs(topGreen - bottomGreen) > 100 + } + guard matches else { return } + collector.images.append(bytes as Data) + }, Unmanaged.passUnretained(collector).toOpaque()) + } + return collector.images + } +} + +private final class PhysicalPreviewCollector { var images: [Data] = [] } +private struct PhysicalAppFile: Equatable, Sendable { let size: Int64; let sha256: String } +private enum PhysicalAppError: Error { case unsafeFixture, fixtureFailed, workflowFailed, invalidPDF, posix(Int32) } +private struct PhysicalAppReceipt: Encodable { + struct Destination: Codable { let index: Int; let pipelineWrittenBytes: Int64; let pipelineReadbackBytes: Int64 } + let pair: Int; let mode: String; let sourceFiles: Int; let selectedFiles: Int; let sourceBytes: Int64 + /// End includes automatic reports + durable journal settlement. Independent + /// test hashes, Saved Check and History regeneration occur after this clock. + let queueSettledSeconds: Double + /// Engine return is before executor authoritative acceptance/MHL/reporting. + let pipelineReturnedSeconds: Double? + let cpuSeconds: Double; let initialMiB: Int; let peakMiB: Int + let telemetry: TransferPerformanceTelemetry; let destinations: [Destination] + let operatingSystem = ProcessInfo.processInfo.operatingSystemVersionString + let activeProcessors = ProcessInfo.processInfo.activeProcessorCount + let appVersion = Bundle.main.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String ?? "unknown" + let buildConfiguration = physicalAppBuildConfiguration + let benchmarkConditions = "Steady state, unique new destinations, same source, no cache purge. Engine default pipelined verification and worker count. Process CPU/footprint sampled; per-destination counters cover pipeline only. MHL bytes are logical aggregate work, with reread avoidance separately recorded. No OS/device-wide I/O attribution. Not an endurance or unplug test." +} + +#if DEBUG +private let physicalAppBuildConfiguration = "Debug app test host" +#else +private let physicalAppBuildConfiguration = "Release app test host" +#endif + +private final class PhysicalAppMeasurements: Sendable { let state = Mutex(PhysicalAppSamples()) } + +private struct PhysicalAppSamples: Sendable { + var initialMiB = 0; var peakMiB = 0 + var pipelineReturnedUptime: Double? + var io: TransferIOProgress? + mutating func sample() { + var info = task_vm_info_data_t() + var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) + let status = withUnsafeMutablePointer(to: &info) { pointer in + pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { + task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count) + } + } + guard status == KERN_SUCCESS else { return } + let value = Int(info.phys_footprint / 1_048_576) + if initialMiB == 0 { initialMiB = value }; peakMiB = max(peakMiB, value) + } +} + +private final class PhysicalAppOperations: FileOperationsService, @unchecked Sendable { + let samples: PhysicalAppMeasurements + private let pipeline = TransferPipeline(fileSystem: MacOSFileSystemService.shared, checksum: ChecksumEngine.shared) + init(samples: PhysicalAppMeasurements) { self.samples = samples } + func performFileOperation(sourceURL: URL, destinationURLs: [URL], verificationMode: VerificationMode, settings: CameraLabelSettings, + estimatedTotalBytes: Int64?, progressCallback: @escaping ProgressCallback, onFileResult: FileResultCallback?) async throws -> FileOperation { + try await performFileOperation(sourceURL: sourceURL, destinationURLs: destinationURLs, verificationMode: verificationMode, + settings: settings, estimatedTotalBytes: estimatedTotalBytes, progressCallback: progressCallback, ioProgressCallback: { _ in }, onFileResult: onFileResult) + } + func performFileOperation(sourceURL: URL, destinationURLs: [URL], verificationMode: VerificationMode, settings: CameraLabelSettings, + estimatedTotalBytes: Int64?, progressCallback: @escaping ProgressCallback, ioProgressCallback: @escaping IOProgressCallback, + onFileResult: FileResultCallback?) async throws -> FileOperation { + let samples = self.samples + let operation = try await pipeline.performFileOperation(sourceURL: sourceURL, destinationURLs: destinationURLs, verificationMode: verificationMode, + settings: settings, estimatedTotalBytes: estimatedTotalBytes, progressCallback: progressCallback, ioProgressCallback: { update in + samples.state.withLock { if update.sequence > ($0.io?.sequence ?? 0) { $0.io = update } } + ioProgressCallback(update) + }, onFileResult: onFileResult) + samples.state.withLock { $0.pipelineReturnedUptime = ProcessInfo.processInfo.systemUptime } + return operation + } + func cancelOperation() { pipeline.cancelOperation() } + func pauseOperation() async { await pipeline.pauseOperation() } + func resumeOperation() async { await pipeline.resumeOperation() } +} + +private final class PhysicalAppPlatform: PlatformManager { + private let real = MacOSPlatformManager.shared + nonisolated let fileOperations: FileOperationsService + init(fileOperations: FileOperationsService) { self.fileOperations = fileOperations } + nonisolated var fileSystem: FileSystemService { real.fileSystem } + nonisolated var checksum: ChecksumService { real.checksum } + nonisolated var cameraDetection: CameraDetectionService { real.cameraDetection } + nonisolated var supportsDragAndDrop: Bool { real.supportsDragAndDrop } + func presentAlert(title: String, message: String) async {} + func presentError(_ error: Error) async {} + func openURL(_ url: URL) async -> Bool { false } +} + +/// Pins only pre-authorized roots. Every child is created exclusively, tracked +/// by device/inode and removed descriptor-relatively without following links. +private final class PhysicalAppFixture { + struct Volume: Codable, Equatable { + let nonce: String; let root: String; let mount: String; let volumeUUID: String + let filesystem: String; let initialFreeBytes: Int64 + } + struct Config: Codable { let nonce: String; let volumes: [String: Volume] } + private final class Root { + let volume: Volume; let descriptor: Int32; let device: dev_t; let inode: ino_t + init(_ volume: Volume) throws { + self.volume = volume + let fd = Darwin.open(volume.root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC) + guard fd >= 0 else { throw PhysicalAppError.unsafeFixture } + var identity = stat() + guard fstat(fd, &identity) == 0 else { Darwin.close(fd); throw PhysicalAppError.unsafeFixture } + descriptor = fd; device = identity.st_dev; inode = identity.st_ino + } + deinit { Darwin.close(descriptor) } + } + private struct Child { let role: String; let name: String; let device: dev_t; let inode: ino_t } + private let nonce: String + private let roots: [String: Root] + private var children: [String: Child] = [:] + private init(nonce: String, roots: [String: Root]) { self.nonce = nonce; self.roots = roots } + + static func load(environment env: [String: String] = ProcessInfo.processInfo.environment) throws -> PhysicalAppFixture { + guard env["BITMATCH_HARDWARE_APP_WORKFLOW"] == "1", let path = env["BITMATCH_HARDWARE_CONFIG"] else { + throw XCTSkip("Physical app workflow requires explicit BITMATCH_HARDWARE_APP_WORKFLOW=1 and a marked config") + } + let config = try JSONDecoder().decode(Config.self, from: Data(contentsOf: URL(fileURLWithPath: path))) + guard UUID(uuidString: config.nonce) != nil else { throw PhysicalAppError.unsafeFixture } + var roots: [String: Root] = [:] + for role in ["sd", "ssdA", "ssdB"] { + guard let volume = config.volumes[role], volume.nonce == config.nonce else { throw PhysicalAppError.unsafeFixture } + let root = URL(fileURLWithPath: volume.root).standardizedFileURL + let mount = URL(fileURLWithPath: volume.mount).standardizedFileURL + guard root.path == volume.root, mount.path == volume.mount, + mount.deletingLastPathComponent().path == "/Volumes", root.deletingLastPathComponent() == mount, + root.lastPathComponent.hasPrefix("BitMatch-validation-"), root.lastPathComponent.contains(config.nonce) else { throw PhysicalAppError.unsafeFixture } + let pinned = try Root(volume) + roots[role] = pinned + } + guard Set(roots.values.map { $0.volume.volumeUUID.lowercased() }).count == 3, + Set(roots.values.map(\.device)).count == 3 else { throw PhysicalAppError.unsafeFixture } + let fixture = PhysicalAppFixture(nonce: config.nonce, roots: roots) + for role in roots.keys { try fixture.validate(role) } + return fixture + } + + private func validate(_ role: String) throws { + guard let root = roots[role], root.volume.nonce == nonce else { throw PhysicalAppError.unsafeFixture } + var named = stat(); var pinned = stat() + guard lstat(root.volume.root, &named) == 0, fstat(root.descriptor, &pinned) == 0, + (named.st_mode & S_IFMT) == S_IFDIR, named.st_dev == root.device, named.st_ino == root.inode, + pinned.st_dev == root.device, pinned.st_ino == root.inode else { throw PhysicalAppError.unsafeFixture } + let url = URL(fileURLWithPath: root.volume.root) + let properties = try url.resourceValues(forKeys: [.volumeUUIDStringKey]) + guard properties.volumeUUIDString?.caseInsensitiveCompare(root.volume.volumeUUID) == .orderedSame else { throw PhysicalAppError.unsafeFixture } + let markerFD = openat(root.descriptor, ".bitmatch-validation-owned.json", O_RDONLY | O_NOFOLLOW | O_CLOEXEC) + guard markerFD >= 0 else { throw PhysicalAppError.unsafeFixture } + let handle = FileHandle(fileDescriptor: markerFD, closeOnDealloc: true) + defer { try? handle.close() } + var markerStat = stat() + guard fstat(markerFD, &markerStat) == 0, (markerStat.st_mode & S_IFMT) == S_IFREG, + markerStat.st_size > 0, markerStat.st_size <= 8_192 else { throw PhysicalAppError.unsafeFixture } + let data = try handle.read(upToCount: 8_193) ?? Data() + guard data.count <= 8_192, try JSONDecoder().decode(Volume.self, from: data) == root.volume else { throw PhysicalAppError.unsafeFixture } + var fs = statfs() + guard fstatfs(root.descriptor, &fs) == 0 else { throw PhysicalAppError.unsafeFixture } + let actual = withUnsafeBytes(of: fs.f_fstypename) { String(decoding: $0.prefix { $0 != 0 }, as: UTF8.self) } + let mount = withUnsafeBytes(of: fs.f_mntonname) { String(decoding: $0.prefix { $0 != 0 }, as: UTF8.self) } + guard mount == root.volume.mount else { throw PhysicalAppError.unsafeFixture } + let expected = root.volume.filesystem.lowercased() + guard actual == expected || (actual == "hfs" && expected.contains("hfs")) || (actual == "msdos" && expected.contains("fat")) else { throw PhysicalAppError.unsafeFixture } + } + + func child(role: String, name: String) throws -> URL { + try validate(role) + guard name.hasPrefix("app-"), !name.contains("/"), !name.contains("\u{0}"), let root = roots[role] else { throw PhysicalAppError.unsafeFixture } + guard mkdirat(root.descriptor, name, 0o700) == 0 else { throw PhysicalAppError.posix(errno) } + var identity = stat() + guard fstatat(root.descriptor, name, &identity, AT_SYMLINK_NOFOLLOW) == 0, + (identity.st_mode & S_IFMT) == S_IFDIR, identity.st_dev == root.device else { throw PhysicalAppError.unsafeFixture } + let path = URL(fileURLWithPath: root.volume.root).appendingPathComponent(name) + children[path.path] = Child(role: role, name: name, device: identity.st_dev, inode: identity.st_ino) + return path + } + + func removeChild(_ path: URL) throws { + guard let child = children[path.path], let root = roots[child.role], + path.deletingLastPathComponent().path == root.volume.root else { throw PhysicalAppError.unsafeFixture } + try validate(child.role) + try Self.removeTree(parent: root.descriptor, name: child.name, device: child.device, inode: child.inode) + children.removeValue(forKey: path.path) + } + + fileprivate static func removeTree(parent: Int32, name: String, device: dev_t, inode: ino_t) throws { + let fd = openat(parent, name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC) + guard fd >= 0 else { throw PhysicalAppError.unsafeFixture } + defer { Darwin.close(fd) } + var identity = stat() + guard fstat(fd, &identity) == 0, identity.st_dev == device, identity.st_ino == inode else { throw PhysicalAppError.unsafeFixture } + let duplicate = dup(fd) + guard duplicate >= 0 else { throw PhysicalAppError.posix(errno) } + guard let directory = fdopendir(duplicate) else { Darwin.close(duplicate); throw PhysicalAppError.posix(errno) } + defer { closedir(directory) } + errno = 0 + while let entry = readdir(directory) { + let child = withUnsafePointer(to: &entry.pointee.d_name) { + $0.withMemoryRebound(to: CChar.self, capacity: Int(entry.pointee.d_namlen) + 1) { String(cString: $0) } + } + if child == "." || child == ".." { continue } + var info = stat() + guard fstatat(fd, child, &info, AT_SYMLINK_NOFOLLOW) == 0, info.st_dev == device else { throw PhysicalAppError.unsafeFixture } + if (info.st_mode & S_IFMT) == S_IFDIR { + try removeTree(parent: fd, name: child, device: device, inode: info.st_ino) + } else { + guard unlinkat(fd, child, 0) == 0 else { throw PhysicalAppError.posix(errno) } + } + errno = 0 + } + guard errno == 0 else { throw PhysicalAppError.posix(errno) } + var named = stat() + guard fstatat(parent, name, &named, AT_SYMLINK_NOFOLLOW) == 0, + named.st_dev == device, named.st_ino == inode else { throw PhysicalAppError.unsafeFixture } + guard unlinkat(parent, name, AT_REMOVEDIR) == 0 else { throw PhysicalAppError.posix(errno) } + } +} +#endif diff --git a/docs/HARDWARE_TESTING.md b/docs/HARDWARE_TESTING.md index 6cabd511..48353532 100644 --- a/docs/HARDWARE_TESTING.md +++ b/docs/HARDWARE_TESTING.md @@ -66,6 +66,16 @@ To reuse an existing build directory, set `BITMATCH_DERIVED_DATA_PATH` to an abs Before sharing evidence, review it for local paths, usernames, device identifiers, and private filenames. Keep full raw records privately and publish a redacted copy. Record the revision and whether the working tree contained changes; a revision alone does not identify uncommitted source changes. +### Physical app workflow and paired measurements + +`PhysicalWorkflowEvidenceTests` is an opt-in Mac app test over an already authorized, owner-marked configuration. Its `sd`, `ssdA`, and `ssdB` roles must identify three separate mounted volumes. It creates unique children only, copies a generated playable clip and streaming payload through the real coordinator, then checks MHL, automatic reports, durable History, regenerated reports, embedded preview pixels, and independent destination hashes. It does not format or eject a drive. Failed fixtures remain for inspection; cleanup refuses changed identities and never follows links. + +After reserving the hardware and Mac app test slot, supply `TEST_RUNNER_BITMATCH_HARDWARE_APP_WORKFLOW=1`, `TEST_RUNNER_BITMATCH_HARDWARE_CONFIG` (the marked config JSON), and `TEST_RUNNER_BITMATCH_HARDWARE_APP_ARTIFACTS` (a private directory on the host) to `xcodebuild test -scheme BitMatch -destination 'platform=macOS' -jobs 4 -parallel-testing-enabled NO -only-testing:BitMatchTests/PhysicalWorkflowEvidenceTests/testPhysicalCardToTwoBackupsAndSavedEvidence`. Normal test runs skip this case. + +`TEST_RUNNER_BITMATCH_HARDWARE_APP_BENCHMARK=1` adds three Standard/Thorough pairs with alternating order and fresh destination folders. The payload defaults to 256 MiB; `TEST_RUNNER_BITMATCH_HARDWARE_APP_MIB` accepts 32–4096 MiB. The private receipt records process CPU/physical footprint, time through automatic reports and journal settlement, pipeline return, phase durations/overlap, and per-destination pipeline bytes. Saved Check and independent test hashes happen after the measured transfer. These are steady-state measurements without cache purging or device-wide I/O attribution; they are not a cable-pull test or a general performance promise. + +Before touching hardware, `TEST_RUNNER_BITMATCH_PHYSICAL_APP_HOST_SMOKE=1` runs the same evidence assertions on small host scratch folders through `testHostEvidenceHarnessSmoke`. Two ordinary tests also check that invalid configuration leaves existing files untouched and descriptor cleanup preserves an outside symlink target and a replacement directory. + ## Recording and publishing physical results Use the [hardware report template](HARDWARE_REPORT_TEMPLATE.md) for each configuration and submit it through the repository's **Hardware test report** issue form. Keep outcomes separate for baseline copy, source removal, one-destination removal, sleep, and cancellation. Mark cases you did not run as **Not tested**. A completed transfer without independent destination hashes is **Inconclusive**, not a verified pass. From df43d6f28e96864e9702afc27211061ad621947a Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 22:04:02 -0400 Subject: [PATCH 14/56] Require requested matrix tests to execute --- Scripts/test-full-transfer-matrix.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Scripts/test-full-transfer-matrix.sh b/Scripts/test-full-transfer-matrix.sh index 0274c118..2935029a 100755 --- a/Scripts/test-full-transfer-matrix.sh +++ b/Scripts/test-full-transfer-matrix.sh @@ -40,9 +40,12 @@ for role in source destination; do done done TESTS=("-only-testing:BitMatchTests/CopyVerifyExecutorIntegrityTests/$TEST_NAME") +EXPECTED_TESTS=1 if [[ "${MATRIX_FOCUS:-full}" == evidence ]]; then TESTS=("-only-testing:BitMatchTests/MountedWorkflowEvidenceMatrixTests") + EXPECTED_TESTS=3 elif [[ "${MATRIX_FOCUS:-full}" == all ]]; then + EXPECTED_TESTS=8 TESTS=( "-only-testing:BitMatchTests/CopyVerifyExecutorIntegrityTests/testFullSyntheticFilesystemMatrix" "-only-testing:BitMatchTests/CopyVerifyExecutorIntegrityTests/testCameraRenameFilesystemMatrix" @@ -58,4 +61,13 @@ TEST_RUNNER_BITMATCH_MATRIX_ROOT="$PAIR" \ xcodebuild -project "$ROOT/BitMatch.xcodeproj" -derivedDataPath "${MATRIX_DERIVED_DATA:-$ROOT/.derived-data/mac-test}" \ CODE_SIGNING_ALLOWED=NO test -scheme BitMatch -destination 'platform=macOS' \ -jobs 4 -parallel-testing-enabled NO \ - "${TESTS[@]}" + "${TESTS[@]}" | awk -v expected="$EXPECTED_TESTS" ' + { print } + /^Test Case .* passed \(/ { passed++ } + END { + if (passed != expected) { + printf "Matrix execution incomplete: expected %d passing XCTest cases, observed %d.\n", expected, passed > "/dev/stderr" + exit 1 + } + printf "MATRIX_EXECUTION_RECEIPT expectedTests=%d passedTests=%d\n", expected, passed + }' From 888ae459749a3797dc953cd84d869bfd02a22e76 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 22:14:04 -0400 Subject: [PATCH 15/56] Keep tall Mac Settings panes fully reachable --- BitMatch/Views/PreferencesWindow.swift | 26 ++++++---- ...ortableTransferSettingsSnapshotTests.swift | 50 ++++++++++++++++++- CHANGELOG.md | 1 + 3 files changed, 65 insertions(+), 12 deletions(-) diff --git a/BitMatch/Views/PreferencesWindow.swift b/BitMatch/Views/PreferencesWindow.swift index 867f010d..b071ade9 100644 --- a/BitMatch/Views/PreferencesWindow.swift +++ b/BitMatch/Views/PreferencesWindow.swift @@ -95,17 +95,23 @@ struct PreferencesWindow: View { } private func settingsPane(_ content: Content, pane: PreferencesPane) -> some View { - content - .frame(width: 680, alignment: .topLeading) - .fixedSize(horizontal: false, vertical: true) - .background { - GeometryReader { proxy in - Color.clear.preference( - key: SettingsPaneHeightKey.self, - value: [pane.rawValue: proxy.size.height] - ) + // Measure the complete form for window sizing, but give capped panes + // a real viewport. A fixed-height TabView otherwise centers and clips + // an intrinsically taller form, leaving its first/last rows unreachable. + ScrollView { + content + .frame(width: 680, alignment: .topLeading) + .fixedSize(horizontal: false, vertical: true) + .background { + GeometryReader { proxy in + Color.clear.preference( + key: SettingsPaneHeightKey.self, + value: [pane.rawValue: proxy.size.height] + ) + } } - } + } + .frame(width: 680, alignment: .topLeading) } // MARK: - General diff --git a/BitMatchTests/PortableTransferSettingsSnapshotTests.swift b/BitMatchTests/PortableTransferSettingsSnapshotTests.swift index 38a0bfaa..0b3894c8 100644 --- a/BitMatchTests/PortableTransferSettingsSnapshotTests.swift +++ b/BitMatchTests/PortableTransferSettingsSnapshotTests.swift @@ -28,7 +28,15 @@ final class PortableTransferSettingsSnapshotTests: XCTestCase { } try capture(PreferencesWindow(coordinator: app, cameraAutoSource: companions.cameraAutoSource, remoteBackups: companions.remoteBackups, updater: updater), size: CGSize(width: 680, height: 720), - name: "mac-transfer-settings", expected: ["transfer settings", "import settings", "export settings"]) + name: "mac-transfer-settings", expected: ["notifications", "notify when a card needs attention"], + scrolledExpected: ["transfer settings", "import settings", "export settings", "updates", "check now"], + exactHeading: "notifications") + UserDefaults.standard.set("Reports", forKey: "BitMatchSelectedSettingsPane") + try capture(PreferencesWindow(coordinator: app, cameraAutoSource: companions.cameraAutoSource, + remoteBackups: companions.remoteBackups, updater: updater), size: CGSize(width: 680, height: 720), + name: "mac-report-settings", expected: ["reports", "create pdf"], + scrolledExpected: ["include clip thumbnails in pdfs", "unavailable clips stay listed"], + exactHeading: "reports", allowFittingContent: true) let current = app.portableTransferSettings let proposed = PortableTransferSettings(verificationMode: .quick, generateASCMHL: !current.generateASCMHL, ignoreAppleDouble: !current.ignoreAppleDouble, @@ -38,7 +46,8 @@ final class PortableTransferSettingsSnapshotTests: XCTestCase { size: CGSize(width: 480, height: 600), name: "mac-transfer-settings-preview", expected: ["verification", "quick"]) } - private func capture(_ view: V, size: CGSize, name: String, expected: [String]) throws { + private func capture(_ view: V, size: CGSize, name: String, expected: [String], + scrolledExpected: [String]? = nil, exactHeading: String? = nil, allowFittingContent: Bool = false) throws { let host = NSHostingView(rootView: view.frame(width: size.width, height: size.height) .preferredColorScheme(.dark).transaction { $0.disablesAnimations = true }) let window = NSWindow(contentRect: CGRect(origin: .zero, size: size), styleMask: [.titled], backing: .buffered, defer: true) @@ -52,6 +61,38 @@ final class PortableTransferSettingsSnapshotTests: XCTestCase { window.layoutIfNeeded() host.layoutSubtreeIfNeeded() host.displayIfNeeded() + try attach(host, name: name, expected: expected, exactHeading: exactHeading) + if let scrolledExpected { + guard let scroll = findOverflowingScrollView(in: host) else { + XCTAssertTrue(allowFittingContent, "Tall Settings content must have a real scroll viewport") + try attach(host, name: name + "-fits", expected: scrolledExpected) + return + } + let document = try XCTUnwrap(scroll.documentView) + let bottom = document.isFlipped ? max(0, document.bounds.height - scroll.contentView.bounds.height) : 0 + scroll.contentView.scroll(to: CGPoint(x: 0, y: bottom)) + scroll.reflectScrolledClipView(scroll.contentView) + RunLoop.main.run(until: Date().addingTimeInterval(0.2)) + host.layoutSubtreeIfNeeded() + try attach(host, name: name + "-scrolled", expected: scrolledExpected) + let top = document.isFlipped ? 0 : max(0, document.bounds.height - scroll.contentView.bounds.height) + scroll.contentView.scroll(to: CGPoint(x: 0, y: top)) + scroll.reflectScrolledClipView(scroll.contentView) + RunLoop.main.run(until: Date().addingTimeInterval(0.2)) + host.layoutSubtreeIfNeeded() + try attach(host, name: name + "-top-restored", expected: expected, exactHeading: exactHeading) + } + } + + private func findOverflowingScrollView(in view: NSView) -> NSScrollView? { + if let scroll = view as? NSScrollView, !scroll.isHiddenOrHasHiddenAncestor, + let document = scroll.documentView, document.bounds.height > scroll.contentView.bounds.height + 1 { + return scroll + } + return view.subviews.lazy.compactMap { self.findOverflowingScrollView(in: $0) }.first + } + + private func attach(_ host: NSView, name: String, expected: [String], exactHeading: String? = nil) throws { let bitmap = try XCTUnwrap(host.bitmapImageRepForCachingDisplay(in: host.bounds)) host.cacheDisplay(in: host.bounds, to: bitmap) let data = try XCTUnwrap(bitmap.representation(using: .png, properties: [:])) @@ -60,6 +101,11 @@ final class PortableTransferSettingsSnapshotTests: XCTestCase { try VNImageRequestHandler(cgImage: try XCTUnwrap(bitmap.cgImage)).perform([request]) let text = (request.results ?? []).compactMap { $0.topCandidates(1).first?.string }.joined(separator: " ").lowercased() for value in expected { XCTAssertTrue(text.contains(value), "\(name) missing \(value): \(text)") } + if let exactHeading { + let heading = (request.results ?? []).first { $0.topCandidates(1).first?.string.lowercased() == exactHeading } + XCTAssertNotNil(heading, "\(name) must show its complete \(exactHeading) heading") + if let heading { XCTAssertLessThan(heading.boundingBox.maxY, 0.995, "Heading must sit inside the viewport") } + } let attachment = XCTAttachment(data: data, uniformTypeIdentifier: "public.png") attachment.name = name attachment.lifetime = .keepAlways diff --git a/CHANGELOG.md b/CHANGELOG.md index ba4933cd..80f683bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +- Mac Settings now scroll when a pane is taller than the window, keeping the first and last controls reachable. - Move reusable transfer settings between Mac, iPad and iPhone from Settings. Imports show the changes before Apply and leave backup locations, camera labels, private report details and saved transfers alone. - Older History entries without a saved selected-file plan now stay visible but do not claim the whole card is safe to erase, even if their saved rows say verified. They also no longer trigger the “Backed up before…” hint. Quick entries without that plan show Needs attention instead of a clean size-only copy. History exports keep the original message and add the saved safety verdict. - If a new transfer cannot save its selected-file plan, BitMatch stops before creating a completion report or marking a project card safe. The final record keeps the source unsafe. Finish, queue, menu bar, Dock and notifications follow the saved verdict. A later History-save failure also shows an interruption; any separately saved report or project evidence remains available for review. From ae5eeb9dc3d549cabef445a8b84f219463e8c80b Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 22:19:03 -0400 Subject: [PATCH 16/56] Retain workflow fixtures when assertions or settlement fail --- .../PhysicalWorkflowEvidenceTests.swift | 145 ++++++++++++++---- 1 file changed, 117 insertions(+), 28 deletions(-) diff --git a/BitMatchTests/PhysicalWorkflowEvidenceTests.swift b/BitMatchTests/PhysicalWorkflowEvidenceTests.swift index 209d3a9d..e938901c 100644 --- a/BitMatchTests/PhysicalWorkflowEvidenceTests.swift +++ b/BitMatchTests/PhysicalWorkflowEvidenceTests.swift @@ -29,17 +29,61 @@ final class PhysicalWorkflowEvidenceTests: XCTestCase { let targets = [root.appendingPathComponent("Backup A"), root.appendingPathComponent("Backup B")] let evidence = root.appendingPathComponent("evidence") for url in [source, evidence] + targets { try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) } - defer { try? FileManager.default.removeItem(at: root) } - try await seed(source, mibOverride: 32) - let original = try await Task.detached { try Self.snapshot(source) }.value - for mode in [VerificationMode.standard, .thorough] { - let destinations = try targets.map { target in - let destination = target.appendingPathComponent(mode.rawValue) - try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: false) - return destination + do { + try await seed(source, mibOverride: 32) + let original = try await Task.detached { try Self.snapshot(source) }.value + for mode in [VerificationMode.standard, .thorough] { + let destinations = try targets.map { target in + let destination = target.appendingPathComponent(mode.rawValue) + try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: false) + return destination + } + _ = try await run(source: source, original: original, targets: destinations, mode: mode, pair: 1, evidence: evidence) + } + try PhysicalAppHarnessSettlement.withPassingAssertions(testRun?.totalFailureCount) { + try FileManager.default.removeItem(at: root) + print("PHYSICAL_APP_HOST_SUMMARY transfers=2 cleanup=true") + } + } catch { + print("PHYSICAL_APP_HOST_FAILURE retainedFixture=\(root.path)") + throw error + } + } + + func testAssertionFailurePreventsSuccessfulReceiptAndFixtureCleanup() throws { + for failures: Int? in [1, 2, nil] { + var receiptWritten = false + var fixtureDeleted = false + XCTAssertThrowsError(try PhysicalAppHarnessSettlement.withPassingAssertions(failures) { + receiptWritten = true + fixtureDeleted = true + }) + XCTAssertFalse(receiptWritten) + XCTAssertFalse(fixtureDeleted) + } + var receiptWritten = false + var fixtureDeleted = false + try PhysicalAppHarnessSettlement.withPassingAssertions(0) { + receiptWritten = true + fixtureDeleted = true + } + XCTAssertTrue(receiptWritten) + XCTAssertTrue(fixtureDeleted) + } + + func testCooperativeSettlementCancelsAndDrainsItsWaiter() async { + var cancelledWaiter = false + let timedOut = await PhysicalAppHarnessSettlement.waitForCooperativeSettlement(.milliseconds(10)) { + do { try await Task.sleep(for: .seconds(60)) } + catch { + cancelledWaiter = error is CancellationError + throw error } - _ = try await run(source: source, original: original, targets: destinations, mode: mode, pair: 1, evidence: evidence) } + XCTAssertFalse(timedOut) + XCTAssertTrue(cancelledWaiter) + let settled = await PhysicalAppHarnessSettlement.waitForCooperativeSettlement(.seconds(10)) {} + XCTAssertTrue(settled) } func testPhysicalConfigRefusesUnmarkedHostDirectoriesWithoutWrites() throws { @@ -93,6 +137,9 @@ final class PhysicalWorkflowEvidenceTests: XCTestCase { do { try await seed(source) let original = try await Task.detached { try Self.snapshot(source) }.value + let manifestEncoder = JSONEncoder() + manifestEncoder.outputFormatting = [.sortedKeys] + try manifestEncoder.encode(original).write(to: evidence.appendingPathComponent("source-manifest.json"), options: .atomic) let benchmark = ProcessInfo.processInfo.environment["BITMATCH_HARDWARE_APP_BENCHMARK"] == "1" let trials: [[VerificationMode]] = benchmark ? [[.standard, .thorough], [.thorough, .standard], [.standard, .thorough]] : [[.standard]] @@ -104,16 +151,20 @@ final class PhysicalWorkflowEvidenceTests: XCTestCase { fixture.child(role: "ssdB", name: "app-\(label)")] let receipt = try await run(source: source, original: original, targets: targets, mode: mode, pair: pair + 1, evidence: evidence) - receipts.append(receipt) - try JSONEncoder().encode(receipts).write(to: evidence.appendingPathComponent("trials.json"), options: .atomic) - try fixture.removeChild(targets[0]); try fixture.removeChild(targets[1]) + try PhysicalAppHarnessSettlement.withPassingAssertions(testRun?.totalFailureCount) { + receipts.append(receipt) + try JSONEncoder().encode(receipts).write(to: evidence.appendingPathComponent("trials.json"), options: .atomic) + try fixture.removeChild(targets[0]); try fixture.removeChild(targets[1]) + } } } XCTAssertEqual(receipts.count, benchmark ? 6 : 1) let after = try await Task.detached { try Self.snapshot(source) }.value XCTAssertEqual(after, original, "Physical source changed") - try fixture.removeChild(source) - print("PHYSICAL_APP_SUMMARY trials=\(receipts.count) pairedBenchmark=\(benchmark) backups=2 realJournal=true independentHashes=true previewsEmbedded=true cleanup=true") + try PhysicalAppHarnessSettlement.withPassingAssertions(testRun?.totalFailureCount) { + try fixture.removeChild(source) + print("PHYSICAL_APP_SUMMARY trials=\(receipts.count) pairedBenchmark=\(benchmark) backups=2 realJournal=true independentHashes=true previewsEmbedded=true cleanup=true") + } } catch { // Keep owned children on failure for review; never delete a replaced // or disconnected fixture by falling back to a pathname operation. @@ -156,12 +207,19 @@ final class PhysicalWorkflowEvidenceTests: XCTestCase { let ended = await waitUntil(timeout: .seconds(900)) { !coordinator.queueIsRunning && !coordinator.isOperationInProgress } - if !ended { coordinator.cancelOperation() } + if !ended { + let settled = await PhysicalAppHarnessSettlement.waitForCooperativeSettlement(.seconds(30)) { + // The public enum has no test-harness origin. Record the real + // caller below; .user here means an explicit cancellation API + // request, not evidence that a person clicked Cancel. + try await coordinator.cancelOperationAndWaitForSettlement(origin: .user) + } + print("PHYSICAL_APP_TIMEOUT cancellationRequestedBy=harness originAPI=user executorSettled=\(settled) fixturesRetained=true") + throw PhysicalAppError.workflowTimedOut(settled: settled) + } let queueSettled = ProcessInfo.processInfo.systemUptime - started let cpu = try Self.cpuSeconds() - cpuStart monitor.cancel(); await monitor.value - XCTAssertTrue(ended, "Physical app workflow timed out") - guard ended else { throw PhysicalAppError.workflowFailed } let records = try JSONDecoder().decode([LocalTransferRecord].self, from: Data(contentsOf: journalURL)) let record = try XCTUnwrap(records.first { $0.id == id }) XCTAssertEqual(record.state, excluded.isEmpty ? .completed : .issues, record.summary) @@ -248,15 +306,17 @@ final class PhysicalWorkflowEvidenceTests: XCTestCase { try master.pdfData.write(to: evidence.appendingPathComponent("\(prefix)-master.pdf")) let after = try await Task.detached { try Self.snapshot(source) }.value XCTAssertEqual(after, original, "Source changed after reports/check") - let receipt = PhysicalAppReceipt(pair: pair, mode: mode.rawValue, sourceFiles: original.count, - selectedFiles: selected.count, sourceBytes: totalBytes, queueSettledSeconds: queueSettled, - pipelineReturnedSeconds: pipelineSamples.pipelineReturnedUptime.map { $0 - started }, - cpuSeconds: cpu, initialMiB: pipelineSamples.initialMiB, peakMiB: pipelineSamples.peakMiB, - telemetry: telemetry, destinations: io.destinations.enumerated().map { - .init(index: $0.offset, pipelineWrittenBytes: $0.element.writtenBytes, pipelineReadbackBytes: $0.element.readbackBytes) - }) - print("PHYSICAL_APP_TRIAL pair=\(pair) mode=\(mode.rawValue) sourceFiles=\(original.count) selectedFiles=\(selected.count) bytes=\(totalBytes) queueSettledSeconds=\(queueSettled) cpuSeconds=\(cpu) initialMiB=\(receipt.initialMiB) peakMiB=\(receipt.peakMiB) previews=embedded history=real") - return receipt + return try PhysicalAppHarnessSettlement.withPassingAssertions(testRun?.totalFailureCount) { + let receipt = PhysicalAppReceipt(pair: pair, mode: mode.rawValue, sourceFiles: original.count, + selectedFiles: selected.count, sourceBytes: totalBytes, queueSettledSeconds: queueSettled, + pipelineReturnedSeconds: pipelineSamples.pipelineReturnedUptime.map { $0 - started }, + cpuSeconds: cpu, initialMiB: pipelineSamples.initialMiB, peakMiB: pipelineSamples.peakMiB, + telemetry: telemetry, destinations: io.destinations.enumerated().map { + .init(index: $0.offset, pipelineWrittenBytes: $0.element.writtenBytes, pipelineReadbackBytes: $0.element.readbackBytes) + }) + print("PHYSICAL_APP_TRIAL pair=\(pair) mode=\(mode.rawValue) sourceFiles=\(original.count) selectedFiles=\(selected.count) bytes=\(totalBytes) queueSettledSeconds=\(queueSettled) cpuSeconds=\(cpu) initialMiB=\(receipt.initialMiB) peakMiB=\(receipt.peakMiB) previews=embedded history=real") + return receipt + } } private func evidenceDirectory() throws -> URL { @@ -403,8 +463,37 @@ final class PhysicalWorkflowEvidenceTests: XCTestCase { } private final class PhysicalPreviewCollector { var images: [Data] = [] } -private struct PhysicalAppFile: Equatable, Sendable { let size: Int64; let sha256: String } -private enum PhysicalAppError: Error { case unsafeFixture, fixtureFailed, workflowFailed, invalidPDF, posix(Int32) } +private struct PhysicalAppFile: Codable, Equatable, Sendable { let size: Int64; let sha256: String } +private enum PhysicalAppError: Error { case unsafeFixture, fixtureFailed, workflowFailed, workflowTimedOut(settled: Bool), invalidPDF, posix(Int32) } + +/// XCTest assertions record failures without throwing. A failing or missing +/// test context must never authorize a successful receipt or fixture removal. +private enum PhysicalAppHarnessSettlement { + static func withPassingAssertions(_ failureCount: Int?, operation: () throws -> Result) throws -> Result { + guard failureCount == 0 else { throw PhysicalAppError.workflowFailed } + return try operation() + } + + /// The deadline cancels a cooperative waiter. Structured draining cannot + /// force a synchronous filesystem call or a task that ignores cancellation. + @MainActor + static func waitForCooperativeSettlement(_ timeout: Duration, operation: @escaping @MainActor @Sendable () async throws -> Void) async -> Bool { + await withTaskGroup(of: Bool.self) { group in + group.addTask { + do { try await operation(); return true } + catch { return false } + } + group.addTask { + do { try await Task.sleep(for: timeout) } + catch { return false } + return false + } + let settled = await group.next() ?? false + group.cancelAll() + return settled + } + } +} private struct PhysicalAppReceipt: Encodable { struct Destination: Codable { let index: Int; let pipelineWrittenBytes: Int64; let pipelineReadbackBytes: Int64 } let pair: Int; let mode: String; let sourceFiles: Int; let selectedFiles: Int; let sourceBytes: Int64 From e482faf55535acc910123359f67bd174c7638276 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 22:22:13 -0400 Subject: [PATCH 17/56] Resolve selected folders to their physical backup disk --- .../File/BackupIndependencePolicy.swift | 9 +- .../File/DriveConnectionInfo.swift | 19 +-- .../File/SelectedVolumeDiskResolver.swift | 19 +++ .../DriveConnectionInfoTests.swift | 50 -------- .../PhysicalDiskIdentityFolderTests.swift | 118 ++++++++++++++++++ .../SelectedVolumeDiskResolverTests.swift | 54 ++++++++ 6 files changed, 200 insertions(+), 69 deletions(-) create mode 100644 Packages/BitMatchEngine/Sources/BitMatchEngine/File/SelectedVolumeDiskResolver.swift create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/PhysicalDiskIdentityFolderTests.swift create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/SelectedVolumeDiskResolverTests.swift diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/BackupIndependencePolicy.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/BackupIndependencePolicy.swift index 43442f4f..9b1607b1 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/BackupIndependencePolicy.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/BackupIndependencePolicy.swift @@ -14,8 +14,15 @@ public struct SystemPhysicalDiskIdentityProvider: PhysicalDiskIdentityProviding public func physicalDiskIdentity(for url: URL) -> String? { #if os(macOS) + guard url.isFileURL, !Task.isCancelled else { return nil } + let scoped = url.startAccessingSecurityScopedResource() + defer { if scoped { url.stopAccessingSecurityScopedResource() } } guard let session = DASessionCreate(kCFAllocatorDefault), - let disk = DADiskCreateFromVolumePath(kCFAllocatorDefault, session, url as CFURL) else { + let disk = SelectedVolumeDiskResolver.disk( + for: url, + readVolumeRoot: { try? $0.resourceValues(forKeys: [.volumeURLKey]).volume }, + makeDisk: { DADiskCreateFromVolumePath(kCFAllocatorDefault, session, $0 as CFURL) } + ) else { return nil } let wholeDisk = DADiskCopyWholeDisk(disk) ?? disk diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DriveConnectionInfo.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DriveConnectionInfo.swift index 679b0dd2..1b6b6f00 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DriveConnectionInfo.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DriveConnectionInfo.swift @@ -75,7 +75,7 @@ public struct SystemDriveConnectionInfoProvider: DriveConnectionInfoProviding { /// The returned registry handle is owned by the caller. private static func mediaService(for url: URL) -> io_service_t? { guard let session = DASessionCreate(kCFAllocatorDefault), - let disk = DriveConnectionVolumeResolver.disk( + let disk = SelectedVolumeDiskResolver.disk( for: url, readVolumeRoot: { try? $0.resourceValues(forKeys: [.volumeURLKey]).volume }, makeDisk: { DADiskCreateFromVolumePath(kCFAllocatorDefault, session, $0 as CFURL) } @@ -94,23 +94,6 @@ public struct SystemDriveConnectionInfoProvider: DriveConnectionInfoProviding { #endif } -/// Disk Arbitration needs a mount root, even when the selected backup is a -/// folder several levels below it. Read only that selection's volume fact; -/// never discover or enumerate other mounted volumes. -enum DriveConnectionVolumeResolver { - static func disk( - for selectedURL: URL, - readVolumeRoot: (URL) -> URL?, - makeDisk: (URL) -> Disk?, - isCancelled: () -> Bool = { Task.isCancelled } - ) -> Disk? { - guard selectedURL.isFileURL, !isCancelled(), - let volumeRoot = readVolumeRoot(selectedURL), volumeRoot.isFileURL, - !isCancelled() else { return nil } - return makeDisk(volumeRoot) - } -} - /// Consumes its initial handle and every retained parent. Stop at the nearest /// USB device: a hub's speed must never stand in for its attached drive. enum DriveConnectionRegistryWalk { diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/SelectedVolumeDiskResolver.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/SelectedVolumeDiskResolver.swift new file mode 100644 index 00000000..d569d1b0 --- /dev/null +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/SelectedVolumeDiskResolver.swift @@ -0,0 +1,19 @@ +import Foundation + +/// Disk Arbitration needs a mount root, even when a selected folder is several +/// levels below it. Read only that selection's volume fact; never discover or +/// enumerate other mounted volumes. Both identity and connection lookup use +/// this adapter before calling DADiskCreateFromVolumePath. +enum SelectedVolumeDiskResolver { + static func disk( + for selectedURL: URL, + readVolumeRoot: (URL) -> URL?, + makeDisk: (URL) -> Disk?, + isCancelled: () -> Bool = { Task.isCancelled } + ) -> Disk? { + guard selectedURL.isFileURL, !isCancelled(), + let volumeRoot = readVolumeRoot(selectedURL), volumeRoot.isFileURL, + !isCancelled() else { return nil } + return makeDisk(volumeRoot) + } +} diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DriveConnectionInfoTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DriveConnectionInfoTests.swift index 1ff6fcc4..f5c4b60a 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DriveConnectionInfoTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DriveConnectionInfoTests.swift @@ -2,56 +2,6 @@ import XCTest @testable import BitMatchEngine final class DriveConnectionInfoTests: XCTestCase { - func testSelectedFolderResolvesItsOwnVolumeBeforeDiskLookup() { - let root = URL(fileURLWithPath: "/Volumes/Backup") - for selected in [root, root.appendingPathComponent("Card"), root.appendingPathComponent("Job/Day/Card")] { - var reads: [URL] = [] - var lookups: [URL] = [] - let value = DriveConnectionVolumeResolver.disk( - for: selected, - readVolumeRoot: { reads.append($0); return root }, - makeDisk: { lookups.append($0); return "selected-disk" } - ) - XCTAssertEqual(value, "selected-disk") - XCTAssertEqual(reads, [selected]) - XCTAssertEqual(lookups, [root], "Disk Arbitration must receive the mount root") - } - } - - func testMissingOrNonFileVolumeCannotTriggerDiskLookup() { - let selected = URL(fileURLWithPath: "/Volumes/Backup/Card") - let remote = URL(string: "sftp://example.invalid/backup")! - for volume in [nil, remote] as [URL?] { - let value: String? = DriveConnectionVolumeResolver.disk( - for: selected, readVolumeRoot: { _ in volume }, - makeDisk: { _ in XCTFail("No reliable selected volume"); return "unexpected" } - ) - XCTAssertNil(value) - } - let value: String? = DriveConnectionVolumeResolver.disk( - for: remote, readVolumeRoot: { _ in XCTFail("Not a local selection"); return selected }, - makeDisk: { _ in XCTFail("Not a local selection"); return "unexpected" } - ) - XCTAssertNil(value) - } - - func testCancelledVolumeResolutionCannotStartDiskLookup() { - for cancelDuringRead in [false, true] { - var cancelled = !cancelDuringRead - let value: String? = DriveConnectionVolumeResolver.disk( - for: URL(fileURLWithPath: "/Volumes/Backup/Card"), - readVolumeRoot: { _ in - XCTAssertTrue(cancelDuringRead) - cancelled = true - return URL(fileURLWithPath: "/Volumes/Backup") - }, - makeDisk: { _ in XCTFail("Cancelled lookup"); return "unexpected" }, - isCancelled: { cancelled } - ) - XCTAssertNil(value) - } - } - func testOnlyUSBHostHighSpeedProducesHint() { // IOUSBHost high-speed is 3. Legacy USBDeviceSpeed uses 2; it must // not accidentally label low/full-speed or unknown links USB 2. diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/PhysicalDiskIdentityFolderTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/PhysicalDiskIdentityFolderTests.swift new file mode 100644 index 00000000..84080b41 --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/PhysicalDiskIdentityFolderTests.swift @@ -0,0 +1,118 @@ +#if os(macOS) +import Darwin +import Foundation +import XCTest +@testable import BitMatchEngine + +/// Read-only opt-in proof over two already authorized, owner-marked physical +/// SSDs. The provisioning step must establish separate physical disks; distinct +/// disk-image or filesystem identities alone are not that proof. No identifiers, +/// footage paths or marker contents are printed, and no files are created. +final class PhysicalDiskIdentityFolderTests: XCTestCase { + func testNestedFoldersOnOnePhysicalSSDGroupOnce() throws { + let selections = try loadSelections() + let provider = SystemPhysicalDiskIdentityProvider() + for selection in selections { + let mount = try identity(provider, selection.mount) + XCTAssertTrue(try identity(provider, selection.root) == mount, + "A selected validation root must resolve its mounted disk") + XCTAssertTrue(try identity(provider, selection.child) == mount, + "A nested selected folder must resolve its mounted disk") + let result = BackupIndependencePolicy.assess( + destinations: [selection.root, selection.child], names: ["First folder", "Second folder"], + provider: provider + ) + XCTAssertEqual(result.selectedDestinationCount, 2) + XCTAssertEqual(result.resolvedIdentityGroupCount, 1) + XCTAssertEqual(result.unresolvedDestinationCount, 0) + XCTAssertEqual(result.independentCopyCount, 1) + XCTAssertEqual(result.warnings.count, 1) + } + } + + func testFoldersOnTwoAuthorizedPhysicalSSDsGroupSeparately() throws { + let selections = try loadSelections() + let provider = SystemPhysicalDiskIdentityProvider() + XCTAssertFalse(try identity(provider, selections[0].mount) == identity(provider, selections[1].mount), + "The prevalidated two-SSD fixture must resolve distinct disks") + let result = BackupIndependencePolicy.assess( + destinations: [selections[0].child, selections[1].child], names: ["First SSD", "Second SSD"], + provider: provider + ) + XCTAssertEqual(result.selectedDestinationCount, 2) + XCTAssertEqual(result.resolvedIdentityGroupCount, 2) + XCTAssertEqual(result.unresolvedDestinationCount, 0) + XCTAssertEqual(result.independentCopyCount, 2) + XCTAssertTrue(result.warnings.isEmpty) + XCTAssertNil(result.physicalSeparationNote) + } + + private func identity(_ provider: SystemPhysicalDiskIdentityProvider, _ url: URL) throws -> String { + guard let value = provider.physicalDiskIdentity(for: url) else { throw FixtureError.unresolvedDisk } + return value + } + + private struct Selection { let mount: URL; let root: URL; let child: URL } + private struct Config: Decodable { + struct Volume: Decodable, Equatable { + let nonce: String; let root: String; let mount: String; let volumeUUID: String + } + let nonce: String + let volumes: [String: Volume] + } + private enum FixtureError: Error { case unsafeFixture, unavailableChild, unresolvedDisk } + + private func loadSelections() throws -> [Selection] { + guard let path = ProcessInfo.processInfo.environment["BITMATCH_PHYSICAL_IDENTITY_CONFIG"] else { + throw XCTSkip("Set BITMATCH_PHYSICAL_IDENTITY_CONFIG for the read-only marked two-SSD proof") + } + let config: Config = try readJSON(URL(fileURLWithPath: path)) + guard UUID(uuidString: config.nonce) != nil else { throw FixtureError.unsafeFixture } + var selections: [Selection] = [] + var volumeIDs = Set() + for role in ["ssdA", "ssdB"] { + guard let volume = config.volumes[role], volume.nonce == config.nonce else { throw FixtureError.unsafeFixture } + let root = URL(fileURLWithPath: volume.root) + let mount = URL(fileURLWithPath: volume.mount) + var info = stat() + guard root.standardizedFileURL.path == volume.root, + mount.standardizedFileURL.path == volume.mount, + mount.deletingLastPathComponent().path == "/Volumes", + root.deletingLastPathComponent() == mount, + root.lastPathComponent.hasPrefix("BitMatch-validation-"), + root.lastPathComponent.contains(config.nonce), + lstat(root.path, &info) == 0, (info.st_mode & S_IFMT) == S_IFDIR else { + throw FixtureError.unsafeFixture + } + let marker: Config.Volume = try readJSON(root.appendingPathComponent(".bitmatch-validation-owned.json")) + guard marker == volume, + try root.resourceValues(forKeys: [.volumeUUIDStringKey]).volumeUUIDString? + .caseInsensitiveCompare(volume.volumeUUID) == .orderedSame else { throw FixtureError.unsafeFixture } + volumeIDs.insert(volume.volumeUUID.lowercased()) + let entries = try FileManager.default.contentsOfDirectory( + at: root, includingPropertiesForKeys: [], options: [.skipsHiddenFiles] + ) + guard let child = entries.sorted(by: { $0.lastPathComponent < $1.lastPathComponent }).first(where: { + var childInfo = stat() + return lstat($0.path, &childInfo) == 0 && (childInfo.st_mode & S_IFMT) == S_IFDIR + }) else { throw FixtureError.unavailableChild } + selections.append(Selection(mount: mount, root: root, child: child)) + } + guard volumeIDs.count == 2 else { throw FixtureError.unsafeFixture } + return selections + } + + private func readJSON(_ url: URL) throws -> Value { + let fd = Darwin.open(url.path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC) + guard fd >= 0 else { throw FixtureError.unsafeFixture } + let handle = FileHandle(fileDescriptor: fd, closeOnDealloc: true) + defer { try? handle.close() } + var info = stat() + guard fstat(fd, &info) == 0, (info.st_mode & S_IFMT) == S_IFREG, + info.st_size > 0, info.st_size <= 16_384 else { throw FixtureError.unsafeFixture } + let data = try handle.read(upToCount: 16_385) ?? Data() + guard data.count <= 16_384 else { throw FixtureError.unsafeFixture } + return try JSONDecoder().decode(Value.self, from: data) + } +} +#endif diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SelectedVolumeDiskResolverTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SelectedVolumeDiskResolverTests.swift new file mode 100644 index 00000000..eb412f66 --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SelectedVolumeDiskResolverTests.swift @@ -0,0 +1,54 @@ +import XCTest +@testable import BitMatchEngine + +final class SelectedVolumeDiskResolverTests: XCTestCase { + func testSelectedFolderResolvesItsOwnVolumeBeforeDiskLookup() { + let root = URL(fileURLWithPath: "/Volumes/Backup") + for selected in [root, root.appendingPathComponent("Card"), root.appendingPathComponent("Job/Day/Card")] { + var reads: [URL] = [] + var lookups: [URL] = [] + let value = SelectedVolumeDiskResolver.disk( + for: selected, + readVolumeRoot: { reads.append($0); return root }, + makeDisk: { lookups.append($0); return "selected-disk" } + ) + XCTAssertEqual(value, "selected-disk") + XCTAssertEqual(reads, [selected]) + XCTAssertEqual(lookups, [root], "Disk Arbitration must receive the mount root") + } + } + + func testMissingOrNonFileVolumeCannotTriggerDiskLookup() { + let selected = URL(fileURLWithPath: "/Volumes/Backup/Card") + let remote = URL(string: "sftp://example.invalid/backup")! + for volume in [nil, remote] as [URL?] { + let value: String? = SelectedVolumeDiskResolver.disk( + for: selected, readVolumeRoot: { _ in volume }, + makeDisk: { _ in XCTFail("No reliable selected volume"); return "unexpected" } + ) + XCTAssertNil(value) + } + let value: String? = SelectedVolumeDiskResolver.disk( + for: remote, readVolumeRoot: { _ in XCTFail("Not a local selection"); return selected }, + makeDisk: { _ in XCTFail("Not a local selection"); return "unexpected" } + ) + XCTAssertNil(value) + } + + func testCancelledVolumeResolutionCannotStartDiskLookup() { + for cancelDuringRead in [false, true] { + var cancelled = !cancelDuringRead + let value: String? = SelectedVolumeDiskResolver.disk( + for: URL(fileURLWithPath: "/Volumes/Backup/Card"), + readVolumeRoot: { _ in + XCTAssertTrue(cancelDuringRead) + cancelled = true + return URL(fileURLWithPath: "/Volumes/Backup") + }, + makeDisk: { _ in XCTFail("Cancelled lookup"); return "unexpected" }, + isCancelled: { cancelled } + ) + XCTAssertNil(value) + } + } +} From e473944a16e531414f2a606960521ce3bbbd0758 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 22:32:24 -0400 Subject: [PATCH 18/56] Preserve destination progress through MHL and reports --- .../CopyVerifyExecutorIntegrityTests.swift | 76 +++++++++++++++++++ CHANGELOG.md | 1 + Shared/Core/Services/CopyVerifyExecutor.swift | 25 ++++-- 3 files changed, 97 insertions(+), 5 deletions(-) diff --git a/BitMatchTests/CopyVerifyExecutorIntegrityTests.swift b/BitMatchTests/CopyVerifyExecutorIntegrityTests.swift index 659e4b2a..dbceb66f 100644 --- a/BitMatchTests/CopyVerifyExecutorIntegrityTests.swift +++ b/BitMatchTests/CopyVerifyExecutorIntegrityTests.swift @@ -181,6 +181,82 @@ final class CopyVerifyExecutorIntegrityTests: XCTestCase { XCTAssertTrue(harness.completedRows[0].isSuccessStatus) } + func testRealDestinationProgressSurvivesMHLAndReportHandoffs() async throws { + for includeMHL in [false, true] { + try await assertRealDestinationHandoff(includeMHL: includeMHL, failSecondDestination: false) + } + } + + func testFailedDestinationKeepsItsMeasuredProgressThroughHandoff() async throws { + try await assertRealDestinationHandoff(includeMHL: true, failSecondDestination: true) + } + + private func assertRealDestinationHandoff(includeMHL: Bool, failSecondDestination: Bool) async throws { + let fm = FileManager.default + let root = fm.temporaryDirectory.appendingPathComponent("executor-handoff-progress-\(UUID())") + let source = root.appendingPathComponent("Card") + let destinations = [root.appendingPathComponent("Backup A"), root.appendingPathComponent("Backup B")] + for folder in [source] + destinations { try fm.createDirectory(at: folder, withIntermediateDirectories: true) } + defer { try? fm.removeItem(at: root) } + let payloads = ["tiny.bin": Data("small payload".utf8), + "large.bin": Data(repeating: 0xA7, count: 4 * 1_024 * 1_024 + 17)] + for (name, data) in payloads { try data.write(to: source.appendingPathComponent(name)) } + let rejected = destinations[1] + let pipeline = TransferPipeline(fileSystem: MacOSFileSystemService.shared, checksum: ChecksumEngine.shared, + pipelinedVerification: true, destinationSetupHook: { url in + if failSecondDestination && url == rejected { throw NSError(domain: "SyntheticDestination", code: 5) } + }) + let harness = ExecutorHarness(returnedResults: [], emittedResults: [], sourceURL: source, + destinationURLs: destinations, generateASCMHL: includeMHL, makeReport: true, + realFileOperations: pipeline, estimatedFiles: payloads.count, + estimatedBytes: payloads.values.reduce(Int64(0)) { $0 + Int64($1.count) }) + let executed = try await harness.execute() + let operation = try XCTUnwrap(executed) + let firstHandoff = try XCTUnwrap(harness.observedProgress.firstIndex { $0.currentStage == .generating }) + let lastMedia = try XCTUnwrap(harness.observedProgress[.. OperationProgress { + precedingCompletedWorkBytes: Double? = nil, reservedHandoffBytes: Double? = nil, + mediaProgress: OperationProgress? = nil) -> OperationProgress { let fraction = update.totalBytes > 0 ? min(1, Double(update.bytesProcessed) / Double(update.totalBytes)) : 0 return OperationProgress( @@ -560,7 +572,10 @@ final class CopyVerifyExecutor { currentFile: update.published ? "ASC MHL records saved" : "Creating ASC MHL — checking destination media…", filesProcessed: update.filesProcessed, totalFiles: update.totalFiles, currentStage: .generating, speed: nil, elapsedTime: nil, averageSpeed: nil, peakSpeed: nil, - bytesProcessed: update.bytesProcessed, totalBytes: update.totalBytes, stageProgress: fraction, isASCMHL: true) + bytesProcessed: update.bytesProcessed, totalBytes: update.totalBytes, stageProgress: fraction, + perDestinationTotals: mediaProgress?.perDestinationTotals, + perDestinationCompleted: mediaProgress?.perDestinationCompleted, isASCMHL: true, + perDestinationProgress: mediaProgress?.perDestinationProgress) } private func handleError( From 332596805cae2249cc4b7964a7cd77e6b2b62b65 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 22:35:42 -0400 Subject: [PATCH 19/56] Add attributable classic MHL interoperability fixtures --- .../ClassicMHLCompatibilityTests.swift | 159 ++++++++++++++++++ .../Fixtures/EvidenceCompatibility/README.md | 2 + .../classic-reference/README.md | 72 ++++++++ .../classic-reference/REFERENCE-LICENSE.txt | 21 +++ .../media/Camera A/clip & cafe.txt | 1 + .../media/classic-md5-sha1.mhl | 61 +++++++ .../classic-reference/media/sidecar.xml | 1 + .../classic-reference/media/support file.txt | 1 + Scripts/ascmhl/create_classic_fixture.py | 66 ++++++++ Scripts/ascmhl/validate_classic_corpus.py | 125 ++++++++++++++ docs/EVIDENCE_COMPATIBILITY_VALIDATION.md | 10 ++ docs/TODO.md | 2 +- .../ascmhl/classic-reference-2026-10-19.json | 27 +++ 13 files changed, 547 insertions(+), 1 deletion(-) create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/ClassicMHLCompatibilityTests.swift create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/README.md create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/REFERENCE-LICENSE.txt create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/media/Camera A/clip & cafe.txt create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/media/classic-md5-sha1.mhl create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/media/sidecar.xml create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/media/support file.txt create mode 100644 Scripts/ascmhl/create_classic_fixture.py create mode 100644 Scripts/ascmhl/validate_classic_corpus.py create mode 100644 docs/validation/ascmhl/classic-reference-2026-10-19.json diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ClassicMHLCompatibilityTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ClassicMHLCompatibilityTests.swift new file mode 100644 index 00000000..91918303 --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ClassicMHLCompatibilityTests.swift @@ -0,0 +1,159 @@ +import CryptoKit +import Foundation +#if canImport(FoundationXML) +import FoundationXML +#endif +import Testing +@testable import BitMatchEngine + +/// Original-format reference output is a corpus, not a new supported reader. +/// All Check calls below are read-only and must keep that boundary explicit. +struct ClassicMHLCompatibilityTests { + private var fixture: URL { + get throws { + try #require(Bundle.module.resourceURL) + .appendingPathComponent("Fixtures/EvidenceCompatibility/classic-reference/media") + } + } + private var checker: SavedChecksumCheck { + SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + } + private func copyFixture(nested: Bool = false) throws -> (root: URL, media: URL) { + let root = FileManager.default.temporaryDirectory.appendingPathComponent("bitmatch-classic-corpus-\(UUID())") + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + let media = nested ? root.appendingPathComponent("Camera card") : root.appendingPathComponent("media") + try FileManager.default.copyItem(at: fixture, to: media) + return (root, media) + } + private func snapshot(_ root: URL) throws -> [String: Data] { + try Dictionary(uniqueKeysWithValues: CardSource.enumerateRegularFiles(base: root).map { + ($0.relativePath, try Data(contentsOf: $0.url)) + }) + } + + /// A test-only XML inventory extractor, deliberately separate from Check. + /// CryptoKit independently checks every MD5/SHA-1 record in the fixture. + @Test func originalReferenceInventoryHasSixValidDigestRecords() throws { + let expected = ["Camera A/clip & cafe.txt": Data("Classic MHL independent fixture\n".utf8), + "support file.txt": Data("Whitespace is part of this filename.\n".utf8), + "sidecar.xml": Data("synthetic\n".utf8)] + let root = try fixture + let before = try snapshot(root) + let parser = XMLParser(data: try Data(contentsOf: root.appendingPathComponent("classic-md5-sha1.mhl"))) + let delegate = ClassicInventoryFixtureParser() + parser.shouldResolveExternalEntities = false + parser.delegate = delegate + #expect(parser.parse()) + #expect(delegate.version == "1.1") + #expect(delegate.items.count == 6) + #expect(Set(delegate.items.map(\.path)) == Set(expected.keys)) + for item in delegate.items { + let bytes = try #require(expected[item.path]) + #expect(try Data(contentsOf: root.appendingPathComponent(item.path)) == bytes) + #expect(item.size == bytes.count) + let digest: String + if item.algorithm == "md5" { + digest = Insecure.MD5.hash(data: bytes).map { String(format: "%02x", $0) }.joined() + } else { + #expect(item.algorithm == "sha1") + digest = Insecure.SHA1.hash(data: bytes).map { String(format: "%02x", $0) }.joined() + } + #expect(item.digest == digest) + } + #expect(delegate.items.filter { $0.algorithm == "md5" }.count == 3) + #expect(delegate.items.filter { $0.algorithm == "sha1" }.count == 3) + #expect(try snapshot(root) == before) + } + + /// Standalone discovery is not implemented. Finding a classic inventory + /// cannot fabricate ASC records, a named copy tier, or an intact verdict. + @Test(arguments: [false, true]) + func standaloneClassicInventoryDoesNotAcquireAnASCClaim(nested: Bool) async throws { + let fixture = try copyFixture(nested: nested) + defer { try? FileManager.default.removeItem(at: fixture.root) } + let before = try snapshot(fixture.root) + await #expect(throws: SavedChecksumCheck.CheckError.notFound) { + _ = try await checker.discover(under: nested ? fixture.root : fixture.media) + } + #expect(try snapshot(fixture.root) == before) + } + + /// A valid ASC neighbour must never conceal unsupported classic evidence + /// inside the ASC evidence tree. Mutants are disposable test data only. + @Test(arguments: ["original", "sha1-only", "size-only", "future-version", "truncated", "traversal"]) + func classicEvidenceInsideASCHistoryRemainsUnsupported(kind: String) async throws { + let fixture = try copyFixture() + defer { try? FileManager.default.removeItem(at: fixture.root) } + let original = fixture.media.appendingPathComponent("classic-md5-sha1.mhl") + let originalText = try String(contentsOf: original, encoding: .utf8) + var text = originalText + switch kind { + case "sha1-only": + text = text.replacingOccurrences(of: #"(?s)(?:(?!).)*[^<]+(?:(?!).)*"#, + with: "", options: .regularExpression) + case "size-only": + text = text.replacingOccurrences(of: #"<(md5|sha1)>[^<]+"#, with: "", options: .regularExpression) + case "future-version": text = text.replacingOccurrences(of: "version=\"1.1\"", with: "version=\"9.0\"") + case "truncated": text = String(text.dropLast(30)) + case "traversal": text = text.replacingOccurrences(of: "sidecar.xml", with: "../outside.xml") + default: break + } + if kind != "original" { #expect(text != originalText) } + if kind == "sha1-only" { + let parser = XMLParser(data: Data(text.utf8)) + let delegate = ClassicInventoryFixtureParser() + parser.delegate = delegate + #expect(parser.parse()) + #expect(delegate.items.count == 3) + #expect(delegate.items.allSatisfy { $0.algorithm == "sha1" }) + } + if kind == "size-only" { + #expect(!text.contains("")) + #expect(!text.contains("")) + #expect(text.components(separatedBy: "").count - 1 == 6) + } + let history = fixture.media.appendingPathComponent("ascmhl") + try FileManager.default.createDirectory(at: history, withIntermediateDirectories: true) + try Data(text.utf8).write(to: history.appendingPathComponent("classic.mhl")) + let payload = Data("A".utf8) + try payload.write(to: fixture.media.appendingPathComponent("supported.txt")) + let md5 = Insecure.MD5.hash(data: payload).map { String(format: "%02x", $0) }.joined() + let asc = """ + + 2026-10-01T00:00:00Z + supported.txt\(md5) + """ + try Data(asc.utf8).write(to: history.appendingPathComponent("supported.mhl")) + let before = try snapshot(fixture.root) + await #expect(throws: SavedChecksumCheck.CheckError.unsupportedEvidence) { + _ = try await checker.discover(under: fixture.media) + } + #expect(try snapshot(fixture.root) == before) + } +} + +private final class ClassicInventoryFixtureParser: NSObject, XMLParserDelegate { + struct Item { + var path = "" + var size: Int? + var algorithm = "" + var digest = "" + } + var version: String? + var items: [Item] = [] + private var item: Item? + private var text = "" + func parser(_ parser: XMLParser, didStartElement name: String, namespaceURI: String?, qualifiedName: String?, attributes: [String: String]) { + text = "" + if name == "hashlist" { version = attributes["version"] } + if name == "hash" { item = Item() } + } + func parser(_ parser: XMLParser, foundCharacters string: String) { text += string } + func parser(_ parser: XMLParser, didEndElement name: String, namespaceURI: String?, qualifiedName: String?) { + if name == "file" { item?.path = text } + if name == "size" { item?.size = Int(text) } + if name == "md5" || name == "sha1" { item?.algorithm = name; item?.digest = text } + if name == "hash", let item { items.append(item); self.item = nil } + text = "" + } +} diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/README.md b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/README.md index f184f54f..477e65ce 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/README.md +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/README.md @@ -30,6 +30,8 @@ Generate just new datasets without replacing existing originals using `--dataset `classic-v1.mhl` is a handcrafted boundary fixture, not external-tool output. It is rejected inside an ASC history both alone and beside a valid reference history; standalone classic-MHL discovery is still unsupported. Check's supported inventory-chain scope is described below; directory hashes, rename lineage and historical authenticity remain outside it. Histories are never rewritten by these checks. +October 19 original-format addition: [`classic-reference/`](classic-reference/README.md) contains three original tiny media files and a sanitized classic MHL 1.1 inventory generated by the unchanged, MIT-licensed original reference CLI at `pomfort/mhl-tool@34f06f22ae965d12e49590adc301373d5ce611ff`. Its six MD5/SHA-1 digest records pass independent Python and CryptoKit checks, the original v1.1 schema and reference-tool verification. The retained MIT notice, exact commands, schema hashes and disposable negative controls are documented there. This expands attributable input/boundary coverage; it does not implement standalone classic discovery, label generic hashes as a BitMatch verification tier, or add history writes. + October 19 addition: `reference/bitmatch-fat-extended/` is a **sanitized derivative** of the actual engine's initial FAT-v1 history generated by `ASCMHLFixture` on a disposable mounted exFAT image. The real mounted sample separately passed official extension/verification/diff with its original status unchanged. For the public fixture, the initial hostname was replaced with `bitmatch-fixture-host` and the synthetic clip filename changed to ASCII `Camera A/clip.txt`, and its exact v1 single-generation chain/status was rebuilt during fixture creation only. The pinned official `ascmhl create -h md5 -i '._*'` then appended generation two to that sanitized copy; this is not a byte-for-byte mounted-volume capture. The first media payload comes from the original/MIT fixture executable; `added.txt` is original synthetic text. The new reference output retains the ASC MIT notice above. The official CLI hostname is the synthetic `bitmatch-fixture-host`. Its explicit metadata pattern excludes only test-volume AppleDouble companions from this portable dataset; it does not define BitMatch's production filtering policy. No existing fixture was rewritten. diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/README.md b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/README.md new file mode 100644 index 00000000..225d837f --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/README.md @@ -0,0 +1,72 @@ +# Original-format MHL reference fixture + +This tiny corpus is an independently generated **classic MHL 1.1** inventory. +It does not add classic-MHL support to BitMatch. Check currently discovers ASC +inventories under `ascmhl/`; standalone classic inventories are not discovered, +and classic records inside that evidence tree are explicitly unsupported. + +The three media files are original synthetic text, under BitMatch's MIT license. +The inventory came from the original-format [reference CLI](https://github.com/pomfort/mhl-tool) +at revision `34f06f22ae965d12e49590adc301373d5ce611ff`, built from unchanged source. +Its exact MIT notice is retained in `REFERENCE-LICENSE.txt`. Only the generated +`name`, `username` and `hostname` creator fields were replaced with synthetic +values. The manifest was renamed to `classic-md5-sha1.mhl`. Paths, file sizes, +digest values and date fields are unchanged. This is a sanitized derivative, +not a byte-for-byte capture of the raw command output. + +The [original specification](https://mediahashlist.org/mhl-specification/) links +the [v1.1 schema](https://mediahashlist.org/download/MediaHashList_v1_1.xsd). +Its SHA-256 is `c1adcd2591069249c88f2fc1fabac81363e7572b952834cce66f72a264e4c703`. +The imported [W3C signature schema](https://www.w3.org/TR/xmldsig-core/xmldsig-core-schema.xsd) +has SHA-256 `d102ad3df7664c307e0c2c776ba4a90513b1969974d8a940bae1a77f9f21e15d`. +These schema files are downloaded privately for validation, not redistributed. + +## Generation + +Build the pinned reference checkout outside this repository. On this Mac, the +unmodified 34 C sources built with Apple clang, the SDK's libxml2, system iconv +and the existing Homebrew OpenSSL 3 installation: + +```sh +sdk=$(xcrun --show-sdk-path) +xcrun clang -O2 -D_GNU_SOURCE -Wno-deprecated-declarations \ + -I"$reference/src" -I"$sdk/usr/include/libxml2" \ + -I/opt/homebrew/opt/openssl@3/include \ + "$reference"/src/*/*.c "$reference"/src/generics/filesystem_handlers/*.c \ + "$reference"/src/mhl_verify/mhl_verification/*.c \ + -L/opt/homebrew/opt/openssl@3/lib -lcrypto -lxml2 -liconv -o "$mhl_cli" +python3 Scripts/ascmhl/create_classic_fixture.py "$reference" "$mhl_cli" /tmp/new-classic-fixture +``` + +The generator uses `mhl hash -t md5 sha1 MEDIA`, then passes those lines to +`mhl file -s -o MEDIA`. It sanitizes in disposable scratch and requires +`mhl verify -f MANIFEST` to succeed before copying the derivative to a new output +directory. Regeneration retains the tool's current dates, so it need not produce +byte-identical metadata. No application or reference implementation code is +copied into BitMatch's engine. + +## Validation and limits + +Run with Python `lxml`, the pinned checkout/CLI and the two downloaded schemas: + +```sh +python3 Scripts/ascmhl/validate_classic_corpus.py \ + "$reference" "$mhl_cli" "$classic_xsd" "$xmldsig_xsd" +``` + +The runner validates the schema and exact original payloads, independently +checks all six digest/size records with Python MD5/SHA-1, and verifies the media +with the reference CLI. There are three paths and six records: that CLI emits a +separate `` entry for each algorithm on each path. Same-size corruption +must return 16, missing media 3 and truncated XML 11. All negative controls use +disposable copies; permanent fixture bytes must stay unchanged. + +Swift tests independently check all six digests with CryptoKit and assert the +current unsupported/not-found boundary, including a valid ASC neighbour and +disposable SHA-1-only, size-only, future-version, truncated and traversal cases. +Generic hashes do not establish a BitMatch verification tier or safe-to-erase +verdict. This corpus does not cover classic history references, signatures, +XXHash encoding variants, empty files or every filename encoding. The older +reference CLI trimmed leading/trailing filename whitespace in an exploratory +sample; that sample is excluded from the passing corpus. Existing ASC fixtures +cover whitespace and Unicode independently. No history is extended or rewritten. diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/REFERENCE-LICENSE.txt b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/REFERENCE-LICENSE.txt new file mode 100644 index 00000000..9245bb9d --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/REFERENCE-LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Pomfort GmbH + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/media/Camera A/clip & cafe.txt b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/media/Camera A/clip & cafe.txt new file mode 100644 index 00000000..08730fb9 --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/media/Camera A/clip & cafe.txt @@ -0,0 +1 @@ +Classic MHL independent fixture diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/media/classic-md5-sha1.mhl b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/media/classic-md5-sha1.mhl new file mode 100644 index 00000000..99a47726 --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/media/classic-md5-sha1.mhl @@ -0,0 +1,61 @@ + + + + + BitMatch fixture + fixture +bitmatch-fixture-host + mhl ver. 0.2.0 + 2026-10-20T02:28:55Z + 2026-10-20T02:28:55Z + + + + support file.txt + 37 + 2026-10-18T20:00:00Z + 01d0734d719fbac42635526e9e4c3213 + 2026-10-20T02:28:55Z + + + + support file.txt + 37 + 2026-10-18T20:00:00Z + 5b68a53be4c5587e9498d1e1b2333219ae8d7a1c + 2026-10-20T02:28:55Z + + + + Camera A/clip & cafe.txt + 32 + 2026-10-18T20:00:00Z + f0da98d79b3b7388cce022a84dbfff56 + 2026-10-20T02:28:55Z + + + + Camera A/clip & cafe.txt + 32 + 2026-10-18T20:00:00Z + d21df1382517d6846720aa351d8e1c3d4d4eb33c + 2026-10-20T02:28:55Z + + + + sidecar.xml + 23 + 2026-10-18T20:00:00Z + cf64d1ed2a8e8e5d795e2d8bbdca91d8 + 2026-10-20T02:28:55Z + + + + sidecar.xml + 23 + 2026-10-18T20:00:00Z + 040182793e3030a9439bd2e179df16edf5299fa2 + 2026-10-20T02:28:55Z + + + diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/media/sidecar.xml b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/media/sidecar.xml new file mode 100644 index 00000000..733e7fae --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/media/sidecar.xml @@ -0,0 +1 @@ +synthetic diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/media/support file.txt b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/media/support file.txt new file mode 100644 index 00000000..fe7c86d2 --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/media/support file.txt @@ -0,0 +1 @@ +Whitespace is part of this filename. diff --git a/Scripts/ascmhl/create_classic_fixture.py b/Scripts/ascmhl/create_classic_fixture.py new file mode 100644 index 00000000..d4775706 --- /dev/null +++ b/Scripts/ascmhl/create_classic_fixture.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Generate an original-media classic MHL fixture with a pinned reference CLI. + +No reference source is copied into BitMatch. Output must not exist. The three +personal creator fields are replaced; inventory paths/digests remain untouched. +""" +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile + +REFERENCE_COMMIT = '34f06f22ae965d12e49590adc301373d5ce611ff' +PAYLOADS = { + 'Camera A/clip & cafe.txt': b'Classic MHL independent fixture\n', + 'support file.txt': b'Whitespace is part of this filename.\n', + 'sidecar.xml': b'synthetic\n', +} + + +def require_reference(reference): + revision = subprocess.check_output(['git', '-C', str(reference), 'rev-parse', 'HEAD'], text=True).strip() + if revision != REFERENCE_COMMIT: + raise RuntimeError('Wrong classic MHL reference revision') + if subprocess.check_output(['git', '-C', str(reference), 'status', '--porcelain', '--untracked-files=no']): + raise RuntimeError('Classic MHL reference source was modified') + + +def main(): + if len(sys.argv) != 4: + raise SystemExit('Usage: create_classic_fixture.py REFERENCE_CHECKOUT MHL_CLI NEW_OUTPUT') + reference, cli, output = (pathlib.Path(value).resolve() for value in sys.argv[1:]) + require_reference(reference) + if output.exists(): + raise RuntimeError('Output must not exist') + # Raw creator data stays in disposable scratch even if generation fails. + with tempfile.TemporaryDirectory(prefix='bitmatch-classic-generate-') as temporary: + prepared = pathlib.Path(temporary) / 'prepared' + media = prepared / 'media' + media.mkdir(parents=True) + for name, data in PAYLOADS.items(): + file = media / name + file.parent.mkdir(parents=True, exist_ok=True) + file.write_bytes(data) + hashes = subprocess.check_output([str(cli), 'hash', '-t', 'md5', 'sha1', str(media)]) + subprocess.run([str(cli), 'file', '-s', '-o', str(media)], input=hashes, check=True, capture_output=True) + manifests = list(media.glob('*.mhl')) + if len(manifests) != 1: + raise RuntimeError('Expected exactly one classic inventory') + generated = manifests[0] + text = generated.read_text(encoding='utf-8') + for tag, value in [('name', 'BitMatch fixture'), ('username', 'fixture'), ('hostname', 'bitmatch-fixture-host')]: + text, count = re.subn(r'<' + tag + r'>.*?', '<' + tag + '>' + value + '', text) + if count != 1: + raise RuntimeError('Expected exactly one creator field: ' + tag) + generated.write_text(text, encoding='utf-8') + generated.rename(media / 'classic-md5-sha1.mhl') + subprocess.run([str(cli), 'verify', '-f', str(media / 'classic-md5-sha1.mhl')], check=True, capture_output=True) + (prepared / 'REFERENCE-LICENSE.txt').write_bytes((reference / 'LICENSE').read_bytes()) + shutil.copytree(prepared, output) + print('PASS: 3 original files, 6 digest records; sanitized reference output verified') + + +if __name__ == '__main__': + main() diff --git a/Scripts/ascmhl/validate_classic_corpus.py b/Scripts/ascmhl/validate_classic_corpus.py new file mode 100644 index 00000000..b7b3f98d --- /dev/null +++ b/Scripts/ascmhl/validate_classic_corpus.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Read-only classic MHL corpus checks using primary schemas/reference CLI. + +Requires lxml. Schemas are caller-supplied private downloads, not vendored. +All corruption controls run in disposable copies. This does not test or enable +classic-MHL support in BitMatch's ASC-only saved-checksum reader. +""" +import hashlib +import json +import pathlib +import shutil +import subprocess +import sys +import tempfile + +from lxml import etree +from create_classic_fixture import PAYLOADS, REFERENCE_COMMIT, require_reference + +SCHEMA_SHA256 = 'c1adcd2591069249c88f2fc1fabac81363e7572b952834cce66f72a264e4c703' +SIGNATURE_SCHEMA_SHA256 = 'd102ad3df7664c307e0c2c776ba4a90513b1969974d8a940bae1a77f9f21e15d' +SIGNATURE_SCHEMA_URL = 'http://www.w3.org/TR/xmldsig-core/xmldsig-core-schema.xsd' + + +def snapshot(root): + return {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in root.rglob('*') if p.is_file()} + + +def main(): + if len(sys.argv) != 5: + raise SystemExit('Usage: validate_classic_corpus.py REFERENCE_CHECKOUT MHL_CLI MHL_XSD XMLDSIG_XSD') + reference, cli, schema_path, signature_path = (pathlib.Path(value).resolve() for value in sys.argv[1:]) + checks = [] + require_reference(reference) + checks.append('pinned unmodified classic reference source') + project = pathlib.Path(__file__).resolve().parents[2] + corpus = project / 'Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference' + before = snapshot(corpus) + if (corpus / 'REFERENCE-LICENSE.txt').read_bytes() != (reference / 'LICENSE').read_bytes(): + raise RuntimeError('Reference MIT notice changed') + checks.append('reference MIT notice retained exactly') + for path, expected in [(schema_path, SCHEMA_SHA256), (signature_path, SIGNATURE_SCHEMA_SHA256)]: + if hashlib.sha256(path.read_bytes()).hexdigest() != expected: + raise RuntimeError('Unexpected primary schema bytes: ' + path.name) + checks.append('pinned primary schema: ' + path.name) + + class Resolver(etree.Resolver): + def resolve(self, url, public_id, context): + if url == SIGNATURE_SCHEMA_URL: + return self.resolve_filename(str(signature_path), context) + raise RuntimeError('Unexpected external schema dependency: ' + url) + + parser = etree.XMLParser(no_network=True, load_dtd=False, resolve_entities=False) + parser.resolvers.add(Resolver()) + schema = etree.XMLSchema(etree.fromstring(schema_path.read_bytes(), parser)) + media = corpus / 'media' + manifest = media / 'classic-md5-sha1.mhl' + tree = etree.fromstring(manifest.read_bytes(), parser) + schema.assertValid(tree) + checks.append('original-format v1.1 inventory XSD') + hashes = tree.findall('hash') + if len(hashes) != 6 or {h.findtext('file') for h in hashes} != set(PAYLOADS): + raise RuntimeError('Incomplete classic inventory') + if {str(p.relative_to(media)) for p in media.rglob('*') if p.is_file()} != set(PAYLOADS) | {'classic-md5-sha1.mhl'}: + raise RuntimeError('Unexpected classic fixture file') + for name, expected in PAYLOADS.items(): + if (media / name).read_bytes() != expected: + raise RuntimeError('Original synthetic payload changed') + checks.append('exact original-media inventory coverage') + for entry in hashes: + name = entry.findtext('file') + data = (media / name).read_bytes() + digest_elements = [child for child in entry if child.tag in ('md5', 'sha1')] + if len(digest_elements) != 1 or int(entry.findtext('size')) != len(data): + raise RuntimeError('Invalid digest or size record') + digest = digest_elements[0] + if hashlib.new(digest.tag, data).hexdigest() != digest.text: + raise RuntimeError('Independent digest mismatch') + checks.append('Python ' + digest.tag.upper() + ' and size: ' + name) + if sorted(child.tag for entry in hashes for child in entry if child.tag in ('md5', 'sha1')) != ['md5'] * 3 + ['sha1'] * 3: + raise RuntimeError('Unexpected digest coverage') + invalid = etree.fromstring(manifest.read_bytes(), parser) + invalid.find('hash/md5').text = 'bad' + if schema.validate(invalid): + raise RuntimeError('Schema accepted a malformed digest') + checks.append('primary schema rejects malformed digest') + for kind in ['sha1-only', 'size-only']: + variant = etree.fromstring(manifest.read_bytes(), parser) + for entry in list(variant.findall('hash')): + if kind == 'sha1-only' and entry.find('md5') is not None: + variant.remove(entry) + elif kind == 'size-only': + for digest in list(entry): + if digest.tag in ('md5', 'sha1'): + digest.tag = 'null' + digest.text = None + schema.assertValid(variant) + checks.append('primary schema accepts unsupported-in-BitMatch ' + kind + ' variant') + + with tempfile.TemporaryDirectory(prefix='bitmatch-classic-corpus-') as temporary: + for kind, expected in [('original', 0), ('changed-same-size', 16), ('missing', 3), ('truncated', 11)]: + root = pathlib.Path(temporary) / kind + shutil.copytree(media, root) + record = root / manifest.name + if kind == 'changed-same-size': + file = root / 'sidecar.xml' + file.write_bytes(b'X' + file.read_bytes()[1:]) + elif kind == 'missing': + (root / 'sidecar.xml').unlink() + elif kind == 'truncated': + record.write_bytes(record.read_bytes()[:-30]) + result = subprocess.run([str(cli), 'verify', '-f', str(record)], capture_output=True) + if result.returncode != expected: + raise RuntimeError('Unexpected reference result for ' + kind + ': ' + str(result.returncode)) + checks.append('classic reference ' + kind + ': exit ' + str(expected)) + if snapshot(corpus) != before: + raise RuntimeError('Validation changed the permanent corpus') + checks.append('permanent corpus unchanged') + print(json.dumps({'reference_commit': REFERENCE_COMMIT, + 'scope': 'classic reference inventory/boundary corpus; no BitMatch classic support, hardware or history writes', + 'passed': len(checks), 'checks': checks}, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/docs/EVIDENCE_COMPATIBILITY_VALIDATION.md b/docs/EVIDENCE_COMPATIBILITY_VALIDATION.md index f9dd582d..caa28383 100644 --- a/docs/EVIDENCE_COMPATIBILITY_VALIDATION.md +++ b/docs/EVIDENCE_COMPATIBILITY_VALIDATION.md @@ -46,6 +46,16 @@ After merging #20 into this branch, the full Mac run passed: 1099 passed, 13 ski This is file-content interoperability coverage, not certification against every offload tool. BitMatch still does not discover standalone classic MHL, verify ASC directory/root hashes, interpret rename history, or extend existing histories. The October 19 read-only chain slice below adds a narrower inventory-chain check. Those roadmap items remain open. Supported BitMatch reports still take precedence over ASC inventories for Check. The saved-report scanner summarizes past evidence; it does not freshly verify media. Synthetic fixtures do not establish physical cable, dock or drive reliability. +## October 19 original-format reference corpus + +The [`classic-reference` fixture](../Packages/BitMatchEngine/Tests/BitMatchEngineTests/Fixtures/EvidenceCompatibility/classic-reference/README.md) adds an independently generated classic MHL 1.1 inventory from the original reference CLI at `pomfort/mhl-tool@34f06f22ae965d12e49590adc301373d5ce611ff`. Its exact MIT notice is retained. The three media files are original synthetic text; only personal creator fields in the generated XML were replaced with synthetic values. The original fixture remains separate from the earlier handcrafted classic-v1 boundary. + +The private validation run passed **20 checks**: pinned unmodified reference source, retained license, exact primary-schema hashes, v1.1 schema validity, all six MD5/SHA-1 digest/size records independently checked with Python, schema acceptance of SHA-1-only and size-only variants, malformed-digest rejection, and reference-tool original/corrupt/missing/truncated controls. Same-size corruption returned 16, missing media 3 and truncated XML 11; the intact derivative returned 0. All controls used disposable copies and permanent fixture bytes were unchanged. The generator was also run independently into a new directory and reproduced three files with six correct digest records and synthetic creator fields. + +The new Swift suite passed **three test functions with nine invocations**: CryptoKit checks all six fixture digests; root/nested standalone discovery remains `notFound`; six variants beside valid ASC evidence remain explicitly unsupported. These are current-boundary tests, not classic-MHL reader implementation. No generic hash is mapped to a named BitMatch verification tier, no history is extended or rewritten, and no app host or physical media is involved. The fixture README records commands, provenance, primary-schema URLs/hashes, encoding limits and an exploratory whitespace limitation in the older reference CLI. + +The surrounding engine compatibility run passed **52 test functions across four suites**, zero failures, in 0.660 seconds: `ClassicMHLCompatibilityTests`, `EvidenceCompatibilityTests`, `SavedChecksumCheckTests` and `ASCMHLChainValidationTests`. This focused check does not replace the integrated full engine or app gates. + ## Signed-walkthrough follow-up The first signed 0.2.8 candidate completed a four-file Standard transfer to APFS and exFAT, with matching source/destination hashes, MHL on both backups and reports. Check then refused the valid exFAT history because macOS had created `._.mhl` for its extended attributes. The stricter reader was treating that metadata container as an XML document. This was reproduced in the signed UI before publication. diff --git a/docs/TODO.md b/docs/TODO.md index 3dff0cb3..6194c4a6 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -124,7 +124,7 @@ The independent ASC corpus covers unsupported file hashes, XXH64 boundaries and - [ ] **Upstream MHL / source provenance.** Reuse `SavedChecksumCheck` where appropriate to offer source verification against accompanying classic MHL or ASC MHL evidence. Record “source arrived intact” separately from “BitMatch created verified backups.” Preserve original manifests. Current Check discovers ASC inventories under `ascmhl/`; standalone classic-MHL discovery and an upstream transfer verdict still need work. **Acceptance:** incomplete, mismatched, malformed and unsupported evidence gets an honest separate verdict; a failed provenance check never becomes success and never prevents preservation of damaged-but-irreplaceable media. -- [ ] **MHL interoperability corpus.** The permanent corpus covers official ASC reference-tool MD5/XXH64 output, two generations, unsupported C4/SHA-1/XXH128 file hashes, Unicode/whitespace paths, nested inventories, damaged records, a recorded file rename and directory move, externally extended FAT history, and report compatibility. The classic-v1 boundary is handcrafted and explicitly identified as such. See [the validation record](EVIDENCE_COMPATIBILITY_VALIDATION.md). Continue with redistributable, independently attributable classic MHL and representative output from another implementation before claiming broad interoperability. Keep multi-file saved-record and relative-path regressions current. **Acceptance:** validate with independent reference implementations, record fixture provenance/licensing and expected outcomes, and never silently treat unsupported evidence as fully verified. +- [ ] **MHL interoperability corpus.** The permanent corpus covers official ASC reference-tool MD5/XXH64 output, two generations, unsupported C4/SHA-1/XXH128 file hashes, Unicode/whitespace paths, nested inventories, damaged records, a recorded file rename and directory move, externally extended FAT history, and report compatibility. The handcrafted classic-v1 boundary now has an independently generated classic MHL 1.1 companion corpus: three original tiny files, six MD5/SHA-1 records, primary-schema/reference-tool checks and an exact MIT notice. Classic discovery remains unsupported; its tests preserve that boundary rather than widening the reader. See [the validation record](EVIDENCE_COMPATIBILITY_VALIDATION.md). Continue with representative output from another implementation and the remaining classic format/encoding cases before claiming broad interoperability. Keep multi-file saved-record and relative-path regressions current. **Acceptance:** validate with independent reference implementations, record fixture provenance/licensing and expected outcomes, and never silently treat unsupported evidence as fully verified. - [ ] **ASC MHL structural verification.** The #57 slice validates supported inventory-chain records read-only: referenced inventory digests, ordered generations, safe paths, exact inventory coverage and unchanged evidence during Check. Legacy no-chain inventories remain content-only. **Remaining:** supported root/directory hashes and rename/move lineage; unsupported records must stay explicit. Chain validation does not authenticate custody or prove that an unanchored historical tail was never removed. **Acceptance:** keep file contents, supported chain records and directory/rename structure as separate scopes; do not call this full history verification. diff --git a/docs/validation/ascmhl/classic-reference-2026-10-19.json b/docs/validation/ascmhl/classic-reference-2026-10-19.json new file mode 100644 index 00000000..a94acd5c --- /dev/null +++ b/docs/validation/ascmhl/classic-reference-2026-10-19.json @@ -0,0 +1,27 @@ +{ + "reference_commit": "34f06f22ae965d12e49590adc301373d5ce611ff", + "scope": "classic reference inventory/boundary corpus; no BitMatch classic support, hardware or history writes", + "passed": 20, + "checks": [ + "pinned unmodified classic reference source", + "reference MIT notice retained exactly", + "pinned primary schema: bitmatch-oct19-classic-MediaHashList_v1_1.xsd", + "pinned primary schema: bitmatch-oct19-classic-xmldsig-core-schema.xsd", + "original-format v1.1 inventory XSD", + "exact original-media inventory coverage", + "Python MD5 and size: support file.txt", + "Python SHA1 and size: support file.txt", + "Python MD5 and size: Camera A/clip & cafe.txt", + "Python SHA1 and size: Camera A/clip & cafe.txt", + "Python MD5 and size: sidecar.xml", + "Python SHA1 and size: sidecar.xml", + "primary schema rejects malformed digest", + "primary schema accepts unsupported-in-BitMatch sha1-only variant", + "primary schema accepts unsupported-in-BitMatch size-only variant", + "classic reference original: exit 0", + "classic reference changed-same-size: exit 16", + "classic reference missing: exit 3", + "classic reference truncated: exit 11", + "permanent corpus unchanged" + ] +} From a4c2c079927435720decba40ae44cc3cf4c2f440 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 22:38:31 -0400 Subject: [PATCH 20/56] Make coordinator workflow fixtures explicit about disk identity --- .../TestHelpers/SharedCoordinatorTestSupport.swift | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/BitMatchTests/TestHelpers/SharedCoordinatorTestSupport.swift b/BitMatchTests/TestHelpers/SharedCoordinatorTestSupport.swift index 53404069..f9870bf5 100644 --- a/BitMatchTests/TestHelpers/SharedCoordinatorTestSupport.swift +++ b/BitMatchTests/TestHelpers/SharedCoordinatorTestSupport.swift @@ -177,6 +177,13 @@ final class SharedProjectFixture { self.coordinator = coordinator } + /// Workflow tests use distinct abstract disks; their temporary folders + /// are not independent physical backups. Real provider/grouping behavior + /// is checked separately, and callers can inject shared/unknown identities. + private struct DistinctFixtureDisks: PhysicalDiskIdentityProviding { + func physicalDiskIdentity(for url: URL) -> String? { "fixture-disk:" + url.standardizedFileURL.path } + } + /// Waits for the source scan before preparing, so the coordinator's /// launch-time source events have all been delivered. static func make( @@ -184,7 +191,7 @@ final class SharedProjectFixture { reportsStage: ProgressStage? = nil, corruptJournal: Bool = false, prepareCard: Bool = true, - physicalDiskIdentityProvider: any PhysicalDiskIdentityProviding = SystemPhysicalDiskIdentityProvider() + physicalDiskIdentityProvider: any PhysicalDiskIdentityProviding = DistinctFixtureDisks() ) async throws -> SharedProjectFixture { let folders = try CoordinatorFolders() if corruptJournal { try Data("corrupt history".utf8).write(to: folders.journalURL) } From 6ed8a2b7e78d5b4cbef102cad92be9f960194608 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 22:38:31 -0400 Subject: [PATCH 21/56] Clarify where automatic transfer reports are saved --- BitMatch-iPad/Views/ModularContentView.swift | 2 +- BitMatch/Views/PreferencesWindow.swift | 2 +- Shared/Views/MasterReport/MasterReportScreen.swift | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/BitMatch-iPad/Views/ModularContentView.swift b/BitMatch-iPad/Views/ModularContentView.swift index 10d6e7b0..a912af78 100644 --- a/BitMatch-iPad/Views/ModularContentView.swift +++ b/BitMatch-iPad/Views/ModularContentView.swift @@ -360,7 +360,7 @@ struct SettingsSheetView: View { } Section { - Text("A report is a record of what happened during a transfer, saved next to your destinations so you can hand it to anyone.") + Text("Transfer reports are saved in Reports on your first destination.") .font(.footnote) .foregroundColor(.secondary) } header: { diff --git a/BitMatch/Views/PreferencesWindow.swift b/BitMatch/Views/PreferencesWindow.swift index b071ade9..1875e243 100644 --- a/BitMatch/Views/PreferencesWindow.swift +++ b/BitMatch/Views/PreferencesWindow.swift @@ -233,7 +233,7 @@ struct PreferencesWindow: View { } header: { Text("Reports") } footer: { - Text("Reports record what happened and are saved beside each destination.") + Text("Transfer reports are saved in Reports on your first destination.") } if coordinator.reportSettings.makeReport { diff --git a/Shared/Views/MasterReport/MasterReportScreen.swift b/Shared/Views/MasterReport/MasterReportScreen.swift index 067ddb39..f3a5381c 100644 --- a/Shared/Views/MasterReport/MasterReportScreen.swift +++ b/Shared/Views/MasterReport/MasterReportScreen.swift @@ -281,7 +281,7 @@ struct MasterReportScreen: View { VStack(alignment: .leading, spacing: 4) { Text("No BitMatch reports from \(dayText)") .font(.headline) - Text("BitMatch looks for the reports it writes beside each destination, by the day they were written. Pick another day, or a different drive or folder.") + Text("BitMatch looks for saved reports by the day they were written. Pick another day, or a different drive or folder.") .font(.callout) .foregroundStyle(.secondary) .fixedSize(horizontal: false, vertical: true) From 583dcbcae75e7ba5cc74296f43abaa3c97bf12a6 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 22:38:31 -0400 Subject: [PATCH 22/56] Validate filesystem matrix execution and safe cleanup --- Scripts/test-filesystem-fault-matrix.sh | 59 ++++++++++++++++---- Scripts/test_filesystem_matrix_cleanup.py | 66 +++++++++++++++++++++++ 2 files changed, 115 insertions(+), 10 deletions(-) create mode 100644 Scripts/test_filesystem_matrix_cleanup.py diff --git a/Scripts/test-filesystem-fault-matrix.sh b/Scripts/test-filesystem-fault-matrix.sh index 2fcab5e5..8b749259 100755 --- a/Scripts/test-filesystem-fault-matrix.sh +++ b/Scripts/test-filesystem-fault-matrix.sh @@ -2,31 +2,70 @@ # Run the fault/reuse/safety suites with fixtures on each real filesystem. set -euo pipefail ROOT=$(cd "$(dirname "$0")/.." && pwd) -PAIR=$(mktemp -d /tmp/bitmatch-fault-matrix.XXXXXX) -LOG_DIR="$ROOT/dist/validation/fault-matrix-$(date +%Y%m%d-%H%M%S)" -mkdir -p "$LOG_DIR" +source "$ROOT/Scripts/harness_evidence.sh" +PAIR=$(mktemp -d /private/tmp/bitmatch-fault-matrix.XXXXXX) +touch "$PAIR/.bitmatch-fault-matrix-owned" +initialize_evidence filesystem-fault-matrix +LOG_DIR="$EVIDENCE" MOUNTS=() cleanup() { + local status=$? + trap - EXIT INT TERM local busy=0 - for mount in "${MOUNTS[@]}"; do hdiutil detach -quiet "$mount" || busy=1; done - if [[ "$busy" == 0 ]]; then rm -rf "$PAIR"; else echo "Busy images retained: $PAIR" >&2; fi + mount > "$PAIR/mounts-before-cleanup.txt" || busy=1 + for mount in "${MOUNTS[@]}"; do + if grep -F -- " on $mount (" "$PAIR/mounts-before-cleanup.txt" >/dev/null; then + hdiutil detach -quiet "$mount" || busy=1 + fi + done + mount > "$PAIR/mounts-after-cleanup.txt" || busy=1 + if grep -F -- " on $PAIR/" "$PAIR/mounts-after-cleanup.txt" >/dev/null; then busy=1; fi + if [[ "$busy" == 0 && -f "$PAIR/.bitmatch-fault-matrix-owned" ]]; then + rm -rf "$PAIR" || status=1 + else + echo "Busy or unmarked disposable images retained: $PAIR" >&2 + status=1 + fi + finish_evidence "$status" || status=1 + exit "$status" } trap cleanup EXIT -swift build --build-tests --package-path "$ROOT/Packages/BitMatchEngine" > "$LOG_DIR/build.log" 2>&1 +trap 'exit 130' INT +trap 'exit 143' TERM +SWIFT=(swift) +if [[ "$(uname -m)" == arm64 ]]; then SWIFT=(arch -arm64 swift); fi +"${SWIFT[@]}" build --build-tests --jobs 4 --package-path "$ROOT/Packages/BitMatchEngine" \ + --scratch-path "$PAIR/build" > "$LOG_DIR/build.log" 2>&1 failed=0 -for item in 'apfs:APFS' 'hfs:JHFS+' 'exfat:ExFAT' 'apfsx:Case-sensitive APFS' 'fat:MS-DOS FAT32'; do +for item in 'apfs:APFS' 'hfs:Journaled HFS+' 'exfat:ExFAT' 'apfsx:Case-sensitive APFS' 'hfsx:Case-sensitive Journaled HFS+' 'fat:MS-DOS FAT32'; do name=${item%%:*}; fs=${item#*:} mkdir "$PAIR/$name" - hdiutil create -size 3g -type SPARSE -fs "$fs" -volname "BM_${name}" -o "$PAIR/$name.sparseimage" - hdiutil attach -quiet -nobrowse -mountpoint "$PAIR/$name" "$PAIR/$name.sparseimage" + hdiutil create -quiet -size 3g -type SPARSE -fs "$fs" -volname "BM_${name}" -o "$PAIR/$name.sparseimage" + # Record intent before attach so a signal or partial attachment is cleaned. MOUNTS+=("$PAIR/$name") + hdiutil attach -quiet -nobrowse -mountpoint "$PAIR/$name" "$PAIR/$name.sparseimage" + python3 - "$PAIR" "$PAIR/$name" <<'PY' +import os, sys +assert os.stat(sys.argv[1]).st_dev != os.stat(sys.argv[2]).st_dev, 'Expected a mounted image, not a host directory' +PY mkdir "$PAIR/$name/tmp" touch "$PAIR/$name/.bitmatch-fault-owned" expected="${name/apfsx/apfs}" + if [[ "$name" == hfsx ]]; then expected=hfs; fi if [[ "$name" == fat ]]; then expected=msdos; fi - BITMATCH_FAULT_MATRIX_MOUNT="$PAIR/$name" BITMATCH_FAULT_MATRIX_TYPE="$expected" swift test --skip-build --package-path "$ROOT/Packages/BitMatchEngine" \ + BITMATCH_FAULT_MATRIX_MOUNT="$PAIR/$name" BITMATCH_FAULT_MATRIX_TYPE="$expected" "${SWIFT[@]}" test \ + --skip-build --disable-swift-testing --no-parallel --package-path "$ROOT/Packages/BitMatchEngine" --scratch-path "$PAIR/build" \ --filter MountedFilesystemFaultTests \ > "$LOG_DIR/$name.log" 2>&1 || failed=1 + # A framework-loader failure or a zero-test run is not a successful matrix. + expected_skips=1 + if [[ "$name" == exfat || "$name" == fat ]]; then expected_skips=0; fi + python3 - "$LOG_DIR/$name.log" "$expected_skips" <<'PY' || failed=1 +import pathlib, re, sys +text = pathlib.Path(sys.argv[1]).read_text() +summaries = re.findall(r'Executed 12 tests, with (?:(\d+) tests? skipped and )?0 failures', text) +assert summaries and all(int(value or 0) == int(sys.argv[2]) for value in summaries), 'Missing, failed or unexpectedly skipped tests' +PY echo "Filesystem $name tested; log: $LOG_DIR/$name.log" done exit "$failed" diff --git a/Scripts/test_filesystem_matrix_cleanup.py b/Scripts/test_filesystem_matrix_cleanup.py new file mode 100644 index 00000000..c9029b6c --- /dev/null +++ b/Scripts/test_filesystem_matrix_cleanup.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Exercise mounted-matrix cleanup failures without mounting any filesystem.""" +from pathlib import Path +import os +import subprocess +import tempfile +import unittest + + +class FilesystemMatrixCleanupTests(unittest.TestCase): + def exercise(self, detach_fails=False, table_fails=False, finish_fails=False): + temporary = tempfile.TemporaryDirectory(prefix="bitmatch-matrix-cleanup-") + self.addCleanup(temporary.cleanup) + parent = Path(temporary.name) + pair = parent / "owned" + pair.mkdir() + (pair / ".bitmatch-fault-matrix-owned").touch() + sentinel = pair / "fixture.bin" + sentinel.write_bytes(b"owned fixture") + unrelated = parent / "unrelated.bin" + unrelated.write_bytes(b"leave me alone") + source = (Path(__file__).parent / "test-filesystem-fault-matrix.sh").read_text() + cleanup = source[source.index("cleanup() {"):source.index("\ntrap cleanup EXIT")] + environment = dict(os.environ, PAIR=str(pair), DETACH_FAIL=str(int(detach_fails)), + TABLE_FAIL=str(int(table_fails)), FINISH_FAIL=str(int(finish_fails)), + TRACE=str(parent / "trace")) + script = "\n".join([ + "set -euo pipefail", "attached=1", 'MOUNTS=("$PAIR/mount")', + 'mount() { [[ "$TABLE_FAIL" == 0 ]] || return 1; if [[ "$attached" == 1 ]]; then echo "fixture on $PAIR/mount (exfat)"; fi; }', + 'hdiutil() { echo detach >> "$TRACE"; [[ "$DETACH_FAIL" == 0 ]] || return 1; attached=0; }', + 'finish_evidence() { echo "status=$1" >> "$TRACE"; [[ "$FINISH_FAIL" == 0 ]]; }', + cleanup, "trap cleanup EXIT", "exit 0", + ]) + result = subprocess.run(["/bin/bash", "-c", script], env=environment, + capture_output=True, text=True) + self.assertEqual(unrelated.read_bytes(), b"leave me alone") + return pair, sentinel, parent / "trace", result + + def test_success_detaches_before_removing_owned_files(self): + pair, _, trace, result = self.exercise() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(pair.exists()) + self.assertEqual(trace.read_text().splitlines(), ["detach", "status=0"]) + + def test_failed_detach_retains_mounted_files_and_fails(self): + pair, sentinel, trace, result = self.exercise(detach_fails=True) + self.assertEqual(result.returncode, 1) + self.assertTrue(pair.exists()) + self.assertEqual(sentinel.read_bytes(), b"owned fixture") + self.assertIn("status=1", trace.read_text()) + + def test_unreadable_mount_table_retains_files_and_fails(self): + pair, sentinel, trace, result = self.exercise(table_fails=True) + self.assertEqual(result.returncode, 1) + self.assertTrue(pair.exists()) + self.assertEqual(sentinel.read_bytes(), b"owned fixture") + self.assertNotIn("detach", trace.read_text()) + + def test_receipt_failure_remains_a_failed_run_after_cleanup(self): + pair, _, _, result = self.exercise(finish_fails=True) + self.assertEqual(result.returncode, 1) + self.assertFalse(pair.exists()) + + +if __name__ == "__main__": + unittest.main() From 5343e4f3ed4cb26a4f0fea1607f9498ce0af1192 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 22:41:08 -0400 Subject: [PATCH 23/56] Keep Mac transfer settings reviews compact and scrollable --- ...ortableTransferSettingsSnapshotTests.swift | 98 +++++++++++++ CHANGELOG.md | 1 + Shared/Views/TransferSettingsSection.swift | 138 +++++++++++++----- 3 files changed, 203 insertions(+), 34 deletions(-) diff --git a/BitMatchTests/PortableTransferSettingsSnapshotTests.swift b/BitMatchTests/PortableTransferSettingsSnapshotTests.swift index 0b3894c8..2f369b94 100644 --- a/BitMatchTests/PortableTransferSettingsSnapshotTests.swift +++ b/BitMatchTests/PortableTransferSettingsSnapshotTests.swift @@ -7,6 +7,104 @@ import BitMatchEngine @MainActor final class PortableTransferSettingsSnapshotTests: XCTestCase { + func testMacImportReviewLayout() throws { + let environment = ProcessInfo.processInfo.environment + guard environment["BITMATCH_CAPTURE_WORKFLOW_SNAPSHOTS"] == "1" + || environment["TEST_RUNNER_BITMATCH_CAPTURE_WORKFLOW_SNAPSHOTS"] == "1" else { + throw XCTSkip("Enable workflow snapshots for settings captures") + } + let current = PortableTransferSettings(verificationMode: .standard, generateASCMHL: true, + ignoreAppleDouble: true, reports: .init(makeReport: true, includeThumbnails: false), + naming: .init(position: .prefix, separator: .underscore, autoNumber: true, groupByCamera: false)) + let full = PortableTransferSettings(verificationMode: .quick, generateASCMHL: false, + ignoreAppleDouble: false, reports: .init(makeReport: false, includeThumbnails: true), + naming: .init(position: .suffix, separator: .dash, autoNumber: false, groupByCamera: true)) + let few = PortableTransferSettings(verificationMode: current.verificationMode, + generateASCMHL: current.generateASCMHL, ignoreAppleDouble: current.ignoreAppleDouble, + reports: full.reports, naming: current.naming) + let fullReview = TransferSettingsReview(current: current, proposed: full) + let fewReview = TransferSettingsReview(current: current, proposed: few) + XCTAssertEqual(fullReview.changes.count, 9) + XCTAssertEqual(fewReview.changes.count, 2) + let fullHeight = try captureReview(fullReview, name: "mac-import-nine-changes", requiresScroll: true) + let fewHeight = try captureReview(fewReview, name: "mac-import-two-changes", requiresScroll: false) + XCTAssertLessThan(fewHeight, fullHeight - 80, "A short review should size to its content") + } + + private func captureReview(_ review: TransferSettingsReview, name: String, requiresScroll: Bool) throws -> CGFloat { + let host = NSHostingView(rootView: TransferSettingsReviewView(review: review, apply: {}, cancel: {}) + .background(Color(nsColor: .windowBackgroundColor)) + .preferredColorScheme(.dark).transaction { $0.disablesAnimations = true }) + let window = NSWindow(contentRect: CGRect(x: 0, y: 0, width: 480, height: 600), + styleMask: [.titled], backing: .buffered, defer: true) + window.isReleasedWhenClosed = false + window.appearance = NSAppearance(named: .darkAqua) + window.contentView = host + window.orderFront(nil) + defer { window.orderOut(nil); window.close() } + RunLoop.main.run(until: Date().addingTimeInterval(0.3)) + let height = host.fittingSize.height + XCTAssertGreaterThan(height, 180) + XCTAssertLessThanOrEqual(height, 600) + window.setContentSize(CGSize(width: 480, height: height)) + host.layoutSubtreeIfNeeded() + RunLoop.main.run(until: Date().addingTimeInterval(0.2)) + var seen = Set() + var actionBounds: [String: CGRect] = [:] + func record(_ suffix: String) throws { + let bitmap = try XCTUnwrap(host.bitmapImageRepForCachingDisplay(in: host.bounds)) + host.cacheDisplay(in: host.bounds, to: bitmap) + let request = VNRecognizeTextRequest() + request.recognitionLevel = .accurate + try VNImageRequestHandler(cgImage: try XCTUnwrap(bitmap.cgImage)).perform([request]) + let observations = request.results ?? [] + for observation in observations { + if let text = observation.topCandidates(1).first?.string.lowercased() { seen.insert(text) } + } + let heading = try XCTUnwrap(observations.first { + $0.topCandidates(1).first?.string.lowercased() == "import settings" + }, "The review title must remain visible") + let headingInset = (1 - heading.boundingBox.maxY) * height + XCTAssertGreaterThan(headingInset, 10) + XCTAssertLessThan(headingInset, 60, "The title belongs at the top, without an empty vertical band") + for title in ["cancel", "apply"] { + let action = try XCTUnwrap(observations.first { $0.topCandidates(1).first?.string.lowercased() == title }) + XCTAssertLessThan(action.boundingBox.maxY * height, 65, "Review actions must stay at the bottom") + if let previous = actionBounds[title] { + XCTAssertEqual(action.boundingBox.midY, previous.midY, accuracy: 0.005, "Actions must not scroll") + } else { actionBounds[title] = action.boundingBox } + } + let data = try XCTUnwrap(bitmap.representation(using: .png, properties: [:])) + let attachment = XCTAttachment(data: data, uniformTypeIdentifier: "public.png") + attachment.name = name + suffix + attachment.lifetime = .keepAlways + add(attachment) + } + try record("-top") + if requiresScroll { + let scroll = try XCTUnwrap(findOverflowingScrollView(in: host), "Nine changes need a scrollable review") + let document = try XCTUnwrap(scroll.documentView) + let extent = max(0, document.bounds.height - scroll.contentView.bounds.height) + for (suffix, fraction) in [("-middle", 0.5), ("-bottom", 1.0)] { + let offset = document.isFlipped ? extent * fraction : extent * (1 - fraction) + scroll.contentView.scroll(to: CGPoint(x: 0, y: offset)) + scroll.reflectScrolledClipView(scroll.contentView) + RunLoop.main.run(until: Date().addingTimeInterval(0.2)) + host.layoutSubtreeIfNeeded() + try record(suffix) + } + } else { + XCTAssertNil(findOverflowingScrollView(in: host), "Two changes should fit without a tall empty sheet") + } + let text = seen.joined(separator: " ") + for change in review.changes { XCTAssertTrue(text.contains(change.title.lowercased()), "Unreachable change: \(change.title)") } + if review.proposed.verificationMode == .quick { + XCTAssertTrue(text.contains("does not verify contents"), "Quick assurance warning must be reachable") + } + XCTAssertTrue(text.contains("changes apply to future transfers")) + return height + } + func testCaptureMacTransferSettings() throws { let environment = ProcessInfo.processInfo.environment guard environment["BITMATCH_CAPTURE_WORKFLOW_SNAPSHOTS"] == "1" diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ef993d2..2f829bae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. - Destination progress no longer resets to Waiting while ASC MHL and transfer reports are being saved. Processed counts and measured progress stay visible without implying the transfer is complete. - Mac Settings now scroll when a pane is taller than the window, keeping the first and last controls reachable. +- Importing transfer settings on Mac uses a compact review sheet, with longer change lists scrollable above Cancel and Apply. - Move reusable transfer settings between Mac, iPad and iPhone from Settings. Imports show the changes before Apply and leave backup locations, camera labels, private report details and saved transfers alone. - Older History entries without a saved selected-file plan now stay visible but do not claim the whole card is safe to erase, even if their saved rows say verified. They also no longer trigger the “Backed up before…” hint. Quick entries without that plan show Needs attention instead of a clean size-only copy. History exports keep the original message and add the saved safety verdict. - If a new transfer cannot save its selected-file plan, BitMatch stops before creating a completion report or marking a project card safe. The final record keeps the source unsafe. Finish, queue, menu bar, Dock and notifications follow the saved verdict. A later History-save failure also shows an interruption; any separately saved report or project evidence remains available for review. diff --git a/Shared/Views/TransferSettingsSection.swift b/Shared/Views/TransferSettingsSection.swift index d63cba58..1a76504b 100644 --- a/Shared/Views/TransferSettingsSection.swift +++ b/Shared/Views/TransferSettingsSection.swift @@ -79,45 +79,47 @@ struct TransferSettingsReviewView: View { var errorMessage: String? = nil let apply: () -> Void let cancel: () -> Void + #if os(macOS) + @State private var contentHeight: CGFloat = 440 + #endif var body: some View { - NavigationStack { - Form { - if review.changes.isEmpty { - Section { Text("These transfer settings already match.") } - } else { - Section("Changes") { - ForEach(review.changes) { change in - VStack(alignment: .leading, spacing: 4) { - Text(change.title) - Text("\(change.before) → \(change.after)") - .font(.subheadline).foregroundStyle(.secondary) - } - .padding(.vertical, 2) - .accessibilityElement(children: .combine) + #if os(macOS) + VStack(alignment: .leading, spacing: 0) { + Text("Import settings") + .font(.title2.weight(.semibold)) + .accessibilityAddTraits(.isHeader) + .padding(.horizontal, 24) + .padding(.top, 24) + .padding(.bottom, 4) + ScrollView { + macReviewContent + .background { + GeometryReader { proxy in + Color.clear.preference(key: TransferSettingsReviewHeightKey.self, value: proxy.size.height) } } - } - if review.proposed.verificationMode == .quick { - Section { - Label("Quick checks file sizes. It does not verify contents.", systemImage: "exclamationmark.triangle") - .foregroundStyle(.orange) - } - } - if let errorMessage { - Section { Text(errorMessage).foregroundStyle(.red) } - } - Section { - Text("Changes apply to future transfers. Backups, project details and saved transfers stay as they are.") - .font(.footnote).foregroundStyle(.secondary) - } } - #if os(iOS) + // Size short reviews to their content; keep long lists and errors + // in a bounded viewport while the actions remain visible. + .frame(height: min(440, contentHeight)) + Divider() + HStack { + Spacer() + Button("Cancel", action: cancel).keyboardShortcut(.cancelAction) + Button("Apply", action: apply).keyboardShortcut(.defaultAction) + .disabled(review.changes.isEmpty) + } + .padding(.horizontal, 24) + .padding(.vertical, 16) + } + .frame(width: 480) + .onPreferenceChange(TransferSettingsReviewHeightKey.self) { contentHeight = $0 } + #else + NavigationStack { + Form { reviewFields } .navigationTitle("Import") .navigationBarTitleDisplayMode(.inline) - #else - .navigationTitle("Import settings") - #endif .toolbar { ToolbarItem(placement: .cancellationAction) { Button("Cancel", action: cancel) } ToolbarItem(placement: .confirmationAction) { @@ -125,11 +127,79 @@ struct TransferSettingsReviewView: View { } } } - #if os(macOS) - .frame(width: 480, height: 600) #endif } + + #if os(macOS) + private var macReviewContent: some View { + VStack(alignment: .leading, spacing: 16) { + if review.changes.isEmpty { + Text("These transfer settings already match.") + } else { + GroupBox("Changes") { + VStack(alignment: .leading, spacing: 0) { + ForEach(review.changes) { change in + changeRow(change) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.vertical, 8) + if change.id != review.changes.last?.id { Divider() } + } + } + .padding(.horizontal, 8) + } + } + if review.proposed.verificationMode == .quick { quickWarning.fixedSize(horizontal: false, vertical: true) } + if let errorMessage { Text(errorMessage).foregroundStyle(.red) } + preservationNote.fixedSize(horizontal: false, vertical: true) + } + .fixedSize(horizontal: false, vertical: true) + .padding(.horizontal, 24) + .padding(.vertical, 16) + } + #endif + + @ViewBuilder + private var reviewFields: some View { + if review.changes.isEmpty { + Section { Text("These transfer settings already match.") } + } else { + Section("Changes") { + ForEach(review.changes) { change in + changeRow(change).padding(.vertical, 2) + } + } + } + if review.proposed.verificationMode == .quick { Section { quickWarning } } + if let errorMessage { Section { Text(errorMessage).foregroundStyle(.red) } } + Section { preservationNote } + } + + private func changeRow(_ change: PortableTransferSettings.Change) -> some View { + VStack(alignment: .leading, spacing: 4) { + Text(change.title) + Text("\(change.before) → \(change.after)") + .font(.subheadline).foregroundStyle(.secondary) + } + .accessibilityElement(children: .combine) + } + + private var quickWarning: some View { + Label("Quick checks file sizes. It does not verify contents.", systemImage: "exclamationmark.triangle") + .foregroundStyle(.orange) + } + + private var preservationNote: some View { + Text("Changes apply to future transfers. Backups, project details and saved transfers stay as they are.") + .font(.footnote).foregroundStyle(.secondary) + } +} + +#if os(macOS) +private struct TransferSettingsReviewHeightKey: PreferenceKey { + static let defaultValue: CGFloat = 0 + static func reduce(value: inout CGFloat, nextValue: () -> CGFloat) { value = max(value, nextValue()) } } +#endif private struct TransferSettingsDocument: FileDocument { static var readableContentTypes: [UTType] { [.json] } From 4f1488836d07aeb006d4259a4f082f6d4f563209 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 22:47:25 -0400 Subject: [PATCH 24/56] Bound diagnostic export reads and report rejected inputs --- .../TransferDiagnosticStore.swift | 63 +++++++- .../TransferDiagnosticStoreTests.swift | 135 ++++++++++++++++++ docs/DIAGNOSTICS.md | 2 +- 3 files changed, 196 insertions(+), 4 deletions(-) diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferDiagnosticStore.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferDiagnosticStore.swift index a57450aa..febb57cf 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferDiagnosticStore.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferDiagnosticStore.swift @@ -1,6 +1,8 @@ import Foundation #if canImport(Darwin) import Darwin +#elseif canImport(Glibc) +import Glibc #endif /// A bounded, local record of structured events only. Never accepts free-form @@ -58,6 +60,18 @@ public final class TransferDiagnosticStore: @unchecked Sendable { private var current: URL { directory.appendingPathComponent("current.jsonl") } private var previous: URL { directory.appendingPathComponent("previous.jsonl") } + private enum InputFile: String, Encodable { case previous, current } + private enum InputRejection: String, Encodable { case unreadable, oversized, unsupportedFileType } + private struct RejectedInput: Encodable { + let file: InputFile + let reason: InputRejection + } + private enum InputRead { + case missing + case data(Data) + case rejected(InputRejection) + } + init(directory: URL, limit: Int = 256 * 1024, systemEvidenceStore: LocalCrashEvidenceStore = .shared) { self.directory = directory @@ -133,8 +147,16 @@ public final class TransferDiagnosticStore: @unchecked Sendable { decoder.dateDecodingStrategy = .iso8601 var events: [Event] = [] var incompleteRecords = 0 - for file in [previous, current] where FileManager.default.fileExists(atPath: file.path) { - let data = try Data(contentsOf: file) + var rejectedInputs: [RejectedInput] = [] + for (slot, url) in [(InputFile.previous, previous), (.current, current)] { + let data: Data + switch readInput(url) { + case .missing: continue + case .rejected(let reason): + rejectedInputs.append(RejectedInput(file: slot, reason: reason)) + continue + case .data(let contents): data = contents + } for line in data.split(separator: 0x0A) { do { events.append(try decoder.decode(Event.self, from: Data(line))) } catch { incompleteRecords += 1 } @@ -148,6 +170,7 @@ public final class TransferDiagnosticStore: @unchecked Sendable { let build: String let recordingHadErrors: Bool let incompleteRecords: Int + let rejectedInputs: [RejectedInput] let events: [Event] let systemEvidenceEnabled: Bool let systemEvidenceRecordingHadErrors: Bool @@ -163,11 +186,45 @@ public final class TransferDiagnosticStore: @unchecked Sendable { operatingSystem: ProcessInfo.processInfo.operatingSystemVersionString, appVersion: Bundle.main.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String ?? "unknown", build: Bundle.main.object(forInfoDictionaryKey: "CFBundleVersion") as? String ?? "unknown", - recordingHadErrors: writeFailed, incompleteRecords: incompleteRecords, events: events, + recordingHadErrors: writeFailed || incompleteRecords > 0 || !rejectedInputs.isEmpty, + incompleteRecords: incompleteRecords, rejectedInputs: rejectedInputs, events: events, systemEvidenceEnabled: systemEvidence.enabled, systemEvidenceRecordingHadErrors: systemEvidence.recordingHadErrors, systemEvidenceDeletionHadErrors: systemEvidence.deletionHadErrors, systemEvidenceNote: "Apple MetricKit may deliver crash or main-thread hang reports later, or not at all. Absence does not rule out a crash, memory kill, force quit, or background stall. Reports are not tied to a transfer by this export.", systemEvidence: systemEvidence.records)) } + + /// Bound allocation even if a local file was damaged, replaced or grew + /// outside the writer's rotation lock. Nonblocking/no-follow open prevents + /// a FIFO or link from turning Export diagnostics into an unrelated read. + private func readInput(_ url: URL) -> InputRead { +#if canImport(Darwin) || canImport(Glibc) + let fd = open(url.path, O_RDONLY | O_NONBLOCK | O_NOFOLLOW | O_CLOEXEC) + guard fd >= 0 else { + if errno == ENOENT { return .missing } + return .rejected(errno == ELOOP ? .unsupportedFileType : .unreadable) + } + let handle = FileHandle(fileDescriptor: fd, closeOnDealloc: true) + defer { try? handle.close() } + var info = stat() + guard fstat(fd, &info) == 0, info.st_size >= 0 else { return .rejected(.unreadable) } + guard (info.st_mode & mode_t(S_IFMT)) == mode_t(S_IFREG) else { return .rejected(.unsupportedFileType) } + // Production files are at most 256 KiB each. Honor smaller test limits + // without allowing a larger caller-supplied limit to remove this bound. + let byteLimit = min(max(0, limit), 256 * 1024) + guard info.st_size <= byteLimit else { return .rejected(.oversized) } + do { + var data = Data() + while true { + let chunk = try handle.read(upToCount: min(64 * 1024, byteLimit + 1 - data.count)) ?? Data() + guard !chunk.isEmpty else { return .data(data) } + data.append(chunk) + guard data.count <= byteLimit else { return .rejected(.oversized) } + } + } catch { return .rejected(.unreadable) } +#else + return .rejected(.unreadable) +#endif + } } diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TransferDiagnosticStoreTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TransferDiagnosticStoreTests.swift index b35fe8e6..ed61bdba 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TransferDiagnosticStoreTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TransferDiagnosticStoreTests.swift @@ -1,5 +1,8 @@ import Foundation import XCTest +#if canImport(Darwin) +import Darwin +#endif @testable import BitMatchEngine final class TransferDiagnosticStoreTests: XCTestCase { @@ -9,6 +12,138 @@ final class TransferDiagnosticStoreTests: XCTestCase { addTeardownBlock { try? FileManager.default.removeItem(at: root) } return root } + + private func previousEvent(in root: URL, limit: Int = 1024) throws -> TransferDiagnosticStore { + let store = TransferDiagnosticStore(directory: root, limit: limit) + store.record(.pipelineDrained, run: nil) { $0.code = 17 } + try FileManager.default.moveItem(at: root.appendingPathComponent("current.jsonl"), + to: root.appendingPathComponent("previous.jsonl")) + return store + } + + private func exportedObject(_ store: TransferDiagnosticStore) throws -> [String: Any] { + try XCTUnwrap(JSONSerialization.jsonObject(with: store.exportData()) as? [String: Any]) + } + + func testOversizedInputIsRejectedWithoutDiscardingOtherFile() throws { + let root = try fixture() + let store = try previousEvent(in: root) + try Data(repeating: 0, count: 1025).write(to: root.appendingPathComponent("current.jsonl")) + let object = try exportedObject(store) + let events = try XCTUnwrap(object["events"] as? [[String: Any]]) + XCTAssertEqual(events.compactMap { $0["code"] as? Int }, [17]) + XCTAssertEqual(object["formatVersion"] as? Int, 2) + XCTAssertEqual(object["recordingHadErrors"] as? Bool, true) + XCTAssertEqual(object["incompleteRecords"] as? Int, 0, "An unread file has an unknown record count") + let rejected = try XCTUnwrap(object["rejectedInputs"] as? [[String: String]]) + XCTAssertEqual(rejected, [["file": "current", "reason": "oversized"]]) + } + + func testOversizedPreviousDoesNotDiscardValidCurrent() throws { + let root = try fixture() + let store = TransferDiagnosticStore(directory: root, limit: 1024) + store.record(.pipelineDrained, run: nil) { $0.code = 19 } + try Data(repeating: 0, count: 1025).write(to: root.appendingPathComponent("previous.jsonl")) + let object = try exportedObject(store) + XCTAssertEqual((object["events"] as? [[String: Any]])?.compactMap { $0["code"] as? Int }, [19]) + XCTAssertEqual(object["recordingHadErrors"] as? Bool, true) + XCTAssertEqual(object["rejectedInputs"] as? [[String: String]], [["file": "previous", "reason": "oversized"]]) + } + + func testValidInputAtByteLimitIsNotTruncated() throws { + let root = try fixture() + let file = root.appendingPathComponent("current.jsonl") + TransferDiagnosticStore(directory: root).record(.pipelineDrained, run: nil) { $0.code = 17 } + let line = try Data(contentsOf: file) + var data = Data() + for _ in 0..<400 { data.append(line) } + XCTAssertGreaterThan(data.count, 64 * 1024) + XCTAssertLessThanOrEqual(data.count, 256 * 1024) + try data.write(to: file) + let object = try exportedObject(TransferDiagnosticStore(directory: root, limit: data.count)) + XCTAssertEqual((object["events"] as? [Any])?.count, 400) + XCTAssertEqual(object["recordingHadErrors"] as? Bool, false) + XCTAssertEqual(object["incompleteRecords"] as? Int, 0) + XCTAssertEqual(object["rejectedInputs"] as? [[String: String]], []) + } + + func testMalformedInputFlagsErrorWithoutDiscardingEarlierEvents() throws { + let root = try fixture() + let store = try previousEvent(in: root) + try Data("{unfinished SECRET_FOOTAGE".utf8).write(to: root.appendingPathComponent("current.jsonl")) + let object = try exportedObject(store) + XCTAssertEqual((object["events"] as? [[String: Any]])?.compactMap { $0["code"] as? Int }, [17]) + XCTAssertEqual(object["recordingHadErrors"] as? Bool, true) + XCTAssertEqual(object["incompleteRecords"] as? Int, 1) + XCTAssertEqual(object["rejectedInputs"] as? [[String: String]], []) + XCTAssertFalse(String(decoding: try store.exportData(), as: UTF8.self).contains("SECRET_FOOTAGE")) + } + + func testMissingInputsAreOrdinaryEmptyHistory() throws { + let object = try exportedObject(TransferDiagnosticStore(directory: try fixture())) + XCTAssertEqual((object["events"] as? [Any])?.count, 0) + XCTAssertEqual(object["recordingHadErrors"] as? Bool, false) + XCTAssertEqual(object["rejectedInputs"] as? [[String: String]], []) + } + + func testSymlinkInputIsNotFollowed() throws { + let root = try fixture() + let store = try previousEvent(in: root) + let outside = root.appendingPathComponent("outside") + let other = TransferDiagnosticStore(directory: outside) + other.record(.admitted, run: nil) { $0.code = 99 } + let original = try Data(contentsOf: outside.appendingPathComponent("current.jsonl")) + try FileManager.default.createSymbolicLink(at: root.appendingPathComponent("current.jsonl"), + withDestinationURL: outside.appendingPathComponent("current.jsonl")) + let object = try exportedObject(store) + XCTAssertEqual((object["events"] as? [[String: Any]])?.compactMap { $0["code"] as? Int }, [17]) + XCTAssertEqual(object["recordingHadErrors"] as? Bool, true) + XCTAssertEqual(object["rejectedInputs"] as? [[String: String]], [["file": "current", "reason": "unsupportedFileType"]]) + XCTAssertEqual(try Data(contentsOf: outside.appendingPathComponent("current.jsonl")), original) + } + +#if canImport(Darwin) + func testFIFOInputIsRejectedWithoutWaitingForWriter() throws { + let root = try fixture() + let store = try previousEvent(in: root) + let path = root.appendingPathComponent("current.jsonl").path + XCTAssertEqual(mkfifo(path, 0o600), 0) + let finished = DispatchSemaphore(value: 0) + let watchdogDone = DispatchSemaphore(value: 0) + // A regression to a blocking reader must fail without hanging the + // suite. This releases such a reader; correctness uses the rejection + // below, not a hardware-dependent elapsed-time threshold. + DispatchQueue.global().async { + defer { watchdogDone.signal() } + if finished.wait(timeout: .now() + 5) == .timedOut { + let writer = Darwin.open(path, O_WRONLY | O_NONBLOCK | O_NOFOLLOW | O_CLOEXEC) + if writer >= 0 { Darwin.close(writer) } + } + } + defer { + finished.signal() + XCTAssertEqual(watchdogDone.wait(timeout: .now() + 10), .success) + } + let object = try exportedObject(store) + XCTAssertEqual((object["events"] as? [[String: Any]])?.compactMap { $0["code"] as? Int }, [17]) + XCTAssertEqual(object["recordingHadErrors"] as? Bool, true) + XCTAssertEqual(object["rejectedInputs"] as? [[String: String]], [["file": "current", "reason": "unsupportedFileType"]]) + } + + func testPermissionDeniedInputDoesNotHideOtherFile() throws { + guard geteuid() != 0 else { throw XCTSkip("Root cannot enforce a fixture's read-permission denial") } + let root = try fixture() + let store = try previousEvent(in: root) + let current = root.appendingPathComponent("current.jsonl") + try Data("not readable".utf8).write(to: current) + XCTAssertEqual(chmod(current.path, 0o000), 0) + defer { _ = chmod(current.path, 0o600) } + let object = try exportedObject(store) + XCTAssertEqual((object["events"] as? [[String: Any]])?.compactMap { $0["code"] as? Int }, [17]) + XCTAssertEqual(object["recordingHadErrors"] as? Bool, true) + XCTAssertEqual(object["rejectedInputs"] as? [[String: String]], [["file": "current", "reason": "unreadable"]]) + } +#endif func testEventsSurviveReopeningAndContainOnlyStructuredFields() throws { let root = try fixture() let run = UUID() diff --git a/docs/DIAGNOSTICS.md b/docs/DIAGNOSTICS.md index 31758794..4f53361f 100644 --- a/docs/DIAGNOSTICS.md +++ b/docs/DIAGNOSTICS.md @@ -19,7 +19,7 @@ If the device refuses **every** consent write and deletion at once, a later laun MetricKit delivery for BitMatch's Developer ID/notarized Mac distribution has not yet been demonstrated. A signed Mac pilot is required before treating this as a dependable field diagnostic. This feature does not establish the causes of [issue #10](https://github.com/BitmatchApp/Bitmatch/issues/10) or [issue #23](https://github.com/BitmatchApp/Bitmatch/issues/23). -The transfer-event record keeps at most two 256 KiB files. Export soon after the problem, before later activity replaces the older events. A recording failure is reported in the export; diagnostics never change the result of a transfer. +The transfer-event record keeps at most two 256 KiB files. Export soon after the problem, before later activity replaces the older events. Export also limits its input reads: an oversized, unreadable or unsupported file is listed in `rejectedInputs` by its fixed current/previous slot and reason, while valid events from the other file remain available. Malformed individual records are counted separately in `incompleteRecords`. Either case sets `recordingHadErrors`; an absent history file is normal. These flags never change the result of a transfer. Screenshots, transfer reports and Check's differences export can contain names and paths. Review those separately before sharing them; they are not the same as the diagnostics export. From bd43a810601f2ae6ab41a7a705228b1c3854788d Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 22:49:46 -0400 Subject: [PATCH 25/56] Align the guide and hardware checks with saved evidence rules --- README.md | 2 +- docs/GUIDE.md | 6 ++++-- docs/HARDWARE_TESTING.md | 6 +++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index cd002883..c1207f43 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ For new copies, Standard does not independently reread the source: a wrong sourc ## What It Does - **Multi destination copy**: for new files, the copy pass reads the card once and writes to every drive at the same time. Verification reads each drive back; reused files get a fresh source read, and Thorough and Paranoid reread the card too. Each destination gets its own results. -- **Careful about "safe to erase"**: a card or folder that changes during the copy, or online-only iCloud/Dropbox files, won't earn a safe verdict. Two folders on the same physical drive count as one backup, not two independent ones. Recognized AppleDouble `._` metadata companions are skipped by default; other hidden files and camera sidecars stay in the copy. Turn off **Ignore AppleDouble metadata files** in **Settings → File handling** to preserve them. That copy can verify the selected files, but it won't mark the whole card safe to erase. +- **Careful about "safe to erase"**: a card or folder that changes during the copy, or online-only iCloud/Dropbox files, won't earn a safe verdict. Two folders on the same physical drive count as one backup, not two independent ones. Recognized AppleDouble `._` metadata companions are skipped by default; other hidden files and camera sidecars stay in the copy. Turn off **Ignore AppleDouble metadata files** in **Settings → File handling** to preserve them. When files are excluded, BitMatch can verify the selected files, but it won't mark the whole card safe to erase. - **Photographer jobs** for shoots with several photographers, cameras, and cards. Save the setup instead of rebuilding it every time. - **Camera detection** for Sony, Canon, ARRI, RED, Blackmagic, Panasonic, Fujifilm, GoPro, DJI, Insta360, and generic DCIM. - **Check** for stuff you already copied: compare two folders, or check a backup against its saved checksums. diff --git a/docs/GUIDE.md b/docs/GUIDE.md index ae769fda..465a859a 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -49,7 +49,7 @@ This moves options, not a job: it leaves out backup locations, camera labels, fo Choose **Check**, then pick the folder or drive you want to check. Use **Its saved checksums** when BitMatch finds supported records, such as a BitMatch JSON report or an ASC MHL inventory. It reads the files again and compares them with those records; the original card does not need to be connected. -For ASC MHL, this checks recorded file contents, including inventories inside card subfolders. It does not verify the history chain or directory hashes. If a record is damaged or includes file checksums BitMatch cannot read, Check says so instead of silently checking only part of it. Standalone classic MHL and C4-only file checksums aren't supported yet. +For ASC MHL, this checks recorded file contents, including inventories inside card subfolders. When an inventory chain is present, Check also validates its supported records. Older inventories without a chain get a contents-only check. Directory hashes, recorded renames and chain of custody aren't verified. If a record is damaged or includes checksums BitMatch cannot read, Check says so instead of silently checking only part of it. Standalone classic MHL and C4-only file checksums aren't supported yet. Use **Another folder** to compare two copies instead. Standard compares contents with SHA-256; Quick checks sizes only. The results separate changed files from files missing or extra in the other folder. An extra file alone does not mean the matching files are damaged. Check does not copy, delete, or repair anything. @@ -59,6 +59,8 @@ The differences export includes filenames and paths. For a bug report without th Transfer reports record the results for each destination. PDF, CSV, and JSON exports are available on Mac, iPad, and iPhone. **Master Report** scans a selected folder or drive for saved transfer reports for a chosen day and combines them into a PDF and JSON summary. It summarizes those records; it does not re-verify the media. Use **Check** for that. +Automatic transfer reports go in **Reports** on the first destination. Each destination keeps its own ASC MHL record when enabled. + **Advanced → Transfer details** has optional operator and notes fields. They're remembered for the next card; use **Clear details** when the job changes. Each queued card keeps the details it had when you added it. PDF, CSV, JSON and ASC MHL carry those saved details, including exports from History. They stay out of diagnostic logs. ## ASC MHL @@ -103,7 +105,7 @@ You can ignore SFTP entirely and keep everything local. No app analytics uploads ## Before You Trust It With Your Footage -BitMatch reads the source without writing to it. It copies through temporary files and refuses destination files it finds already in place. Checksum modes reuse an existing file only after verifying that it matches. +BitMatch reads the source without writing to it and never overwrites existing destination files. Checksum modes reuse an existing file only after verifying that it matches. On exFAT and FAT, unfinished copies keep their partial bytes and a visible incomplete-transfer warning. Source scanning rejects unreadable metadata, unsafe paths, and portable filename collisions. Apart from the AppleDouble setting and designated macOS metadata folders at a volume root, hidden files and empty folders are preserved; symlink entries are skipped. Verification rejects files that change size or identity while being read. Completion screens and reports count failures and sidecars too. diff --git a/docs/HARDWARE_TESTING.md b/docs/HARDWARE_TESTING.md index 48353532..7cda571c 100644 --- a/docs/HARDWARE_TESTING.md +++ b/docs/HARDWARE_TESTING.md @@ -23,7 +23,7 @@ Start a Standard transfer to two destinations, wait until active copying is visi ### One-destination removal -Start a Standard transfer to two destinations and remove one throwaway destination while leaving the other connected. The removed destination must report failures; the remaining destination must finish and every successful output must match the manifest. Inspect both destinations for `.bitmatch.tmp.*` files after reconnecting the removed device. +Start a Standard transfer to two destinations and remove one throwaway destination while leaving the other connected. The removed destination must report failures; the remaining destination must finish and every successful output must match the manifest. Inspect retained files and incomplete-status warnings after reconnecting the removed device. A retained partial file must never be presented as verified. ### Sleep and backgrounding @@ -31,7 +31,7 @@ During copy and again during verification, put the Mac to sleep and wake it afte ### Cancel and relaunch -Cancel during copy, confirm activity stops, then quit and relaunch BitMatch. Repeat cancellation during verification. Confirm no temporary files remain and no canceled or failed row is classified as successful. A relaunch must not silently convert an incomplete transfer into success. +Cancel during copy, confirm activity stops, then quit and relaunch BitMatch. Repeat cancellation during verification. Confirm no canceled or failed row is classified as successful. On exFAT/FAT, an interrupted copy deliberately retains its partial bytes and incomplete-status warning; check that recovery refuses to treat them as complete. A relaunch must not silently convert an incomplete transfer into success. ### Low-power hubs @@ -39,7 +39,7 @@ Repeat the one-destination-removal and sleep cases through the approved low-powe ## Evidence to retain -For every run, retain the seed or source manifest, BitMatch result export, independent post-run hashes, exact fault timing, expected outcome, actual outcome, and photos or topology notes that identify the devices. Treat a hang, silent retry, stale success row, unexpected overwrite, or leftover temporary file as a failure even if another destination completes. +For every run, retain the seed or source manifest, BitMatch result export, independent post-run hashes, exact fault timing, expected outcome, actual outcome, and photos or topology notes that identify the devices. Treat a hang, silent retry, stale success row, unexpected overwrite, or partial file without its required warning as a failure even if another destination completes. Retained incomplete output is evidence to inspect, not a successful copy. The automated APFS image test in `Scripts/run_apfs_fault_tests.sh` is a safer first step, but it does not simulate cable, bridge firmware, power-loss, or exFAT behavior. Passing it is not evidence that physical removal is safe. From 9669609bd8708f83255bf26ea1a04b71634e0a7c Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 22:48:27 -0400 Subject: [PATCH 26/56] Add read-only counters for marked test drives --- Scripts/sample-device-io.swift | 331 +++++++++++++++++++++++++++++++++ docs/HARDWARE_TESTING.md | 49 +++++ 2 files changed, 380 insertions(+) create mode 100644 Scripts/sample-device-io.swift diff --git a/Scripts/sample-device-io.swift b/Scripts/sample-device-io.swift new file mode 100644 index 00000000..dbc29711 --- /dev/null +++ b/Scripts/sample-device-io.swift @@ -0,0 +1,331 @@ +// Read-only macOS observer for explicitly marked hardware-test volumes. +// Build/run instructions and counter limits: docs/HARDWARE_TESTING.md. +import Darwin +import DiskArbitration +import Foundation +import IOKit + +private let roles = ["sd", "ssdA", "ssdB"] +private enum ObserverError: String, Error { + case invalidArguments, invalidConfiguration, changedRoot, volumeUnavailable + case wrongVolume, missingDriver, ambiguousDriver, missingCounters +} +private struct Volume: Decodable, Equatable { + let nonce: String, root: String, mount: String, volumeUUID: String, filesystem: String + let initialFreeBytes: Int64 +} +private struct Config: Decodable { let nonce: String; let volumes: [String: Volume] } +private struct Counters: Codable, Equatable { + let readBytes: UInt64, writeBytes: UInt64 + static func parse(_ properties: [String: Any]) -> Self? { + func number(_ key: String) -> UInt64? { + guard let value = properties[key] as? NSNumber, + CFGetTypeID(value) == CFNumberGetTypeID() else { return nil } + return UInt64(value.stringValue) + } + // Public IOBlockStorageDriver.h statistics, cumulative since driver + // instantiation. A missing value is not a valid zero counter. + guard let read = number("Bytes (Read)"), let write = number("Bytes (Write)") else { return nil } + return Self(readBytes: read, writeBytes: write) + } +} +private struct Driver { + let identity: UInt64 // Never serialized or printed. + let counters: Counters + static func unique(_ candidates: [Self]) throws -> Self { + guard !candidates.isEmpty else { throw ObserverError.missingDriver } + guard candidates.count == 1 else { throw ObserverError.ambiguousDriver } + return candidates[0] + } +} +private func sharedDriverRoles(_ drivers: [String: Driver]) -> Set { + Set(drivers.compactMap { role, driver in + drivers.values.filter { $0.identity == driver.identity }.count > 1 ? role : nil + }) +} +private struct Row: Encodable { + let role: String, status: String + var reason: String? + var readBytes: UInt64?, writeBytes: UInt64? + var deltaReadBytes: UInt64?, deltaWriteBytes: UInt64? + var intervalSeconds: Double? +} +private struct Previous { let driver: Driver; let uptime: Double } +private struct Tracker { + private var pinnedIdentity: UInt64? + private var previous: Previous? + mutating func observe(_ driver: Driver, role: String, uptime: Double) -> Row { + if let pinnedIdentity, pinnedIdentity != driver.identity { + return Row(role: role, status: "unavailable", reason: "deviceChanged") + } + pinnedIdentity = driver.identity + defer { previous = Previous(driver: driver, uptime: uptime) } + var row = Row(role: role, status: "baseline", readBytes: driver.counters.readBytes, + writeBytes: driver.counters.writeBytes) + guard let previous else { return row } + guard driver.counters.readBytes >= previous.driver.counters.readBytes, + driver.counters.writeBytes >= previous.driver.counters.writeBytes, + uptime > previous.uptime else { + return Row(role: role, status: "reset", reason: "counterOrClockReset", + readBytes: driver.counters.readBytes, writeBytes: driver.counters.writeBytes) + } + row = Row(role: role, status: "ok", readBytes: driver.counters.readBytes, + writeBytes: driver.counters.writeBytes, + deltaReadBytes: driver.counters.readBytes - previous.driver.counters.readBytes, + deltaWriteBytes: driver.counters.writeBytes - previous.driver.counters.writeBytes, + intervalSeconds: uptime - previous.uptime) + return row + } +} + +private final class SelectedRoot { + let volume: Volume + let descriptor: Int32 + let device: dev_t + let inode: ino_t + init(_ volume: Volume, nonce: String) throws { + let root = URL(fileURLWithPath: volume.root).standardizedFileURL + let mount = URL(fileURLWithPath: volume.mount).standardizedFileURL + guard volume.nonce == nonce, root.path == volume.root, mount.path == volume.mount, + mount.deletingLastPathComponent().path == "/Volumes", + root.deletingLastPathComponent() == mount, + root.lastPathComponent.hasPrefix("BitMatch-validation-"), + root.lastPathComponent.contains(nonce) else { throw ObserverError.invalidConfiguration } + let fd = Darwin.open(volume.root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC) + guard fd >= 0 else { throw ObserverError.invalidConfiguration } + var identity = stat() + guard fstat(fd, &identity) == 0 else { Darwin.close(fd); throw ObserverError.invalidConfiguration } + self.volume = volume; descriptor = fd; device = identity.st_dev; inode = identity.st_ino + do { + let markerFD = openat(fd, ".bitmatch-validation-owned.json", O_RDONLY | O_NOFOLLOW | O_NONBLOCK | O_CLOEXEC) + guard markerFD >= 0 else { throw ObserverError.invalidConfiguration } + let handle = FileHandle(fileDescriptor: markerFD, closeOnDealloc: true) + defer { try? handle.close() } + var info = stat() + guard fstat(markerFD, &info) == 0, (info.st_mode & S_IFMT) == S_IFREG, + info.st_size > 0, info.st_size <= 8_192 else { throw ObserverError.invalidConfiguration } + let data = try handle.read(upToCount: 8_193) ?? Data() + guard data.count <= 8_192, try JSONDecoder().decode(Volume.self, from: data) == volume else { + throw ObserverError.invalidConfiguration + } + try validateIdentity() + } catch { + // All stored properties are initialized: Swift runs deinit when + // this initializer throws. It owns the directory descriptor now. + throw error + } + } + deinit { Darwin.close(descriptor) } + func validateIdentity() throws { + var named = stat(); var pinned = stat(); var filesystem = statfs() + guard lstat(volume.root, &named) == 0, fstat(descriptor, &pinned) == 0, + (named.st_mode & S_IFMT) == S_IFDIR, + named.st_dev == device, named.st_ino == inode, + pinned.st_dev == device, pinned.st_ino == inode, + fstatfs(descriptor, &filesystem) == 0 else { throw ObserverError.changedRoot } + let mount = withUnsafeBytes(of: filesystem.f_mntonname) { + String(decoding: $0.prefix { $0 != 0 }, as: UTF8.self) + } + guard mount == volume.mount else { throw ObserverError.changedRoot } + } + func driver(session: DASession) throws -> Driver { + try validateIdentity() + guard let disk = DADiskCreateFromVolumePath(kCFAllocatorDefault, session, + URL(fileURLWithPath: volume.mount) as CFURL), + let description = DADiskCopyDescription(disk) as? [String: Any], + let uuid = description[kDADiskDescriptionVolumeUUIDKey as String], + CFGetTypeID(uuid as CFTypeRef) == CFUUIDGetTypeID() else { throw ObserverError.volumeUnavailable } + guard (CFUUIDCreateString(kCFAllocatorDefault, (uuid as! CFUUID)) as String) + .caseInsensitiveCompare(volume.volumeUUID) == .orderedSame else { throw ObserverError.wrongVolume } + // Start at the selected volume's real IOMedia, not a guessed BSD disk + // number or a host-wide driver enumeration. Parent objects are retained + // by GetParentEntry and released at each step. + var current = DADiskCopyIOMedia(disk) + var candidates: [Driver] = [] + defer { if current != 0 { IOObjectRelease(current) } } + for _ in 0..<48 { + guard current != 0 else { break } + if IOObjectConformsTo(current, "IOBlockStorageDriver") != 0 { + var identity: UInt64 = 0 + guard IORegistryEntryGetRegistryEntryID(current, &identity) == KERN_SUCCESS, + let properties = IORegistryEntryCreateCFProperty(current, "Statistics" as CFString, + kCFAllocatorDefault, 0)?.takeRetainedValue() as? [String: Any], + let counters = Counters.parse(properties) else { throw ObserverError.missingCounters } + candidates.append(Driver(identity: identity, counters: counters)) + } + var parent: io_registry_entry_t = 0 + let status = IORegistryEntryGetParentEntry(current, kIOServicePlane, &parent) + IOObjectRelease(current) + current = status == KERN_SUCCESS ? parent : 0 + } + // Hitting the ancestry limit cannot establish a unique association. + guard current == 0 else { throw ObserverError.ambiguousDriver } + return try Driver.unique(candidates) + } +} + +private struct Options { + var config: String? + var samples = 10 + var interval = 1.0 + var selfTest = false + init(_ arguments: [String]) throws { + var index = 0 + while index < arguments.count { + let flag = arguments[index] + if flag == "--self-test" { selfTest = true; index += 1; continue } + guard index + 1 < arguments.count else { throw ObserverError.invalidArguments } + let value = arguments[index + 1] + switch flag { + case "--config": config = value + case "--samples": guard let count = Int(value) else { throw ObserverError.invalidArguments }; samples = count + case "--interval": guard let seconds = Double(value) else { throw ObserverError.invalidArguments }; interval = seconds + default: throw ObserverError.invalidArguments + } + index += 2 + } + guard (1...600).contains(samples), interval.isFinite, (0.2...60).contains(interval), + Double(samples) * interval <= 3_600, + selfTest || config?.hasPrefix("/") == true else { throw ObserverError.invalidArguments } + } +} +private struct Snapshot: Encodable { + let sample: Int + let timestamp: Date + let uptimeSeconds: Double + let roles: [Row] +} +private func readConfiguration(_ path: String) throws -> Data { + let descriptor = Darwin.open(path, O_RDONLY | O_NOFOLLOW | O_NONBLOCK | O_CLOEXEC) + guard descriptor >= 0 else { throw ObserverError.invalidConfiguration } + let file = FileHandle(fileDescriptor: descriptor, closeOnDealloc: true) + defer { try? file.close() } + var info = stat() + guard fstat(descriptor, &info) == 0, (info.st_mode & S_IFMT) == S_IFREG, + info.st_size > 0, info.st_size <= 32_768 else { throw ObserverError.invalidConfiguration } + let data = try file.read(upToCount: 32_769) ?? Data() + guard data.count <= 32_768 else { throw ObserverError.invalidConfiguration } + return data +} +private func emit(_ value: T) throws { + let encoder = JSONEncoder(); encoder.outputFormatting = [.sortedKeys] + encoder.dateEncodingStrategy = .iso8601 + var data = try encoder.encode(value); data.append(0x0a) + try FileHandle.standardOutput.write(contentsOf: data) +} + +@main private enum DeviceIOObserver { + static func main() { + do { + let options = try Options(Array(CommandLine.arguments.dropFirst())) + if options.selfTest { try selfTests(); return } + guard let path = options.config, + let session = DASessionCreate(kCFAllocatorDefault) else { throw ObserverError.invalidConfiguration } + let data = try readConfiguration(path) + let config = try JSONDecoder().decode(Config.self, from: data) + guard UUID(uuidString: config.nonce) != nil else { throw ObserverError.invalidConfiguration } + var selected: [String: SelectedRoot] = [:] + for role in roles { + guard let volume = config.volumes[role] else { throw ObserverError.invalidConfiguration } + selected[role] = try SelectedRoot(volume, nonce: config.nonce) + } + guard Set(selected.values.map(\.device)).count == roles.count, + Set(selected.values.map { $0.volume.volumeUUID.lowercased() }).count == roles.count else { + throw ObserverError.invalidConfiguration + } + var trackers = Dictionary(uniqueKeysWithValues: roles.map { ($0, Tracker()) }) + let started = ProcessInfo.processInfo.systemUptime + var unavailable = false + for index in 0.. 0 { Thread.sleep(forTimeInterval: delay) } + var drivers: [String: Driver] = [:] + var errors: [String: String] = [:] + for role in roles { + do { drivers[role] = try selected[role]!.driver(session: session) } + catch let error as ObserverError { errors[role] = error.rawValue } + catch { errors[role] = "unavailable" } + } + let uptime = ProcessInfo.processInfo.systemUptime + let shared = sharedDriverRoles(drivers) + let rows = roles.map { role -> Row in + guard let driver = drivers[role] else { + return Row(role: role, status: "unavailable", reason: errors[role] ?? "unavailable") + } + guard !shared.contains(role) else { + return Row(role: role, status: "unavailable", reason: "sharedDevice") + } + return trackers[role]!.observe(driver, role: role, uptime: uptime) + } + unavailable = unavailable || rows.contains { $0.status == "unavailable" || $0.status == "reset" } + try emit(Snapshot(sample: index, timestamp: Date(), uptimeSeconds: uptime, roles: rows)) + } + if unavailable { exit(3) } + } catch let error as ObserverError { + try? FileHandle.standardError.write(contentsOf: Data(("Device I/O observer: " + error.rawValue + "\n").utf8)) + exit(2) + } catch { + // Foundation error descriptions can include config/media paths. + try? FileHandle.standardError.write(contentsOf: Data("Device I/O observer: unavailable\n".utf8)) + exit(2) + } + } +} + +private func selfTests() throws { + var count = 0 + func check(_ condition: @autoclosure () -> Bool) throws { + guard condition() else { throw ObserverError.invalidConfiguration }; count += 1 + } + func fails(_ body: () throws -> Void) -> Bool { do { try body(); return false } catch { return true } } + let valid: [String: Any] = ["Bytes (Read)": NSNumber(value: 0), "Bytes (Write)": NSNumber(value: UInt64.max)] + try check(Counters.parse(valid) == Counters(readBytes: 0, writeBytes: .max)) + for invalid: Any in [NSNumber(value: -1), NSNumber(value: true), NSNumber(value: 1.5), "12"] { + try check(Counters.parse(["Bytes (Read)": invalid, "Bytes (Write)": NSNumber(value: 2)]) == nil) + } + try check(Counters.parse(["Bytes (Read)": NSNumber(value: 2)]) == nil) + let initial = Driver(identity: 123, counters: Counters(readBytes: 100, writeBytes: 200)) + let unique = try Driver.unique([initial]) + try check(unique.identity == initial.identity) + try check(fails { _ = try Driver.unique([]) }) + try check(fails { _ = try Driver.unique([initial, initial]) }) + try check(sharedDriverRoles(["sd": initial, "ssdA": initial]) == Set(["sd", "ssdA"])) + try check(sharedDriverRoles(["sd": initial, "ssdA": .init(identity: 456, counters: initial.counters)]).isEmpty) + var tracker = Tracker() + let baseline = tracker.observe(initial, role: "sd", uptime: 1) + try check(baseline.status == "baseline" && baseline.deltaReadBytes == nil) + let next = tracker.observe(Driver(identity: 123, counters: .init(readBytes: 150, writeBytes: 225)), role: "sd", uptime: 3) + try check(next.status == "ok" && next.deltaReadBytes == 50 && next.deltaWriteBytes == 25 && next.intervalSeconds == 2) + let reset = tracker.observe(Driver(identity: 123, counters: .init(readBytes: 10, writeBytes: 20)), role: "sd", uptime: 4) + try check(reset.status == "reset" && reset.deltaReadBytes == nil && reset.deltaWriteBytes == nil) + let replacement = tracker.observe(Driver(identity: 999, counters: .init(readBytes: 50, writeBytes: 60)), role: "sd", uptime: 5) + try check(replacement.status == "unavailable" && replacement.readBytes == nil && replacement.deltaReadBytes == nil) + for arguments in [["--samples", "0"], ["--config", "/tmp/x", "--interval", "nan"], + ["--config", "/tmp/x", "--samples", "601"], ["--config", "/tmp/x", "--samples", "600", "--interval", "60"]] { + try check(fails { _ = try Options(arguments) }) + } + let encoder = JSONEncoder() + let serialized = try encoder.encode(next) + let object = try JSONSerialization.jsonObject(with: serialized) as? [String: Any] + try check(Set(object?.keys.map { $0 } ?? []) == Set(["role", "status", "readBytes", "writeBytes", + "deltaReadBytes", "deltaWriteBytes", "intervalSeconds"])) + let scratch = FileManager.default.temporaryDirectory.appendingPathComponent("bitmatch-device-io-selftest-\(UUID())") + try FileManager.default.createDirectory(at: scratch, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: scratch) } + let regular = scratch.appendingPathComponent("config.json") + let bytes = Data("{}".utf8) + try bytes.write(to: regular) + let loaded = try readConfiguration(regular.path) + try check(loaded == bytes) + let link = scratch.appendingPathComponent("link.json") + try FileManager.default.createSymbolicLink(at: link, withDestinationURL: regular) + try check(fails { _ = try readConfiguration(link.path) }) + let fifo = scratch.appendingPathComponent("fifo.json") + guard mkfifo(fifo.path, 0o600) == 0 else { throw ObserverError.invalidConfiguration } + try check(fails { _ = try readConfiguration(fifo.path) }) + try Data(repeating: 65, count: 32_769).write(to: regular) + try check(fails { _ = try readConfiguration(regular.path) }) + struct Receipt: Encodable { let selfTests: Int; let physicalIO = false } + try emit(Receipt(selfTests: count)) +} diff --git a/docs/HARDWARE_TESTING.md b/docs/HARDWARE_TESTING.md index 7cda571c..ce3e2feb 100644 --- a/docs/HARDWARE_TESTING.md +++ b/docs/HARDWARE_TESTING.md @@ -76,6 +76,55 @@ After reserving the hardware and Mac app test slot, supply `TEST_RUNNER_BITMATCH Before touching hardware, `TEST_RUNNER_BITMATCH_PHYSICAL_APP_HOST_SMOKE=1` runs the same evidence assertions on small host scratch folders through `testHostEvidenceHarnessSmoke`. Two ordinary tests also check that invalid configuration leaves existing files untouched and descriptor cleanup preserves an outside symlink target and a replacement directory. +### Selected-device OS I/O observer + +`Scripts/sample-device-io.swift` is a separate, read-only Mac observer for the +already authorized `sd`, `ssdA` and `ssdB` roots in that marked configuration. +It checks the nonce, exact owner marker, pinned directory identity and volume +UUID privately. Disk Arbitration supplies the actual selected volume's IOMedia; +bounded ancestor traversal must identify one unique `IOBlockStorageDriver`. +It never guesses a disk number or sums host-wide I/O. Missing/ambiguous counters, +shared drivers, replacement devices and resets stay explicit, rather than +becoming zero bytes. It reads only the config/owner markers and OS metadata; +it does not start a transfer, open media devices, purge caches, format or eject. + +Build on the host, then keep output in a private host directory: + +```sh +xcrun swiftc -swift-version 6 -parse-as-library -O \ + -module-cache-path /tmp/bitmatch-device-io-cache \ + Scripts/sample-device-io.swift -o /tmp/bitmatch-device-io +/tmp/bitmatch-device-io --self-test +/tmp/bitmatch-device-io --config "$private_hardware_config" \ + --samples 60 --interval 1 > "$private_observation_log" +``` + +The self-test uses mocked counters/mappings and tiny host scratch files, including +FIFO/symlink/oversized config controls; it does not access physical volumes. +Sampling accepts 1–600 snapshots, intervals of 0.2–60 seconds, and a requested +window no longer than one hour. Output is bounded to those snapshots. OS calls +themselves can still block on an unhealthy system; this is not a hard deadline +or an unplug watchdog. Exit 2 means invalid arguments/configuration or a fatal +read/export error; exit 3 means at least one unavailable/reset sample. Exit 0 +describes observer availability, never transfer success. + +JSONL contains only generic roles, counters, controlled status/reason codes, +UTC timestamps and monotonic `uptimeSeconds`. The first sample is a baseline; +later rows include deltas and their elapsed interval. Sampling is sequential, +not a simultaneous hardware timestamp. Align windows to the app receipt's +transfer-start/settled uptime so later independent hashes, Saved Check and +regenerated reports are not attributed to the measured transfer. + +Apple documents these [read counters](https://developer.apple.com/documentation/iokit/kioblockstoragedriverstatisticsbytesreadkey) +and [write counters](https://developer.apple.com/documentation/iokit/kioblockstoragedriverstatisticsbyteswrittenkey) +as cumulative since the block driver was instantiated. Deltas include **all +processes and partitions using that driver**, filesystem metadata, background +work and any other tests on the same selected device. They are OS storage-stack +observations, not app-only accounting or proof of NAND/platter traffic. The +app's `F_NOCACHE` request does not prove a cold controller/bridge/device cache. +No cache-purge, power-loss, durability or integrity claim follows from these +counters; copy evidence and independent readback remain separate. + ## Recording and publishing physical results Use the [hardware report template](HARDWARE_REPORT_TEMPLATE.md) for each configuration and submit it through the repository's **Hardware test report** issue form. Keep outcomes separate for baseline copy, source removal, one-destination removal, sleep, and cancellation. Mark cases you did not run as **Not tested**. A completed transfer without independent destination hashes is **Inconclusive**, not a verified pass. From 763d8e8669a9e26defe713c793d366d056a48874 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 22:50:58 -0400 Subject: [PATCH 27/56] Record transfer windows for physical device counter checks --- BitMatchTests/PhysicalWorkflowEvidenceTests.swift | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/BitMatchTests/PhysicalWorkflowEvidenceTests.swift b/BitMatchTests/PhysicalWorkflowEvidenceTests.swift index e938901c..62601373 100644 --- a/BitMatchTests/PhysicalWorkflowEvidenceTests.swift +++ b/BitMatchTests/PhysicalWorkflowEvidenceTests.swift @@ -217,7 +217,8 @@ final class PhysicalWorkflowEvidenceTests: XCTestCase { print("PHYSICAL_APP_TIMEOUT cancellationRequestedBy=harness originAPI=user executorSettled=\(settled) fixturesRetained=true") throw PhysicalAppError.workflowTimedOut(settled: settled) } - let queueSettled = ProcessInfo.processInfo.systemUptime - started + let settledUptime = ProcessInfo.processInfo.systemUptime + let queueSettled = settledUptime - started let cpu = try Self.cpuSeconds() - cpuStart monitor.cancel(); await monitor.value let records = try JSONDecoder().decode([LocalTransferRecord].self, from: Data(contentsOf: journalURL)) @@ -308,7 +309,9 @@ final class PhysicalWorkflowEvidenceTests: XCTestCase { XCTAssertEqual(after, original, "Source changed after reports/check") return try PhysicalAppHarnessSettlement.withPassingAssertions(testRun?.totalFailureCount) { let receipt = PhysicalAppReceipt(pair: pair, mode: mode.rawValue, sourceFiles: original.count, - selectedFiles: selected.count, sourceBytes: totalBytes, queueSettledSeconds: queueSettled, + selectedFiles: selected.count, sourceBytes: totalBytes, + transferStartedUptime: started, transferSettledUptime: settledUptime, + queueSettledSeconds: queueSettled, pipelineReturnedSeconds: pipelineSamples.pipelineReturnedUptime.map { $0 - started }, cpuSeconds: cpu, initialMiB: pipelineSamples.initialMiB, peakMiB: pipelineSamples.peakMiB, telemetry: telemetry, destinations: io.destinations.enumerated().map { @@ -497,6 +500,10 @@ private enum PhysicalAppHarnessSettlement { private struct PhysicalAppReceipt: Encodable { struct Destination: Codable { let index: Int; let pipelineWrittenBytes: Int64; let pipelineReadbackBytes: Int64 } let pair: Int; let mode: String; let sourceFiles: Int; let selectedFiles: Int; let sourceBytes: Int64 + /// Monotonic host times separate automatic transfer work from the test's + /// later hashes, Check, and report regeneration in external I/O samples. + let transferStartedUptime: Double + let transferSettledUptime: Double /// End includes automatic reports + durable journal settlement. Independent /// test hashes, Saved Check and History regeneration occur after this clock. let queueSettledSeconds: Double From 29e0fa8e881701cc8c6d1b0e24d9eb5b273ecdaa Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 22:52:22 -0400 Subject: [PATCH 28/56] Add opt-in physical transfer and large-file stress coverage --- .../PhysicalDriveTransferTests.swift | 374 ++++++++++++++++++ 1 file changed, 374 insertions(+) create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/PhysicalDriveTransferTests.swift diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/PhysicalDriveTransferTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/PhysicalDriveTransferTests.swift new file mode 100644 index 00000000..20cdbf61 --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/PhysicalDriveTransferTests.swift @@ -0,0 +1,374 @@ +#if os(macOS) +import CryptoKit +import Darwin +import Foundation +import Synchronization +import XCTest +@testable import BitMatchEngine + +/// Explicit opt-in only. The caller provisions marked folders on authorized +/// volumes; these tests never format, detach, or erase a physical volume. +final class PhysicalDriveTransferTests: XCTestCase { + func testCardToTwoPhysicalBackupsAcrossModesAndReuse() async throws { + let fixture = try PhysicalFixture.load() + let attempt = UUID().uuidString + let source = try fixture.child("sd", "Card-\(attempt)") + var files: [(String, Int64)] = [("empty.bin", 0), ("one.bin", 1), + ("nested/café & 'take'.bin", 4_097), ("boundary.bin", 4_194_321), + ("clip01.bin", 536_870_925), ("clip02.bin", 1_610_612_765)] + files += (0..<128).map { ("sidecars/deep/level/sidecar-\($0).txt", Int64($0 * 13 + 1)) } + let original = try seed(source, files: files) + var runs = 0 + for mode in VerificationMode.allCases { + for pipelined in [false, true] { + let label = "card-\(mode.rawValue)-\(pipelined)-\(attempt)" + let targets = try [fixture.child("ssdA", label), fixture.child("ssdB", label)] + try await transfer(source, expected: original, targets: targets, mode: mode, + pipelined: pipelined, label: label) + runs += 1 + if mode == .standard && pipelined { + let evidenceBefore = try targets.map { target in + try snapshot(SafetyValidator.resolvedDestinationRoot(source: source, destination: target, settings: .init()).appendingPathComponent("ascmhl")) + } + try await transfer(source, expected: original, targets: targets, mode: mode, + pipelined: pipelined, label: "card-reuse", reused: true) + for (index, target) in targets.enumerated() { + let history = SafetyValidator.resolvedDestinationRoot(source: source, destination: target, settings: .init()).appendingPathComponent("ascmhl") + XCTAssertEqual(try snapshot(history), evidenceBefore[index], "Retry changed prior history") + } + runs += 1 + } + // Retain the attempt's outputs. The provisioning runner owns + // descriptor-checked cleanup of its marked volume roots; + // this test never recursively deletes a public path. + try requirePassingAssertions() + } + } + XCTAssertEqual(try snapshot(source), original, "The physical card changed") + XCTAssertEqual(runs, 9) + try requirePassingAssertions() + print("PHYSICAL_CARD_SUMMARY runs=\(runs) modes=4 pipelineVariants=2 backups=2 sourceFiles=\(original.count) sourceBytes=\(original.values.reduce(0) { $0 + $1.size }) sourceSnapshotCompared=true") + } + + func testEightLargeFilesThroughPhysicalCopyVerificationAndMHL() async throws { + guard ProcessInfo.processInfo.environment["BITMATCH_HARDWARE_LARGE"] == "1" else { + throw XCTSkip("Set BITMATCH_HARDWARE_LARGE=1 for the 119.6 GB physical transfer") + } + let fixture = try PhysicalFixture.load() + let environment = ProcessInfo.processInfo.environment + let prepared = environment["BITMATCH_HARDWARE_REUSE_LARGE_SOURCE"] == "1" + let source = try prepared ? fixture.existingChild("ssdA", "LargeCard") : fixture.child("ssdA", "LargeCard") + let targetKey = environment["BITMATCH_HARDWARE_LARGE_TARGET"] + ?? (fixture.volumes["largeDestination"] == nil ? "ssdB" : "largeDestination") + guard ["ssdB", "largeDestination"].contains(targetKey) else { throw PhysicalFixtureError.unsafeRoot } + let target = try fixture.child(targetKey, "large-standard-\(UUID().uuidString)") + let sizes: [Int64] = [2_000_000_000, 3_000_000_000, 4_000_000_000, + 5_000_000_000, 7_000_000_000, 8_800_000_000, 44_500_000_000, 45_300_000_000] + let files = sizes.enumerated().map { ("clip-\($0.offset + 1).mov", $0.element) } + let original = try prepared ? snapshot(source) : seed(source, files: files) + for (path, size) in files { XCTAssertEqual(original[path]?.size, size) } + let companions = Set(try AppleDoubleSelection.review(source: source)) + guard Set(original.keys).subtracting(files.map(\.0)).isSubset(of: companions), + files.allSatisfy({ original[$0.0]?.size == $0.1 }) else { throw PhysicalFixtureError.unsafeRoot } + XCTAssertEqual(sizes.reduce(0, +), 119_600_000_000) + try await transfer(source, expected: original, targets: [target], mode: .standard, + pipelined: true, label: "119600MB-eight-clips") + XCTAssertEqual(try snapshot(source), original, "The original physical source changed") + try requirePassingAssertions() + print("PHYSICAL_LARGE_SUMMARY mediaFiles=8 bytes=119600000000 sourceSnapshotCompared=true copyVerifyMHLCheckAttempted=true targetKind=\(targetKey) syntheticMOVContainers=true") + } + + private func transfer(_ source: URL, expected: [String: PhysicalFile], targets: [URL], + mode: VerificationMode, pipelined: Bool, label: String, + reused: Bool = false) async throws { + let samples = Mutex(PhysicalSamples()) + let monitor = Task.detached { + while !Task.isCancelled { + samples.withLock { $0.sampleMemory() } + do { try await Task.sleep(for: .milliseconds(50)) } catch { break } + } + } + defer { monitor.cancel() } + let started = Date() + var settings = CameraLabelSettings() + settings.excludedAppleDoublePaths = try AppleDoubleSelection.review(source: source) + let excluded = Set(settings.excludedAppleDoublePaths ?? []) + let selected = expected.filter { !excluded.contains($0.key) } + let largeFiles = selected.filter { $0.value.size >= 40_000_000_000 } + let hooks = DestinationWriter.FanOutHooks(destinationDidOpenForRead: { _, path in + guard largeFiles[path] != nil else { return } + samples.withLock { state in + state.largeReadBytes[path] = 0 + state.activeLargeReads.insert(path) + state.maxLargeReads = max(state.maxLargeReads, state.activeLargeReads.count) + } + }, destinationDidRead: { _, path, count in + guard let original = largeFiles[path] else { return } + samples.withLock { state in + state.largeReadBytes[path, default: 0] += Int64(count) + if state.largeReadBytes[path, default: 0] >= original.size { state.activeLargeReads.remove(path) } + } + }) + let operation = try await TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared, + pipelinedVerification: pipelined, verificationConcurrency: 2, fanOutHooks: hooks).performFileOperation( + sourceURL: source, destinationURLs: targets, verificationMode: mode, settings: settings, + estimatedTotalBytes: expected.values.reduce(0) { $0 + $1.size }, + progressCallback: { update in samples.withLock { $0.observe(update) } }, + ioProgressCallback: { update in samples.withLock { state in + if update.sequence > (state.io?.sequence ?? 0) { state.io = update } + } }, onFileResult: nil) + let copyVerifySeconds = Date().timeIntervalSince(started) + XCTAssertEqual(operation.sourceManifest?.count, expected.count, label) + XCTAssertEqual(operation.results.count, expected.count * targets.count, label) + let selectedResults = operation.results.filter { !$0.excludedAppleDouble } + XCTAssertEqual(selectedResults.count, selected.count * targets.count) + XCTAssertEqual(operation.results.filter(\.excludedAppleDouble).count, excluded.count * targets.count) + XCTAssertTrue(selectedResults.allSatisfy(\.success), "\(label): \(operation.results.compactMap { $0.error?.localizedDescription })") + guard selectedResults.allSatisfy(\.success) else { throw PhysicalFixtureError.failedTransfer } + for result in selectedResults { + let path = result.sourceURL.relativePath(to: source) + let original = try XCTUnwrap(expected[path]) + XCTAssertEqual(result.fileSize, original.size) + if mode == .quick { XCTAssertNil(result.verificationResult) } + else { + XCTAssertEqual(result.verificationResult?.isValid, true) + XCTAssertEqual(result.verificationResult?.sourceChecksum, original.sha256) + XCTAssertEqual(result.verificationResult?.destinationChecksum, original.sha256) + if result.sourceURL.pathExtension == "mov" { XCTAssertEqual(result.clipIntegrity, .complete) } + } + if reused { XCTAssertTrue(result.wasReused, path) } + } + let plan = TransferCompletion.ascmhlPlan(results: operation.results, sourceFiles: operation.sourceManifest, + destinations: targets, source: source, settings: settings) + var mhlSeconds = 0.0 + var checkSeconds = 0.0 + if mode == .quick { + XCTAssertTrue(plan.jobs.isEmpty) + XCTAssertFalse(plan.issues.isEmpty) + for target in targets { + let output = SafetyValidator.resolvedDestinationRoot(source: source, destination: target, settings: .init()) + XCTAssertFalse(FileManager.default.fileExists(atPath: output.appendingPathComponent("ascmhl").path)) + let outputBytes = try snapshot(output, ignoringStatus: true) + for (path, original) in selected { XCTAssertEqual(outputBytes[path], original) } + let generated = Set(try AppleDoubleSelection.review(source: output)) + XCTAssertTrue(Set(outputBytes.keys).subtracting(selected.keys).isSubset(of: generated)) + } + } else { + XCTAssertEqual(plan.jobs.count, targets.count) + XCTAssertTrue(plan.issues.isEmpty) + if !reused { + let mhlStart = Date() + let failures = try TransferCompletion.writeASCMHL(plan.jobs, planIssues: plan.issues, + startTime: operation.startTime, source: source, toolVersion: "physical-validation", + progress: { update in samples.withLock { $0.mhlBytes = max($0.mhlBytes, update.bytesProcessed); $0.sampleMemory() } }) + mhlSeconds = Date().timeIntervalSince(mhlStart) + XCTAssertTrue(failures.isEmpty, "\(label): \(failures)") + guard failures.isEmpty else { throw PhysicalFixtureError.failedTransfer } + } + let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + let checkStart = Date() + for job in plan.jobs { + let discovery = try await checker.discover(under: job.root) + XCTAssertEqual(discovery.expectedFiles.count, selected.count) + for (path, entry) in discovery.expectedFiles { + XCTAssertNotNil(selected[path]) + switch entry.algorithm { + case .sha256: XCTAssertEqual(entry.checksum, expected[path]?.sha256) + case .md5: + // ASC MHL may prefer its interoperability digest. The + // Check below rereads and validates that recorded MD5; + // pipeline SHA-256 is separately checked against our + // source baseline above. Never compare unlike hashes. + XCTAssertNotNil(entry.checksum.range(of: "^[a-f0-9]{32}$", options: .regularExpression)) + case .xxh64: XCTFail("This writer does not request XXH64") + } + } + let checked = try await checker.check(discovery) { _ in samples.withLock { $0.sampleMemory() } } + XCTAssertTrue(checked.isIntact, "\(label): missing=\(checked.missingPaths.count) changed=\(checked.changedPaths.count)") + XCTAssertEqual(checked.matchingPaths.count, selected.count) + } + checkSeconds = Date().timeIntervalSince(checkStart) + } + monitor.cancel() + await monitor.value + let measured = samples.withLock { $0 } + XCTAssertFalse(measured.invalidProgress, label) + XCTAssertEqual(measured.lastStage, .completed) + XCTAssertEqual(measured.last, 1) + let verdict = TransferCompletion.verdict(rows: TransferCompletion.rows(from: operation), + sourceFiles: operation.sourceManifest, destinations: targets, source: source, settings: settings, + mode: mode, generateASCMHL: mode != .quick, handoffIssues: [], reportIssue: nil, + project: .init(didPersist: true, locallySafe: nil)) + XCTAssertEqual(verdict.success, mode != .quick && excluded.isEmpty, + "Quick and excluded-file runs must never claim whole-card safety") + XCTAssertGreaterThan(measured.updates, 0) + XCTAssertGreaterThan(measured.initialMiB, 0) + XCTAssertLessThan(measured.initialMiB, 256) + XCTAssertLessThan(measured.peakMiB, 512) + XCTAssertLessThan(measured.peakMiB - measured.initialMiB, 256, "Streaming footprint grew unexpectedly") + if largeFiles.count == 2 && pipelined { + XCTAssertEqual(measured.maxLargeReads, 2, + "A serial large-file pass does not establish overlapping verification coverage") + } + let totalBytes = selected.values.reduce(Int64(0)) { $0 + $1.size } + let io = try XCTUnwrap(measured.io) + XCTAssertEqual(io.destinations.count, targets.count) + for destination in io.destinations { + XCTAssertEqual(destination.writtenBytes, reused ? 0 : totalBytes) + if mode != .quick { XCTAssertGreaterThanOrEqual(destination.readbackBytes, totalBytes) } + } + if mode != .quick && !reused { XCTAssertEqual(measured.mhlBytes, totalBytes * Int64(targets.count)) } + // XCTAssert records a failure without throwing. Stop before the caller + // removes any outputs, so failed physical evidence remains inspectable. + try requirePassingAssertions() + print("PHYSICAL_TRANSFER label=\(label) mode=\(mode.rawValue) pipelined=\(pipelined) backups=\(targets.count) rows=\(operation.results.count) reused=\(reused) copyVerifySeconds=\(copyVerifySeconds) mhlSeconds=\(mhlSeconds) checkSeconds=\(checkSeconds) initialMiB=\(measured.initialMiB) peakMiB=\(measured.peakMiB) mhlLogicalBytes=\(measured.mhlBytes) progressUpdates=\(measured.updates) observedLargeReadConcurrency=\(measured.maxLargeReads)") + } + + private func requirePassingAssertions() throws { + guard let testRun, testRun.failureCount == 0 else { throw PhysicalFixtureError.failedTransfer } + } + + private func seed(_ source: URL, files: [(String, Int64)]) throws -> [String: PhysicalFile] { + var expected: [String: PhysicalFile] = [:] + for (index, entry) in files.enumerated() { + let file = source.appendingPathComponent(entry.0) + try FileManager.default.createDirectory(at: file.deletingLastPathComponent(), withIntermediateDirectories: true) + let fd = Darwin.open(file.path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0o600) + guard fd >= 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } + let handle = FileHandle(fileDescriptor: fd, closeOnDealloc: true) + defer { try? handle.close() } + guard fcntl(fd, F_NOCACHE, 1) == 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } + var sha = SHA256() + var offset: Int64 = 0 + while offset < entry.1 { + try Task.checkCancellation() + let count = Int(min(Int64(4 * 1_024 * 1_024), entry.1 - offset)) + try autoreleasepool { + var bytes = Data(repeating: UInt8(truncatingIfNeeded: index * 19 + Int(offset / 4_194_304) + 1), count: count) + withUnsafeBytes(of: offset.littleEndian) { value in + bytes.replaceSubrange(0.. [String: PhysicalFile] { + var result: [String: PhysicalFile] = [:] + for entry in try CardSource.enumerateRegularFiles(base: root) { + try Task.checkCancellation() + if ignoringStatus && TransferStatusMarker.hasExactName(entry.url.lastPathComponent) { continue } + let handle = try FileHandle(forReadingFrom: entry.url) + defer { try? handle.close() } + guard fcntl(handle.fileDescriptor, F_NOCACHE, 1) == 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } + var sha = SHA256() + while try autoreleasepool(invoking: { + try Task.checkCancellation() + let data = try handle.read(upToCount: 4 * 1_024 * 1_024) ?? Data() + guard !data.isEmpty else { return false } + sha.update(data: data) + return true + }) {} + result[entry.relativePath] = PhysicalFile(size: entry.size, sha256: sha.finalize().map { String(format: "%02x", $0) }.joined()) + } + return result + } +} + +private struct PhysicalFile: Equatable, Sendable { let size: Int64; let sha256: String } +private enum PhysicalFixtureError: Error { case unsafeRoot, wrongVolume, failedTransfer } +private struct PhysicalFixture: Decodable { + struct Volume: Decodable { let nonce: String; let root: String; let mount: String; let volumeUUID: String } + let nonce: String + let volumes: [String: Volume] + static func load() throws -> PhysicalFixture { + guard let path = ProcessInfo.processInfo.environment["BITMATCH_HARDWARE_CONFIG"] else { + throw XCTSkip("Provide an explicitly provisioned BITMATCH_HARDWARE_CONFIG") + } + let fixture = try JSONDecoder().decode(Self.self, from: Data(contentsOf: URL(fileURLWithPath: path))) + for volume in fixture.volumes.values { + let root = URL(fileURLWithPath: volume.root) + guard root.lastPathComponent.hasPrefix("BitMatch-validation-"), + root.deletingLastPathComponent().path == volume.mount, + volume.nonce == fixture.nonce else { throw PhysicalFixtureError.unsafeRoot } + var st = stat() + guard lstat(root.path, &st) == 0, (st.st_mode & S_IFMT) == S_IFDIR else { throw PhysicalFixtureError.unsafeRoot } + let marker = try JSONDecoder().decode(Volume.self, from: Data(contentsOf: root.appendingPathComponent(".bitmatch-validation-owned.json"))) + guard marker.nonce == fixture.nonce, marker.root == volume.root, + marker.volumeUUID == volume.volumeUUID else { throw PhysicalFixtureError.unsafeRoot } + let current = try root.resourceValues(forKeys: [.volumeUUIDStringKey]) + guard current.volumeUUIDString?.caseInsensitiveCompare(volume.volumeUUID) == .orderedSame else { + throw PhysicalFixtureError.wrongVolume + } + } + guard Set(["sd", "ssdA", "ssdB"].compactMap { fixture.volumes[$0]?.volumeUUID }).count == 3 else { throw PhysicalFixtureError.wrongVolume } + return fixture + } + func child(_ key: String, _ name: String) throws -> URL { + guard let volume = volumes[key], !name.contains("/"), name != ".", name != ".." else { throw PhysicalFixtureError.unsafeRoot } + let path = URL(fileURLWithPath: volume.root).appendingPathComponent(name) + guard mkdir(path.path, 0o700) == 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } + return path + } + func existingChild(_ key: String, _ name: String) throws -> URL { + guard let volume = volumes[key], !name.contains("/"), name != ".", name != ".." else { throw PhysicalFixtureError.unsafeRoot } + let path = URL(fileURLWithPath: volume.root).appendingPathComponent(name) + var st = stat() + guard lstat(path.path, &st) == 0, (st.st_mode & S_IFMT) == S_IFDIR else { throw PhysicalFixtureError.unsafeRoot } + return path + } +} + +private struct PhysicalSamples: Sendable { + var initialMiB = 0, peakMiB = 0, updates = 0 + var invalidProgress = false + var last = 0.0 + var lastStage: ProgressStage? + var mhlBytes: Int64 = 0 + var io: TransferIOProgress? + var largeReadBytes: [String: Int64] = [:] + var activeLargeReads: Set = [] + var maxLargeReads = 0 + mutating func sampleMemory() { + if let value = TransferDiagnosticStore.physicalFootprintMB() { + if initialMiB == 0 { initialMiB = value } + peakMiB = max(peakMiB, value) + } + } + mutating func observe(_ update: OperationProgress) { + sampleMemory() + updates += 1 + if !update.overallProgress.isFinite || update.overallProgress < last - 0.000_001 + || (update.currentStage != .completed && update.overallProgress >= 1) { invalidProgress = true } + last = update.overallProgress + lastStage = update.currentStage + } +} +#endif From 4106a48a42960cf5afe90be853e180622c00d4a8 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 23:08:26 -0400 Subject: [PATCH 29/56] Bound buffer benchmark workers and trace their phases --- .../BufferSizeBenchTests.swift | 198 ++++++++++++++++-- 1 file changed, 177 insertions(+), 21 deletions(-) diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/BufferSizeBenchTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/BufferSizeBenchTests.swift index 024a0859..a893c665 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/BufferSizeBenchTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/BufferSizeBenchTests.swift @@ -8,6 +8,8 @@ import XCTest /// Source writes and timed reads request F_NOCACHE, including the first run. /// This requests a cache bypass but cannot prove a physical-device read; /// confirm that separately with the selected drive's I/O counters. +/// Cancellation is checked between chunks; synchronous filesystem calls may +/// still wait. Phase timestamps use monotonic uptime, without file paths. final class BufferSizeBenchTests: XCTestCase { private let totalBytes: Int64 = 2 * 1_024 * 1_024 * 1_024 private let fileBytes: Int64 = 256 * 1_024 * 1_024 @@ -18,6 +20,7 @@ final class BufferSizeBenchTests: XCTestCase { guard ProcessInfo.processInfo.environment["BITMATCH_BENCH"] == "1" else { throw XCTSkip("Set BITMATCH_BENCH=1 to run the 2 GB buffer benchmark.") } + try Task.checkCancellation() let environment = ProcessInfo.processInfo.environment let base = environment["BITMATCH_BENCH_DIR"].map { @@ -27,19 +30,31 @@ final class BufferSizeBenchTests: XCTestCase { let source = root.appendingPathComponent("source", isDirectory: true) try FileManager.default.createDirectory(at: source, withIntermediateDirectories: true) let keepFiles = environment["BITMATCH_BENCH_KEEP"] == "1" - defer { if !keepFiles { try? FileManager.default.removeItem(at: root) } } + defer { + if !keepFiles { + try? Self.measured(phase: "cleanup", buffer: 0, run: 0, logicalBytes: 0) { + try FileManager.default.removeItem(at: root) + } + } + } - let files = try generateFileSet(at: source) + let (files, _) = try Self.measured(phase: "sourceCreation", buffer: 8 * 1_024 * 1_024, + run: 0, logicalBytes: totalBytes) { + try Self.generateFileSet(at: source, totalBytes: totalBytes, fileBytes: fileBytes) + } var rows: [(buffer: Int, hash: Summary, copy: Summary)] = [] var referenceDigest: String? for bufferSize in bufferSizes { + try Task.checkCancellation() var hashRates: [Double] = [] var copyRates: [Double] = [] for run in 0.. [URL] { + func testSmallFixtureHashAndRawCopyPreserveEveryByte() throws { + let root = try smallFixture() + let source = root.appendingPathComponent("source", isDirectory: true) + let destination = root.appendingPathComponent("backup", isDirectory: true) + try FileManager.default.createDirectory(at: source, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) + let files = try Self.generateFileSet(at: source, totalBytes: 256 * 1_024, + fileBytes: 128 * 1_024, blockBytes: 8 * 1_024) + XCTAssertEqual(files.count, 2) + let original = try files.map { try Data(contentsOf: $0) } + var expected = SHA256() + for data in original { expected.update(data: data) } + let expectedDigest = expected.finalize().map { String(format: "%02x", $0) }.joined() + XCTAssertEqual(try Self.hash(files: files, bufferSize: 4 * 1_024), expectedDigest) + try Self.copy(files: files, to: destination, bufferSize: 7 * 1_024) + let copies = files.map { destination.appendingPathComponent($0.lastPathComponent) } + XCTAssertEqual(try copies.map { try Data(contentsOf: $0) }, original) + XCTAssertEqual(try files.map { try Data(contentsOf: $0) }, original) + XCTAssertEqual(try Self.hash(files: copies, bufferSize: 3 * 1_024), expectedDigest) + } + + func testCancelledWorkersStopBeforeFixtureIO() async throws { + let root = try smallFixture() + let missing = root.appendingPathComponent("not-created") + let cancelled = await Task.detached { + withUnsafeCurrentTask { $0?.cancel() } + // Small parameters keep this regression safe even if a guard is + // removed; wrong errors (such as ENOENT/EBADF) do not count as cancel. + return [ + Self.throwsCancellation { + _ = try Self.generateFileSet(at: root, totalBytes: 256, fileBytes: 256, blockBytes: 64) + }, + Self.throwsCancellation { _ = try Self.hash(files: [missing], bufferSize: 64) }, + Self.throwsCancellation { try Self.copy(files: [missing], to: root, bufferSize: 64) }, + Self.throwsCancellation { try Self.writeAll(descriptor: -1, bytes: [1], count: 1, path: missing.path) } + ] + }.value + XCTAssertEqual(cancelled, [true, true, true, true]) + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: root.path), []) + } + + func testCancelledPhaseHasMonotonicEndWithoutPrivateFields() throws { + var breadcrumbs: [PhaseBreadcrumb] = [] + XCTAssertThrowsError(try Self.measured(phase: "hash", buffer: 4_096, run: 3, logicalBytes: 16_384, + emit: { breadcrumbs.append($0) }) { + throw CancellationError() + }) { XCTAssertTrue($0 is CancellationError) } + XCTAssertEqual(breadcrumbs.map(\.event), ["start", "end"]) + XCTAssertEqual(breadcrumbs.last?.outcome, "cancelled") + XCTAssertGreaterThanOrEqual(try XCTUnwrap(breadcrumbs.last?.monotonicNanoseconds), + try XCTUnwrap(breadcrumbs.first?.monotonicNanoseconds)) + let encoded = try JSONEncoder().encode(try XCTUnwrap(breadcrumbs.last)) + let fields = try XCTUnwrap(JSONSerialization.jsonObject(with: encoded) as? [String: Any]) + XCTAssertEqual(Set(fields.keys), ["event", "phase", "monotonicNanoseconds", "bufferBytes", "run", "logicalBytes", "outcome"]) + XCTAssertEqual(fields["bufferBytes"] as? Int, 4_096) + XCTAssertEqual(fields["run"] as? Int, 3) + XCTAssertEqual(fields["logicalBytes"] as? Int, 16_384) + } + + private func smallFixture() throws -> URL { + let root = FileManager.default.temporaryDirectory.appendingPathComponent("bitmatch-buffer-test-\(UUID())") + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + addTeardownBlock { try? FileManager.default.removeItem(at: root) } + return root + } + + private static func throwsCancellation(_ body: () throws -> Void) -> Bool { + do { try body(); return false } + catch is CancellationError { return true } + catch { return false } + } + + private static func generateFileSet(at directory: URL, totalBytes: Int64, fileBytes: Int64, + blockBytes: Int = 8 * 1_024 * 1_024) throws -> [URL] { + try Task.checkCancellation() let count = Int(totalBytes / fileBytes) - var block = [UInt8](repeating: 0, count: 8 * 1_024 * 1_024) + var block = [UInt8](repeating: 0, count: blockBytes) var state: UInt64 = 0x6a09e667f3bcc909 for index in block.indices { + if index % (64 * 1_024) == 0 { try Task.checkCancellation() } state ^= state << 13 state ^= state >> 7 state ^= state << 17 @@ -88,6 +182,7 @@ final class BufferSizeBenchTests: XCTestCase { } return try (0.. 0 { + try Task.checkCancellation() let count = min(Int64(block.count), remaining) try writeAll(descriptor: descriptor, bytes: block, count: Int(count), path: url.path) remaining -= count } + try Task.checkCancellation() guard Darwin.fsync(descriptor) == 0 else { throw posixError("fsync", url.path) } + try Task.checkCancellation() return url } } - private func hash(files: [URL], bufferSize: Int) throws -> String { + private static func hash(files: [URL], bufferSize: Int) throws -> String { + try Task.checkCancellation() var hasher = SHA256() for file in files { + try Task.checkCancellation() let descriptor = Darwin.open(file.path, O_RDONLY) try requireDescriptor(descriptor, operation: "open", path: file.path) guard Darwin.fcntl(descriptor, F_NOCACHE, 1) != -1 else { @@ -120,16 +220,27 @@ final class BufferSizeBenchTests: XCTestCase { } let handle = FileHandle(fileDescriptor: descriptor, closeOnDealloc: true) defer { try? handle.close() } - while let data = try handle.read(upToCount: bufferSize), !data.isEmpty { - hasher.update(data: data) + while true { + try Task.checkCancellation() + // The read itself must be inside the pool; pooling only the + // hasher update leaves Foundation's read buffers undrained. + let readData = try autoreleasepool { + guard let data = try handle.read(upToCount: bufferSize), !data.isEmpty else { return false } + hasher.update(data: data) + return true + } + if !readData { break } } } + try Task.checkCancellation() return hasher.finalize().map { String(format: "%02x", $0) }.joined() } - private func copy(files: [URL], to directory: URL, bufferSize: Int) throws { + private static func copy(files: [URL], to directory: URL, bufferSize: Int) throws { + try Task.checkCancellation() var buffer = [UInt8](repeating: 0, count: bufferSize) for source in files { + try Task.checkCancellation() let destination = directory.appendingPathComponent(source.lastPathComponent) let input = Darwin.open(source.path, O_RDONLY) try requireDescriptor(input, operation: "open", path: source.path) @@ -141,6 +252,7 @@ final class BufferSizeBenchTests: XCTestCase { guard Darwin.fcntl(output, F_NOCACHE, 1) != -1 else { throw posixError("F_NOCACHE", destination.path) } while true { + try Task.checkCancellation() let bytesRead = buffer.withUnsafeMutableBytes { storage in Darwin.read(input, storage.baseAddress, storage.count) } @@ -151,15 +263,19 @@ final class BufferSizeBenchTests: XCTestCase { } try writeAll(descriptor: output, bytes: buffer, count: bytesRead, path: destination.path) } + try Task.checkCancellation() guard Darwin.fcntl(output, F_FULLFSYNC) != -1 else { throw posixError("F_FULLFSYNC", destination.path) } + try Task.checkCancellation() } } - private func writeAll(descriptor: Int32, bytes: [UInt8], count: Int, path: String) throws { + private static func writeAll(descriptor: Int32, bytes: [UInt8], count: Int, path: String) throws { + try Task.checkCancellation() var written = 0 while written < count { + try Task.checkCancellation() let result = bytes.withUnsafeBytes { storage in Darwin.write(descriptor, storage.baseAddress?.advanced(by: written), count - written) } @@ -167,15 +283,16 @@ final class BufferSizeBenchTests: XCTestCase { if errno == EINTR { continue } throw posixError("write", path) } + guard result > 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(EIO)) } written += result } } - private func requireDescriptor(_ descriptor: Int32, operation: String, path: String) throws { + private static func requireDescriptor(_ descriptor: Int32, operation: String, path: String) throws { guard descriptor >= 0 else { throw posixError(operation, path) } } - private func posixError(_ operation: String, _ path: String) -> NSError { + private static func posixError(_ operation: String, _ path: String) -> NSError { NSError( domain: NSPOSIXErrorDomain, code: Int(errno), @@ -183,9 +300,48 @@ final class BufferSizeBenchTests: XCTestCase { ) } - private func seconds(since start: ContinuousClock.Instant) -> Double { - let duration = start.duration(to: ContinuousClock.now) - return Double(duration.components.seconds) + Double(duration.components.attoseconds) / 1e18 + private struct PhaseBreadcrumb: Encodable { + let event: String + let phase: String + let monotonicNanoseconds: UInt64 + let bufferBytes: Int + let run: Int + /// Intended bytes for this phase; not a count of physical-device I/O. + let logicalBytes: Int64 + let outcome: String? + } + + private static func emit(_ breadcrumb: PhaseBreadcrumb) { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + if let data = try? encoder.encode(breadcrumb) { + // Write immediately so an external counter observer can align the + // first hash with creation/copy/cleanup without stdout buffering. + try? FileHandle.standardOutput.write(contentsOf: data + Data([0x0A])) + } + } + + private static func measured(phase: String, buffer: Int, run: Int, logicalBytes: Int64, + emit: (PhaseBreadcrumb) -> Void = BufferSizeBenchTests.emit, + body: () throws -> T) throws -> (T, Double) { + let start = DispatchTime.now().uptimeNanoseconds + emit(PhaseBreadcrumb(event: "start", phase: phase, monotonicNanoseconds: start, + bufferBytes: buffer, run: run, logicalBytes: logicalBytes, outcome: nil)) + var outcome = "failed" + defer { + emit(PhaseBreadcrumb(event: "end", phase: phase, + monotonicNanoseconds: DispatchTime.now().uptimeNanoseconds, + bufferBytes: buffer, run: run, logicalBytes: logicalBytes, outcome: outcome)) + } + do { + let value = try body() + let seconds = Double(DispatchTime.now().uptimeNanoseconds - start) / 1e9 + outcome = "completed" + return (value, seconds) + } catch is CancellationError { + outcome = "cancelled" + throw CancellationError() + } } private func megabytesPerSecond(bytes: Int64, seconds: Double) -> Double { From 97d773ed586dddf86712015bdcfdf45dbea83226 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 23:25:25 -0400 Subject: [PATCH 30/56] Record overnight validation coverage and remaining limits --- docs/OVERNIGHT_VALIDATION_20261019.md | 89 +++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 docs/OVERNIGHT_VALIDATION_20261019.md diff --git a/docs/OVERNIGHT_VALIDATION_20261019.md b/docs/OVERNIGHT_VALIDATION_20261019.md new file mode 100644 index 00000000..f7494f4e --- /dev/null +++ b/docs/OVERNIGHT_VALIDATION_20261019.md @@ -0,0 +1,89 @@ +# Overnight validation — October 19–20, 2026 + +**Draft checkpoint.** This records local validation and its limits, not a release approval or proof that a field issue is fixed. The work started from `d6096442`; integration was at `4106a48a` when this draft was prepared. Several long-running tests were compiled earlier, as qualified below. Final integration and cleanup results still need to be recorded. + +[Issue #10](https://github.com/BitmatchApp/Bitmatch/issues/10) and [issue #23](https://github.com/BitmatchApp/Bitmatch/issues/23) remain open. These runs do not establish the reporters' exact failure mechanism. + +## Mounted filesystem matrices + +Disposable images exercised the actual macOS filesystem drivers. Source and destination images had separate mounted devices, but shared the host's underlying storage. This tests filesystem behavior, not independent hardware or power-loss recovery. + +| Run | Coverage | Result | +| --- | --- | --- | +| Transfer and evidence matrix | APFS, case-sensitive APFS, journaled HFS+, case-sensitive journaled HFS+, exFAT and FAT32; 288 one-backup cases and 270 mixed two-backup cases | 558 broad scenarios completed; see the split-run qualification below | +| Preserved metadata and history checks | Six source formats to APFS in Standard, Thorough and Paranoid, without the usual companion-file exclusion | 18 scenarios passed | +| Collision and metadata policy checks | 24 existing-content collisions plus 36 companion-file policy combinations | 60 scenarios passed; six expected metadata refusals | +| Recovery matrix | 12 cases on each of APFS, case-sensitive APFS, journaled HFS+, exFAT and FAT32 | 60 passed, zero failures or skips | +| Existing filesystem fault suite | Six filesystems, including case-sensitive HFS+ | 72 scheduled: 68 passed, four expected skips, zero failures | + +The first three rows cover **636 unique scenarios**, not 636 additional XCTest methods. A two-case smoke run overlaps that coverage and is not added to the total. Single-backup coverage uses all four verification modes with pipelining both off and on. Mixed two-backup coverage uses the three verified modes, with pipelining selected across the combinations; it is not every possible feature permutation. + +The initial combined matrix attempt failed on an incorrect harness assumption that a collision always produced one result row. FAT-family metadata can add rows. The 558 broad scenarios had completed; the corrected policy and preserved-history phases were then run separately. This is split-run evidence, not one uninterrupted green 636-case invocation. Six expected metadata-preservation refusals are passing safety assertions, not six verified transfers: those destinations received no complete history or safe verdict. The four existing-suite skips were its FAT-only handoff case on non-FAT filesystems. + +The recovery cases cover: + +- Cancellation during copying, destination readback and ASC MHL work; retry of identical content and refusal of changed existing content. +- Competing publication, case-only name conflicts, source symlinks and destination symlink ancestors, with protected bytes checked afterward. +- A deliberately changed, owned source fixture; the transfer must fail closed. +- Real volume exhaustion returning `ENOSPC` (errno 28), read-only image reattachment, and normal detach after preflight but before descriptor pinning. +- Interrupted MHL publication and retained incomplete evidence that cannot become trusted history or a successful verdict. + +Cancellation, race and mutation hooks control the interleavings. The disk-full and read-only cases use actual mounted filesystem behavior. Normal detach before pinning does not reproduce pulling a cable during a blocked kernel read. Symlink cases prepare their owned fixtures before starting; only the source-growth case deliberately mutates its source during a transfer. The recovery receipt records no remaining owned image mounts and no forced detach; the pair-matrix receipt records no remaining scratch roots. + +Earlier full-executor coverage is documented separately in [TRANSFER_MATRIX.md](TRANSFER_MATRIX.md): 800 configurations and 1,600 initial/repeat operations, plus project/report, naming and collision cases. Those historical counts are not added to this overnight matrix or represented as a fresh rerun here. + +## Physical SD to two SSDs + +One opt-in engine XCTest completed **nine scenarios, zero failures and zero skips** using an exFAT SD source and two exFAT external SSD destinations, called SSD A and SSD B here. Only uniquely marked validation folders were used; no physical drive was formatted or forcibly ejected. + +The generated source inventory contained 272 entries, including OS metadata companions. The selected transfer set contained 134 files and **2,151,787,901 bytes**; the complete inventoried source contained 2,152,353,149 bytes. The tests compared source inventory, sizes and content hashes before and after the run. They do not assert preservation of every filesystem attribute or access timestamp. + +The scenarios exercised Quick, Standard, Thorough and Paranoid with pipelining off and on, plus a Standard reuse pass. New verified transfers continued through ASC MHL and Saved Check; reuse checked that the prior history stayed unchanged. Quick remained unverified. Intentional metadata exclusions did not turn a selected-set result into a whole-card safety claim. + +The sampled engine-process physical footprint peaked at **32 MiB**. This is not the GUI app's memory footprint. The run overlapped another large transfer on SSD B, so its elapsed times are not a fair comparison of verification modes or drive performance. + +The running binary was built from `5396f6c5` plus the corrected test harness, before later assertion, cancellation and fixture-retention guards were added. Those later harness guards were compiled and tested separately. The completed physical run must not be described as a rerun of every final harness revision. + +An earlier physical large-file attempt also had a wrong MD5-versus-SHA-256 harness assertion. Completing its byte work does not make that attempt a pass. It is excluded from the successful physical results. + +## First buffer-benchmark read + +The selected SSD's opt-in raw benchmark completed **one XCTest, zero failures and zero skips**: 25 hash passes and 25 raw-copy passes, each over a 2 GiB fixture. This is a raw I/O/hash benchmark, not a verified application transfer. + +The first timed hash used a 64 KiB buffer and read 2,147,483,648 logical bytes in approximately 8.798 seconds. Read-only samples from the selected OS storage driver counted 2,105,344,000 read bytes in conservative intervals wholly inside that phase. The wider bracket counted 2,185,887,744 read bytes, including roughly 69 ms before and 135 ms after the phase. Selected-driver samples reported success. + +This confirms substantial **OS storage-driver read traffic during the first timed read**. Driver counters include other processes and metadata; they do not identify every byte as this test's work, prove NAND reads, or establish a cold hardware cache. Another long test was reading the other SSD. These conditions do not support a controlled throughput comparison or a buffer-default change. + +Before that run, the benchmark harness was corrected so each FileHandle read and hash update share an autorelease pool. Its long loops check cancellation cooperatively, and monotonic phase records distinguish source creation, hashing, raw copying and cleanup without paths. Three small harness tests passed; the large benchmark was initially skipped during that compile gate and enabled separately for the run above. Synchronous filesystem calls can still wait. + +## App, engine and simulator gates + +These are the latest completed full gates before the final benchmark-harness and high-resolution-probe additions. Logical test counts and parameterized case counts are kept separate. + +| Gate | Completed result | +| --- | --- | +| Engine Debug, XCTest | 220 scheduled: 183 passed, 37 opt-in/platform skips, zero failures | +| Engine Release, XCTest | 220 scheduled: 183 passed, 37 opt-in/platform skips, zero failures | +| Engine Debug, Swift Testing | 219 tests in 39 suites passed | +| Engine Release, Swift Testing | 219 tests in 39 suites passed | +| Full Mac app suite | 1,286 logical tests: 1,265 passed, 21 skipped, zero failures; no reported runtime warnings | +| iPad simulator | 79 tests: 69 passed, ten skipped, zero failures; no reported runtime warnings | +| iPhone simulator | 79 tests: 69 passed, ten skipped, zero failures; no reported runtime warnings | +| Shared mobile build | Universal iOS simulator build passed | +| Local signed Mac archive | Built and signature verified; no release published | + +Skipped opt-in tests are not counted as passes. Some were enabled in separate runs above; those runs do not turn the whole-suite skip count into zero. + +A separate signed-app walkthrough completed a Standard transfer to two physical backups and produced a real PDF with clip previews. Settings import Cancel/Apply and export persistence were also exercised. The final rebuilt app's interactive recheck was interrupted by a closed native automation pipe; the app remained alive and an independent sample showed its main thread idle. That automation failure is neither a passed final walkthrough nor evidence of an app hang. Component captures and simulator builds do not replace physical iPhone/iPad file-provider and background-execution testing. + +The diagnostics-export hardening had its own **32 passing XCTest cases**, zero skips and a universal mobile build. An oversized-input regression was observed failing before the fix. The tests cover malformed, oversized, unreadable, symlink and FIFO inputs, preserving valid events from the other input and reporting incomplete evidence honestly. This does not identify the cause of issue #23. + +## Pending finalization + +- **Large physical transfer:** the corrected 119.6 GB SSD A → SSD B case was still running at this checkpoint. Do not count it as a pass, infer its eventual verdict, or use an earlier flawed-harness attempt as a substitute. +- **Application performance:** isolated repeated Standard/Thorough application trials and their counter-aligned measurements need a settled receipt. Raw buffer throughput is not application performance evidence. +- **4K report-preview memory:** the new host-only 20-path and 200-path probe has preliminary successful runs. Final reviewed receipts and the qualified result belong here before it is counted. Its fixture repeats one two-frame H.264 pattern across distinct paths; it does not cover codec diversity, long GOPs or distinct camera clips. +- **Intel:** a separate translated-x86/Intel compatibility receipt needs final review. Apple Silicon results and a universal build are not execution on a physical Intel Mac. +- **Final revision and cleanup:** update this document after the remaining jobs settle, final combined gates complete, and owned media fixtures, images, caches and simulators are accounted for. Retain compact receipts; preserve uncertain or failed evidence rather than deleting it prematurely. + +No run here reproduces a real HDD through the reporter's dock, a USB controller fault, power loss, a forced unplug during active I/O, physical-media failure, or every iOS file-provider/background condition. The successful cases strengthen the tested paths; they do not close issues #10 or #23 without decisive field evidence. From bb2c99190596b821de822e36430534f5ae291efd Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 23:28:36 -0400 Subject: [PATCH 31/56] test: bound memory during 4K report preview decoding --- BitMatchTests/ReportPreviewMemoryTests.swift | 342 ++++++++++++++++++ .../report-preview-memory-2026-10-19.md | 61 ++++ 2 files changed, 403 insertions(+) create mode 100644 BitMatchTests/ReportPreviewMemoryTests.swift create mode 100644 docs/validation/report-preview-memory-2026-10-19.md diff --git a/BitMatchTests/ReportPreviewMemoryTests.swift b/BitMatchTests/ReportPreviewMemoryTests.swift new file mode 100644 index 00000000..5b097a76 --- /dev/null +++ b/BitMatchTests/ReportPreviewMemoryTests.swift @@ -0,0 +1,342 @@ +#if os(macOS) +@preconcurrency import AVFoundation +import CoreVideo +import CryptoKit +import Darwin +import Foundation +import ImageIO +import PDFKit +import Synchronization +import XCTest +@testable import BitMatch +@testable import BitMatchEngine + +/// Host scratch only. Real 4K decoding, not a reproduction of issue #23. +/// The guard requests cooperative cancellation; native calls may still wait. +@MainActor +final class ReportPreviewMemoryTests: XCTestCase { + func testMemoryLimitCancelsOwnedWorkWithoutLargeAllocations() async throws { + let memory = PreviewProbeMemory() + XCTAssertTrue(memory.record(1_024)) + let worker = Task { try await Task.sleep(for: .seconds(60)) } + let monitor = memory.monitor { worker.cancel() } + do { try await worker.value; XCTFail("Memory guard failed to cancel owned work") } + catch { XCTAssertTrue(error is CancellationError) } + monitor.cancel(); await monitor.value + XCTAssertThrowsError(try memory.requireSafe()) { XCTAssertTrue($0 is PreviewProbeError) } + XCTAssertTrue(memory.snapshot().guardTriggered) + } + + func testMissingMemorySampleStopsWorkRatherThanReportingZero() throws { + let memory = PreviewProbeMemory() + XCTAssertTrue(memory.record(nil)) + XCTAssertTrue(memory.snapshot().sampleFailure) + XCTAssertThrowsError(try memory.requireSafe()) + } + + func testVerified4KClipsProduceBoundedReportPreviews() async throws { + let environment = ProcessInfo.processInfo.environment + guard environment["BITMATCH_HIGH_RES_PREVIEW_MEMORY"] == "1" else { + throw XCTSkip("Set BITMATCH_HIGH_RES_PREVIEW_MEMORY=1 for the guarded 20-clip 4K decoder probe") + } + let clipCount = Int(environment["BITMATCH_HIGH_RES_PREVIEW_CLIPS"] ?? "20") ?? -1 + guard (20...ReportThumbnailService.maximumPreviews).contains(clipCount) else { throw PreviewProbeError.invalidFixture } + let fm = FileManager.default + let root = fm.temporaryDirectory.appendingPathComponent("bitmatch-4k-preview-\(UUID())") + let artifactsBase = URL(fileURLWithPath: environment["BITMATCH_HIGH_RES_PREVIEW_ARTIFACTS"] ?? "/private/tmp") + .standardizedFileURL.resolvingSymlinksInPath() + guard !artifactsBase.path.hasPrefix("/Volumes/") else { throw PreviewProbeError.invalidFixture } + let artifacts = artifactsBase.appendingPathComponent("preview-memory-\(UUID())") + for directory in [root, artifacts] { try fm.createDirectory(at: directory, withIntermediateDirectories: true) } + let source = root.appendingPathComponent("Card") + let backup = root.appendingPathComponent("Backup") + for directory in [source, backup] { try fm.createDirectory(at: directory, withIntermediateDirectories: true) } + let memory = PreviewProbeMemory(clipCount: clipCount) + _ = memory.sample() + let pipeline = TransferPipeline(fileSystem: MacOSFileSystemService.shared, checksum: ChecksumEngine.shared) + let worker = Task { @MainActor in + try await run(source: source, backup: backup, artifacts: artifacts, memory: memory, pipeline: pipeline) + } + let monitor = memory.monitor { worker.cancel(); pipeline.cancelOperation() } + do { + try await withTaskCancellationHandler { try await worker.value } onCancel: { + worker.cancel(); pipeline.cancelOperation() + } + guard !memory.snapshot().guardTriggered else { throw PreviewProbeError.memoryGuard } + guard testRun?.totalFailureCount == 0 else { throw PreviewProbeError.assertionFailure } + memory.mark("settling") + try await Task.sleep(for: .seconds(2)) + _ = memory.sample() + guard !memory.snapshot().guardTriggered else { throw PreviewProbeError.memoryGuard } + monitor.cancel(); await monitor.value + // The monitor may have latched a failure after the prior check. + try memory.requireSafe() + try save(memory, outcome: "passed", to: artifacts) + try fm.removeItem(at: root) + print("PREVIEW_MEMORY_RESULT clips=\(clipCount) width=3840 height=2160 outcome=passed artifacts=\(artifacts.path)") + } catch { + worker.cancel(); pipeline.cancelOperation() + // run() awaits its owned seed task before escaping; collection + // cancellation requests native AVFoundation cancellation as usual. + monitor.cancel(); await monitor.value + _ = memory.sample() + try? save(memory, outcome: memory.snapshot().guardTriggered ? "guarded" : "failed", to: artifacts) + print("PREVIEW_MEMORY_FAILURE retainedFixture=\(root.path) artifacts=\(artifacts.path)") + throw error + } + } + + private func run(source: URL, backup: URL, artifacts: URL, memory: PreviewProbeMemory, + pipeline: TransferPipeline) async throws { + memory.mark("fixture") + try memory.requireSafe() + let clipCount = memory.snapshot().sourceClips + let seed = Task.detached { try await Self.seed(source, clipCount: clipCount) } + let original = try await withTaskCancellationHandler { try await seed.value } onCancel: { seed.cancel() } + try Task.checkCancellation() + let encoder = JSONEncoder(); encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + try encoder.encode(original).write(to: artifacts.appendingPathComponent("source-sha256.json"), options: .atomic) + memory.mark("copyAndVerify") + try memory.requireSafe() + let operation = try await pipeline.performFileOperation(sourceURL: source, destinationURLs: [backup], + verificationMode: .standard, settings: CameraLabelSettings(), estimatedTotalBytes: nil, + progressCallback: { _ in }, onFileResult: nil) + let rows = TransferCompletion.rows(from: operation) + guard rows.count == clipCount, rows.allSatisfy(\.isVerifiedStatus) else { throw PreviewProbeError.incompleteVerification } + memory.state.withLock { $0.verifiedRows = rows.count } + try Task.checkCancellation() + memory.mark("decoding") + try memory.requireSafe() + let started = ProcessInfo.processInfo.systemUptime + let thumbnails = try await ReportThumbnailService.collect(results: rows, enabled: true) + let elapsed = ProcessInfo.processInfo.systemUptime - started + memory.state.withLock { $0.collected = thumbnails.images.count; $0.collectionSeconds = elapsed } + guard thumbnails.requestedCount == clipCount, thumbnails.images.count == clipCount else { throw PreviewProbeError.incompletePreviews } + for (index, path) in thumbnails.images.keys.sorted().enumerated() { + try memory.requireSafe() + guard let data = thumbnails.images[path] else { throw PreviewProbeError.invalidPreview } + try data.write(to: artifacts.appendingPathComponent(String(format: "preview-%02d.jpg", index))) + try Self.validateJPEG(data) + } + memory.mark("pdf") + try memory.requireSafe() + let summary = ReportSummary(jobID: UUID(), started: operation.startTime, finished: Date(), mode: .copyAndVerify, + source: source.path, destinations: [backup.path], totalFiles: rows.count, matched: rows.count, issues: 0, + workers: 1, appVersion: "synthetic probe", osVersion: ProcessInfo.processInfo.operatingSystemVersionString, + client: "", production: "", company: "", verificationMethod: "Standard", verificationMode: .standard, + totalBytesProcessed: rows.reduce(0) { $0 + $1.size }, + averageSpeed: 0, clientLogoData: nil, companyLogoData: nil, photographyJob: nil, + operatorName: "BitMatch validation", notes: "Synthetic 4K preview memory probe", safetyState: .safeToErase) + let pdf = ReportPDFRenderer.renderPDF(summary: summary, results: rows, thumbnails: thumbnails) + // Preserve the generated bytes even if validation or the guard fails. + try pdf.write(to: artifacts.appendingPathComponent("verified-4k-previews.pdf")) + try memory.requireSafe() + guard let document = PDFDocument(data: pdf), let text = document.string, + text.contains("Clip previews: \(clipCount)/\(clipCount)"), text.contains("VERIFICATION SUCCESSFUL") else { throw PreviewProbeError.invalidPDF } + for name in original.keys { guard text.contains(name) else { throw PreviewProbeError.invalidPDF } } + let embedded = try Self.embeddedPreviews(pdf) + guard embedded > 0 else { throw PreviewProbeError.invalidPDF } + memory.state.withLock { $0.pdfPages = document.pageCount; $0.embeddedImageResources = embedded } + memory.mark("sourceAndDestinationChecks") + let checked = Task.detached { + let after = try Self.hashes(source) + let copied = try Self.hashes(backup.appendingPathComponent(source.lastPathComponent), ignoreTransferStatus: true) + guard after == original, copied == original else { throw PreviewProbeError.changedMedia } + } + try await withTaskCancellationHandler { try await checked.value } onCancel: { checked.cancel() } + try Task.checkCancellation() + } + + private func save(_ memory: PreviewProbeMemory, outcome: String, to artifacts: URL) throws { + var snapshot = memory.snapshot(); snapshot.outcome = outcome + let encoder = JSONEncoder(); encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + try encoder.encode(snapshot).write(to: artifacts.appendingPathComponent("memory.json"), options: .atomic) + } + + nonisolated private static func seed(_ source: URL, clipCount: Int) async throws -> [String: String] { + let first = source.appendingPathComponent("clip-00.mov") + let writer = try AVAssetWriter(outputURL: first, fileType: .mov) + defer { if writer.status != .completed { writer.cancelWriting() } } + let input = AVAssetWriterInput(mediaType: .video, outputSettings: [AVVideoCodecKey: AVVideoCodecType.h264, + AVVideoWidthKey: 3_840, AVVideoHeightKey: 2_160, + AVVideoCompressionPropertiesKey: [AVVideoProfileLevelKey: AVVideoProfileLevelH264HighAutoLevel, + AVVideoAverageBitRateKey: 2_000_000, AVVideoAllowFrameReorderingKey: false]]) + let adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input, + sourcePixelBufferAttributes: [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32ARGB]) + writer.add(input) + guard writer.startWriting() else { throw writer.error ?? PreviewProbeError.invalidFixture } + writer.startSession(atSourceTime: .zero) + var pixel: CVPixelBuffer? + guard CVPixelBufferCreate(kCFAllocatorDefault, 3_840, 2_160, kCVPixelFormatType_32ARGB, nil, &pixel) == kCVReturnSuccess, + let buffer = pixel else { throw PreviewProbeError.invalidFixture } + CVPixelBufferLockBaseAddress(buffer, []) + do { + defer { CVPixelBufferUnlockBaseAddress(buffer, []) } + guard let base = CVPixelBufferGetBaseAddress(buffer) else { throw PreviewProbeError.invalidFixture } + let bytes = base.assumingMemoryBound(to: UInt8.self) + let stride = CVPixelBufferGetBytesPerRow(buffer) + for y in 0..<2_160 { + try Task.checkCancellation() + for x in 0..<3_840 { + let offset = y * stride + x * 4 + bytes[offset] = 255; bytes[offset + 1] = x < 1_920 ? 224 : 32 + bytes[offset + 2] = y < 1_080 ? 224 : 32; bytes[offset + 3] = 64 + } + } + } + for index in 0..<2 { + try Task.checkCancellation() + guard await waitUntil(timeout: .seconds(10), { input.isReadyForMoreMediaData }) else { throw PreviewProbeError.invalidFixture } + try Task.checkCancellation() + guard adaptor.append(buffer, withPresentationTime: CMTime(value: Int64(index), timescale: 30)) else { + throw writer.error ?? PreviewProbeError.invalidFixture + } + } + input.markAsFinished(); await writer.finishWriting() + try Task.checkCancellation() + guard writer.status == .completed else { throw writer.error ?? PreviewProbeError.invalidFixture } + let tracks = try await AVURLAsset(url: first).loadTracks(withMediaType: .video) + guard let track = tracks.first, try await track.load(.naturalSize) == CGSize(width: 3_840, height: 2_160) else { + throw PreviewProbeError.invalidFixture + } + for index in 1.. [String: String] { + let files = try CardSource.enumerateRegularFiles(base: root).filter { item in + if ignoreTransferStatus && TransferStatusMarker.hasExactName(item.relativePath) { + guard TransferStatusMarker.validatedStatus(at: item.url, relativePath: item.relativePath) == .filesVerified else { + throw PreviewProbeError.changedMedia + } + return false + } + return true + } + return try Dictionary(uniqueKeysWithValues: files.map { item in + try Task.checkCancellation() + let descriptor = Darwin.open(item.url.path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC) + guard descriptor >= 0 else { throw PreviewProbeError.changedMedia } + let file = FileHandle(fileDescriptor: descriptor, closeOnDealloc: true); defer { try? file.close() } + var hash = SHA256() + while try autoreleasepool(invoking: { + try Task.checkCancellation() + let data = try file.read(upToCount: 1_048_576) ?? Data() + if data.isEmpty { return false }; hash.update(data: data); return true + }) {} + return (item.relativePath, hash.finalize().map { String(format: "%02x", $0) }.joined()) + }) + } + + nonisolated private static func validateJPEG(_ data: Data) throws { + guard data.count <= 64 * 1_024, let provider = CGImageSourceCreateWithData(data as CFData, nil), + let image = CGImageSourceCreateImageAtIndex(provider, 0, nil), + image.width == 256, image.height == 144 else { throw PreviewProbeError.invalidPreview } + var pixels = [UInt8](repeating: 0, count: 256 * 144 * 4) + let valid = pixels.withUnsafeMutableBytes { storage -> Bool in + guard let base = storage.baseAddress, + let context = CGContext(data: base, width: 256, height: 144, bitsPerComponent: 8, + bytesPerRow: 256 * 4, space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return false } + context.draw(image, in: CGRect(x: 0, y: 0, width: 256, height: 144)) + let bytes = storage.bindMemory(to: UInt8.self) + let left = Int(bytes[(36 * 256 + 64) * 4]); let right = Int(bytes[(36 * 256 + 192) * 4]) + let top = Int(bytes[(36 * 256 + 64) * 4 + 1]); let bottom = Int(bytes[(108 * 256 + 64) * 4 + 1]) + return abs(left - right) > 100 && abs(top - bottom) > 100 + } + guard valid else { throw PreviewProbeError.invalidPreview } + } + + nonisolated private static func embeddedPreviews(_ pdf: Data) throws -> Int { + guard let provider = CGDataProvider(data: pdf as CFData), let document = CGPDFDocument(provider) else { throw PreviewProbeError.invalidPDF } + let collector = PreviewProbeImages() + for index in 1...document.numberOfPages { + guard let page = document.page(at: index), let dictionary = page.dictionary else { throw PreviewProbeError.invalidPDF } + var resources: CGPDFDictionaryRef?; var images: CGPDFDictionaryRef? + if CGPDFDictionaryGetDictionary(dictionary, "Resources", &resources), let resources, + CGPDFDictionaryGetDictionary(resources, "XObject", &images), let images { + CGPDFDictionaryApplyFunction(images, { _, object, context in + guard let context else { return } + let collector = Unmanaged.fromOpaque(context).takeUnretainedValue() + var stream: CGPDFStreamRef? + guard CGPDFObjectGetValue(object, .stream, &stream), let stream, + let dictionary = CGPDFStreamGetDictionary(stream) else { return } + var subtype: UnsafePointer? + guard CGPDFDictionaryGetName(dictionary, "Subtype", &subtype), let subtype, + String(cString: subtype) == "Image" else { return } + var format = CGPDFDataFormat.raw + guard let bytes = CGPDFStreamCopyData(stream, &format), format == .jpegEncoded else { collector.invalid = true; return } + do { try ReportPreviewMemoryTests.validateJPEG(bytes as Data); collector.count += 1 } + catch { collector.invalid = true } + }, Unmanaged.passUnretained(collector).toOpaque()) + } + } + guard !collector.invalid else { throw PreviewProbeError.invalidPDF } + return collector.count + } +} + +private final class PreviewProbeImages { var count = 0; var invalid = false } +private enum PreviewProbeError: Error { + case invalidFixture, incompleteVerification, incompletePreviews, invalidPreview, invalidPDF + case memoryGuard, changedMedia, assertionFailure +} +private final class PreviewProbeMemory: Sendable { + struct Phase: Encodable, Sendable { var initialMiB: Int; var peakMiB: Int; var lastMiB: Int } + struct State: Encodable, Sendable { + var outcome = "running" + var phase = "initial" + var phases: [String: Phase] = [:] + var initialMiB = 0; var peakMiB = 0; var afterMiB = 0; var samples = 0 + var guardTriggered = false; var sampleFailure = false + var verifiedRows = 0; var collected = 0; var collectionSeconds = 0.0 + var pdfPages = 0; var embeddedImageResources = 0 + let guardMiB = 1_024 + let sourceWidth = 3_840; let sourceHeight = 2_160; let sourceClips: Int + let maximumPreviews = ReportThumbnailService.maximumPreviews + let collectionBudgetSeconds = ReportThumbnailService.maximumSeconds + let perClipSeconds = ReportThumbnailService.perClipSeconds + let scope = "Synthetic host-only 4K H264 decoding/report probe. Sampled process physical footprint; not a reproduction or proof of issue #23. Native calls are not forcibly interrupted." + } + let state: Mutex + init(clipCount: Int = 20) { state = Mutex(State(sourceClips: clipCount)) } + func snapshot() -> State { state.withLock { $0 } } + func requireSafe() throws { + if sample() { throw PreviewProbeError.memoryGuard } + try Task.checkCancellation() + } + func monitor(cancel: @escaping @Sendable () -> Void) -> Task { + Task.detached { + while !Task.isCancelled { + if self.sample() { cancel() } + do { try await Task.sleep(for: .milliseconds(10)) } catch { break } + } + } + } + func mark(_ phase: String) { state.withLock { $0.phase = phase }; _ = sample() } + @discardableResult func sample() -> Bool { + var info = task_vm_info_data_t() + var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) + let status = withUnsafeMutablePointer(to: &info) { pointer in + pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { + task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count) + } + } + return record(status == KERN_SUCCESS ? Int(info.phys_footprint / 1_048_576) : nil) + } + @discardableResult func record(_ value: Int?) -> Bool { + state.withLock { state in + guard let value else { state.sampleFailure = true; state.guardTriggered = true; return true } + if state.samples == 0 { state.initialMiB = value } + state.samples += 1; state.afterMiB = value; state.peakMiB = max(state.peakMiB, value) + var phase = state.phases[state.phase] ?? Phase(initialMiB: value, peakMiB: value, lastMiB: value) + phase.peakMiB = max(phase.peakMiB, value); phase.lastMiB = value; state.phases[state.phase] = phase + if value >= state.guardMiB { state.guardTriggered = true } + return state.guardTriggered + } + } +} +#endif diff --git a/docs/validation/report-preview-memory-2026-10-19.md b/docs/validation/report-preview-memory-2026-10-19.md new file mode 100644 index 00000000..5664c276 --- /dev/null +++ b/docs/validation/report-preview-memory-2026-10-19.md @@ -0,0 +1,61 @@ +# Synthetic 4K report preview memory probe + +October 19 overnight, 2026. Mac host scratch only, using the real Standard +copy/readback pipeline, `ReportThumbnailService` and PDF renderer. No physical +card/backup, MHL handoff or durable History workflow is claimed by this probe. + +One original, locally generated 3840×2160 H.264 asset contains two frames of a +four-colour quadrant pattern. It was materialized at 20 distinct source paths, +then 200 after the smaller run stayed comfortably below the guard. This tests +repeated-path decoder/report handling, not unique clip content, codec diversity, +long-GOP footage, large encoded files or the cause of issue #23. + +| Final run | Tests passed / skipped | Initial / overall peak / after settling | Decoder peak | PDF peak | Collection | +| --- | --- | --- | --- | --- | --- | +| Normal, opt-in disabled | 2 / 1 | Not run | Not run | Not run | Not run | +| 20 clips | 3 / 0 | 45 / 103 / 83 MiB | 79 MiB | 82 MiB | 20/20 in 0.548 s | +| 200 clips | 3 / 0 | 45 / 103 / 99 MiB | 82 MiB | 97 MiB | 200/200 in 4.532 s | + +All runs had zero failures and zero xcresult runtime warnings. The opt-in runs +include the two small guard tests plus one real probe. Final test times were +0.004 s (normal), 4.058 s (20) and 13.240 s (200). Sampling used the process's +`phys_footprint` every 10 ms, with separate fixture, copy/verify, decoder, PDF, +post-check and two-second settling phases. Brief peaks between samples can be +missed. The largest peak occurred while constructing the 4K fixture. Neither +real run reached the 1,024 MiB guard; small mocked tests check threshold-triggered +worker cancellation and refusal to treat a missing sample as zero memory. + +The guard cancels the owned worker and pipeline and prevents subsequent phases. +Owned fixture/hash tasks are joined before returning. It requests native decoder +cancellation but cannot forcibly interrupt AVFoundation, synchronous PDF drawing +or filesystem calls. Failed/guarded fixtures and generated bytes are retained. +This is a bounded probe, not an out-of-memory survival guarantee or a leak proof. + +Production limits stayed unchanged: 200 previews, a 30-second collection budget +and a two-second per-clip timeout. Every actual pipeline result was independently +hash-verified, and source/destination SHA-256 snapshots matched afterward. Any +engine status marker encountered by the hash helper must be valid; this host +probe does not establish that a marker is required or present on every filesystem. +Every collected JPEG was at most 64 KiB, exactly 256×144, and retained the +expected red/green quadrant differences. PDF resources passed the same pixel +check. Independent Poppler inspection counted 20 and 200 JPEG **placements**; +Quartz reused the identical synthetic image resources. Captions retained every +source clip. All 3/15 PDF pages were rendered privately; this is headless report +evidence, not a signed-app or device walkthrough. + +The initial renders exposed a separate pagination rough edge: a last page with +only Job ID/version. These memory results precede its separate layout work. + +Run only with the Mac host reserved: + +```sh +TEST_RUNNER_BITMATCH_HIGH_RES_PREVIEW_MEMORY=1 \ +TEST_RUNNER_BITMATCH_HIGH_RES_PREVIEW_ARTIFACTS=/private/tmp/bitmatch-preview-evidence \ +xcodebuild -project BitMatch.xcodeproj -scheme BitMatch \ + -destination 'platform=macOS' -jobs 4 -parallel-testing-enabled NO \ + -only-testing:BitMatchTests/ReportPreviewMemoryTests test +``` + +The default is 20 clips. `TEST_RUNNER_BITMATCH_HIGH_RES_PREVIEW_CLIPS` accepts +20–200; increase only after the smaller measured run is safe. Exported probe +PDFs contain generated local paths, so keep raw artifacts private. From c28c7c23809f811a8e50fbf636bea6681df2464b Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 23:29:32 -0400 Subject: [PATCH 32/56] Record translated x86 engine validation and limits --- docs/validation/ROSETTA_ENGINE_20261019.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 docs/validation/ROSETTA_ENGINE_20261019.md diff --git a/docs/validation/ROSETTA_ENGINE_20261019.md b/docs/validation/ROSETTA_ENGINE_20261019.md new file mode 100644 index 00000000..a1c6e874 --- /dev/null +++ b/docs/validation/ROSETTA_ENGINE_20261019.md @@ -0,0 +1,16 @@ +# Translated x86_64 engine validation — October 19, 2026 + +The Release engine at `4106a48a` was cross-compiled as an x86_64-only test bundle and executed through the already installed Rosetta translation on an Apple Silicon Mac running macOS 27.0.1, using Xcode 27.1. This is translated engine coverage, not execution on a physical Intel Mac or an older macOS release. The universal Mac app build is separate evidence. + +| Framework | Scheduled | Passed | Skipped | Failures | +| --- | ---: | ---: | ---: | ---: | +| XCTest | 223 | 186 | 37 | 0 | +| Swift Testing | 219 declarations in 39 suites | 213 | 6 | 0 | + +The active Swift Testing declarations executed 333 invocations: 153 parameterized cases and 180 ordinary tests. Hardware, large-memory, mounted-image, soak and benchmark opt-ins were unset. No app host or physical-drive workload ran. + +Xcode's Swift compiler and stock SwiftPM testing helpers were ARM-only; forcing those helpers to x86_64 failed before discovery. The native compiler built the x86_64 tests with Release optimization and testable imports. XCTest used Xcode's universal host with x86_64 explicitly selected. Swift Testing used a private x86_64 build of the [minimal upstream SwiftPM bundle loader](https://github.com/swiftlang/swift-package-manager/blob/232c7e26344962870cb0d19ae8b7399f584c6c32/Sources/swiftpm-testing-helper/Entrypoint.swift), licensed under Apache 2.0 with the Swift Runtime Library Exception. Its logic was unchanged; Foundation linkage and Xcode framework search paths matched the installed test environment. + +The unsigned private loader initially produced 13 bookmark-setup failures with Cocoa error 256. Giving only that disposable loader an explicit ad-hoc signing identity, with no added entitlements, cleared those failures in the complete rerun. Product code and test assertions were unchanged. The failed control is retained separately from the passing result; it is not hidden as a skip. + +Architecture checks confirmed that both the bundle and private loader contained only x86_64. Temporary binaries and build caches were removed after collecting compact counts, logs, signing/provenance metadata and architecture receipts. These results do not establish native Intel performance, macOS 15.0 runtime compatibility, physical hardware reliability or the cause of an open field report. From d3df29ba530096b54907fa5fc12ba8b50573cdbb Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 23:32:01 -0400 Subject: [PATCH 33/56] Update local validation evidence and remaining release gates --- CHANGELOG.md | 5 +++++ docs/HARDWARE_COMPATIBILITY.md | 8 ++++---- docs/OVERNIGHT_VALIDATION_20261019.md | 14 +++++++------- docs/TODO.md | 18 +++++++++--------- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f829bae..9b5b4615 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +- Check no longer reports preserved Finder metadata as missing when it is listed in the saved checksums. +- Corrected physical-drive checks for selected subfolders, so two folders on one drive count as one backup. +- Cleaned up long names and page breaks in PDFs; metadata companions no longer inflate the clip-preview count. +- Mac: Advanced points out a selected backup using a reported USB 2 connection. +- Diagnostics exports reject oversized or unreadable local records, keep the usable events, and say when the record is incomplete. - Destination progress no longer resets to Waiting while ASC MHL and transfer reports are being saved. Processed counts and measured progress stay visible without implying the transfer is complete. - Mac Settings now scroll when a pane is taller than the window, keeping the first and last controls reachable. - Importing transfer settings on Mac uses a compact review sheet, with longer change lists scrollable above Cancel and Apply. diff --git a/docs/HARDWARE_COMPATIBILITY.md b/docs/HARDWARE_COMPATIBILITY.md index 105467b2..2c43148d 100644 --- a/docs/HARDWARE_COMPATIBILITY.md +++ b/docs/HARDWARE_COMPATIBILITY.md @@ -1,6 +1,6 @@ # Hardware validation status -No physical-device test reports are recorded in this matrix yet. The mounted disk-image matrix covers APFS, HFS+, exFAT and FAT32 where the test host can attach those images; FAT32 could not be mounted on this Mac for the current publication change. Disk images do not validate physical enclosures, hubs, readers or cables. Those combinations remain **unverified here** until a reproducible report is published. This is an evidence register, not a certification or a list of unsupported hardware. +The [October 19–20 validation](OVERNIGHT_VALIDATION_20261019.md) records an exFAT SD card copied to two physical exFAT SSDs, plus mounted-image coverage of APFS, HFS+, exFAT and FAT32, including case-sensitive variants. Results apply only to their recorded revision, devices and scenarios. Disk images do not validate physical enclosures, hubs, readers or cables, and neither kind of passing transfer proves power-loss behavior. This is an evidence register, not a certification or a list of unsupported hardware. | Evidence type | Available workflow | What it establishes | | --- | --- | --- | @@ -8,7 +8,7 @@ No physical-device test reports are recorded in this matrix yet. The mounted dis | Automated seeded soak | `bash Scripts/run_soak_tests.sh` | Repeated synthetic transfers with independent output hashes on the host filesystem, for the recorded seed and iteration count. | | Automated APFS fault | `bash Scripts/run_apfs_fault_tests.sh` | Handling of an injected inaccessible destination in a disposable APFS image. | | Mounted filesystem and feature matrix | [Synthetic transfer matrix](TRANSFER_MATRIX.md) | Copy, verification, ASC MHL, project workflows and reports on disposable mounted images; no physical cable or controller faults. | -| Physical storage and connection tests | [Hardware testing procedure](HARDWARE_TESTING.md) | Only the specific device, filesystem, connection, revision, and scenario recorded in a report. No physical results have been entered. | +| Physical storage and connection tests | [Hardware testing procedure](HARDWARE_TESTING.md) | Only the specific device, filesystem, connection, revision, and scenario recorded in a report. The first local SD/two-SSD result is listed below. | These rows describe available tests, not claims that they passed. A published automated result must include the run date, revision, working-tree state, exit status, test totals (including skipped tests), and a link to retained evidence. Automated results do not count as physical cable-pull or power-loss results. @@ -31,7 +31,7 @@ What the code on `main` does for these cases, with where it lives. These are cod | Sony VENICE SxS card (UDF) on a Mac | If Disk Arbitration reports a removable disk with no readable filesystem and its description matches SxS, BitMatch shows a setup notice with Sony driver guidance. This notice is a heuristic, not a diagnosis of every Sony card. | `BitMatch/Core/Services/UnreadableMediaMonitor.swift` (`UnreadableMediaNotice.make`), shown by `BitMatch/Views/CopyAndVerify/UnreadableMediaBanner.swift`; tests `UnreadableMediaNoticeTests`, `UnreadableMediaMonitorTests` | None recorded. Detection matches "sxs", or "sony" with "card", in the disk's vendor, model or media name. | | Sony AXS card on a Mac | If an unreadable disk's description matches AXS, the same notice path links to Sony reader-software support. | same as above | None recorded. Matches "axs" in the disk's description. | | Any other card macOS sees but cannot read | A generic notice: it may need its maker's driver or use an unsupported format. | same as above | None recorded. | -| exFAT (and FAT) backup drive | BitMatch saves a visible `BITMATCH_TRANSFER_STATUS_.txt` warning before creating media files at their final names with exclusive create. It flushes new folder entries, each media file, and its name. An interrupted write leaves its partial file and incomplete warning in place; an existing file is never replaced. The warning alone does not mean a backup is safe to erase; check BitMatch History. A filesystem without hard links that BitMatch does not recognize fails closed with a clear error. ASC MHL directory publication remains covered by the [handoff matrix](TRANSFER_MATRIX.md). | `Packages/BitMatchEngine/Sources/BitMatchEngine/File/DestinationWriter.swift` and `TransferStatusMarker.swift`; mounted-image tests `ExFATDestinationTests` and `Issue10ExFATHandoffTests` | The original bug was found on real exFAT hardware (lukasisar/Bitmatch#4). Marker flush, root-folder sync, full Standard readback and ASC MHL handoff passed on a mounted exFAT image. No physical drive has been validated. FAT32 mounting failed on this test Mac before its sync behavior could be measured. | +| exFAT (and FAT) backup drive | BitMatch saves a visible `BITMATCH_TRANSFER_STATUS_.txt` warning before creating media files at their final names with exclusive create. It flushes new folder entries, each media file, and its name. An interrupted write leaves its partial file and incomplete warning in place; an existing file is never replaced. The warning alone does not mean a backup is safe to erase; check BitMatch History. A filesystem without hard links that BitMatch does not recognize fails closed with a clear error. ASC MHL directory publication remains covered by the [handoff matrix](TRANSFER_MATRIX.md). | `Packages/BitMatchEngine/Sources/BitMatchEngine/File/DestinationWriter.swift` and `TransferStatusMarker.swift`; mounted-image tests `ExFATDestinationTests` and `Issue10ExFATHandoffTests` | The original bug was found on real exFAT hardware (lukasisar/Bitmatch#4). Marker flush, root-folder sync, full Standard readback and ASC MHL handoff passed on a mounted exFAT image. The October 19–20 run exercised an exFAT SD source and two exFAT physical SSD backups. FAT32 image tests now include sync, handoff, interruption and exhaustion. See the qualified results below; physical FAT32, HDD/dock and power-loss tests remain open. | | A folder on the Mac's startup disk as a backup | Allowed when you pick it (for example in your home folder). The startup disk itself ("/", which `/Volumes/Macintosh HD` resolves to), anything under `/System`, and the root of an internal volume with a system name (Recovery, "Recovery 2", Preboot, any letter case) are refused with a reason. Drive discovery and the launch-time restore never add them. The engine's own preflight also refuses `/Library`, `/usr`, `/bin`, `/sbin`, `/private`, `/var` and `/etc`. | `Packages/BitMatchEngine/Sources/BitMatchEngine/File/BackupTargetPolicy.swift` (`refusal(for:origin:source:)`), `Packages/BitMatchEngine/Sources/BitMatchEngine/File/SafetyValidator.swift` (`isProtectedSystemPath`); tests `BackupTargetPolicyTests`, `BackupTargetPolicyRealVolumeTests` (real paths on the test Mac) | A mounted internal Recovery volume being offered as a backup was confirmed and fixed on a Mac (see [THESIS.md](THESIS.md), "Must fix before the next release"). | | iPad/iPhone backup chosen in Files (On My iPad, iCloud Drive, an external drive) | Works from 0.1.7. On a device these folders live below `/private/var/mobile`, which the system-folder rule refused, so every backup was refused as a "system folder". `/var/mobile` and `/private/var/mobile` are now user storage on iOS; other system folders stay protected. | `Packages/BitMatchEngine/Sources/BitMatchEngine/File/SafetyValidator.swift` (`isProtectedSystemPath`, `iOSUserStorageRoots`; commit `31bc940`); test `BitMatch-iPadTests/IOSStoragePathTests.swift` | **Not yet confirmed on a physical iPhone or iPad.** The simulator cannot show the bug: its paths are under the Mac's `/Users`. | @@ -39,6 +39,6 @@ What the code on `main` does for these cases, with where it lives. These are cod | Report | Date | App revision / OS | Source and destinations / filesystems | Connection | Scenario | Outcome | | --- | --- | --- | --- | --- | --- | --- | -| No reports recorded | — | — | — | — | — | Not tested | +| [Local SD / two-SSD matrix](OVERNIGHT_VALIDATION_20261019.md#physical-sd-to-two-ssds) | 2026-10-19–20 | Engine `5396f6c5` plus corrected test harness; macOS 27.0.1 | exFAT microSD in an adapter → two separate exFAT SSDs | Built-in SD reader; SSD A reports USB 10 Gb/s, SSD B USB 480 Mb/s | Four modes × pipelining off/on, plus Standard reuse; 134 selected files / 2.15 GB; ASC MHL and Saved Check for verified modes | **Pass:** nine scenarios, source inventory and content hashes unchanged; Quick/exclusion safety limits retained. Concurrent large workload makes timings unsuitable for mode comparisons. Not a power-loss or cable-pull test. | Use **Pass**, **Fail**, **Inconclusive**, or **Not tested**. A pass requires the scenario's expected behavior and independent hashes for all outputs reported as successful. Document failures and incomplete runs as well as passes. Use the [report template](HARDWARE_REPORT_TEMPLATE.md) and link the evidence when adding a row. diff --git a/docs/OVERNIGHT_VALIDATION_20261019.md b/docs/OVERNIGHT_VALIDATION_20261019.md index f7494f4e..2f080019 100644 --- a/docs/OVERNIGHT_VALIDATION_20261019.md +++ b/docs/OVERNIGHT_VALIDATION_20261019.md @@ -58,14 +58,14 @@ Before that run, the benchmark harness was corrected so each FileHandle read and ## App, engine and simulator gates -These are the latest completed full gates before the final benchmark-harness and high-resolution-probe additions. Logical test counts and parameterized case counts are kept separate. +The engine gates were rerun after the buffer-harness additions at `4106a48a`; the Mac and mobile rows precede the new high-resolution probe and final PDF layout follow-up. Logical test counts and parameterized case counts are kept separate. | Gate | Completed result | | --- | --- | -| Engine Debug, XCTest | 220 scheduled: 183 passed, 37 opt-in/platform skips, zero failures | -| Engine Release, XCTest | 220 scheduled: 183 passed, 37 opt-in/platform skips, zero failures | -| Engine Debug, Swift Testing | 219 tests in 39 suites passed | -| Engine Release, Swift Testing | 219 tests in 39 suites passed | +| Engine Debug, XCTest | 223 scheduled: 186 passed, 37 opt-in/platform skips, zero failures | +| Engine Release, XCTest | 223 scheduled: 186 passed, 37 opt-in/platform skips, zero failures | +| Engine Debug, Swift Testing | 219 declarations in 39 suites: 213 passed, six opt-in skips, zero failures | +| Engine Release, Swift Testing | 219 declarations in 39 suites: 213 passed, six opt-in skips, zero failures | | Full Mac app suite | 1,286 logical tests: 1,265 passed, 21 skipped, zero failures; no reported runtime warnings | | iPad simulator | 79 tests: 69 passed, ten skipped, zero failures; no reported runtime warnings | | iPhone simulator | 79 tests: 69 passed, ten skipped, zero failures; no reported runtime warnings | @@ -82,8 +82,8 @@ The diagnostics-export hardening had its own **32 passing XCTest cases**, zero s - **Large physical transfer:** the corrected 119.6 GB SSD A → SSD B case was still running at this checkpoint. Do not count it as a pass, infer its eventual verdict, or use an earlier flawed-harness attempt as a substitute. - **Application performance:** isolated repeated Standard/Thorough application trials and their counter-aligned measurements need a settled receipt. Raw buffer throughput is not application performance evidence. -- **4K report-preview memory:** the new host-only 20-path and 200-path probe has preliminary successful runs. Final reviewed receipts and the qualified result belong here before it is counted. Its fixture repeats one two-frame H.264 pattern across distinct paths; it does not cover codec diversity, long GOPs or distinct camera clips. -- **Intel:** a separate translated-x86/Intel compatibility receipt needs final review. Apple Silicon results and a universal build are not execution on a physical Intel Mac. +- **4K report-preview memory:** the [reviewed host-only probe](validation/report-preview-memory-2026-10-19.md) passed at 20 and 200 paths, each with three executed XCTest methods, zero skips/failures and no reported runtime warnings. Sampled process footprint peaked at 103 MiB. Its fixture repeats one two-frame H.264 pattern across distinct paths; it does not cover codec diversity, long GOPs, large encoded files or distinct camera clips. All preview pixels, source/destination hashes and PDF image placements were checked. The separate orphan-page layout follow-up remains pending. +- **Intel:** [translated x86_64 engine tests](validation/ROSETTA_ENGINE_20261019.md) passed: XCTest 186 passed/37 skipped, Swift Testing 213 declarations passed/six skipped, zero final failures. Initial unsigned-loader bookmark setup failures remain documented. This is Rosetta coverage, not execution on a physical Intel Mac or older macOS. - **Final revision and cleanup:** update this document after the remaining jobs settle, final combined gates complete, and owned media fixtures, images, caches and simulators are accounted for. Retain compact receipts; preserve uncertain or failed evidence rather than deleting it prematurely. No run here reproduces a real HDD through the reporter's dock, a USB controller fault, power loss, a forced unplug during active I/O, physical-media failure, or every iOS file-provider/background condition. The successful cases strengthen the tested paths; they do not close issues #10 or #23 without decisive field evidence. diff --git a/docs/TODO.md b/docs/TODO.md index 6194c4a6..3315f5fa 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -22,7 +22,7 @@ Unreleased work on `main`: terminal/History presentation, byte-weighted progress | 4 | Distinguish local-source, local-backup and SFTP assurance (upload guards merged; broader assurance audit open) | High | Low–medium | Low–medium | Quick | | 5 | Expand the MHL interoperability corpus (rename/move fixture merged; classic interop open) | High | Medium | Low | Scoped | | 6 | Reconcile missed Mac card events (merged; physical check open) | Medium–high | Low | Low | Quick | -| 7 | Eliminate the exFAT/FAT media-file publication race (implemented; FAT32 and physical checks open) | Very high | High | High | Field validation open | +| 7 | Eliminate the exFAT/FAT media-file publication race (implemented; FAT32/image and exFAT hardware results recorded) | Very high | High | High | Field validation open | | 8 | Mark failed readback output unmistakably on disk | High | High | High | After publication ownership | | 9 | Mac menu bar companion and icon (merged; signed-app walkthrough open) | High | Medium | Medium | Scoped | | 10 | Per-drive copy/readback speed (merged; signed-app walkthrough open) | Medium–high | Medium | Low–medium | Scoped | @@ -49,7 +49,7 @@ Unreleased work on `main`: terminal/History presentation, byte-weighted progress | 31 | Remote controls and outcomes | High | Very high | Very high | After remote Start | | 32 | Remote connectivity enhancements | Conditional | High | Medium | After core Remote works | | 33 | Shortcuts support | Medium | Medium–high | Medium–high | After command safety model | -| 34 | Slow-cable hint | Low–medium | Medium | Low | If detection is reliable | +| 34 | Slow-connection hint (implemented; release walkthrough recorded) | Low–medium | Medium | Low | Quick | | 35 | Time estimate before Start | Medium | Medium | Low–medium | After measured speeds | | 36 | Phone notification without an account | Medium | Medium | Medium | After privacy decision | | 37 | Persistent SFTP connection | Conditional | Medium | Medium | Demand | @@ -61,7 +61,7 @@ These eight supporting tasks are not competing features. Do them when their evid | Next gate | Supporting work | Usefulness | Difficulty | Blast radius | Shippability | | ---: | --- | --- | --- | --- | --- | -| 1 | Confirm the buffer benchmark's first read reaches physical storage (uncached-write request added) | High | Low | Low | Quick | +| 1 | Confirm the buffer benchmark's first read reaches the selected storage driver (measured; hardware cache still unknown) | High | Low | Low | Quick | | 2 | Run the end-to-end two-drive physical benchmark | High | Medium | Low | Hardware-dependent | | 3 | Keep automated tests free of drive-access prompts (Mac unit slice merged) | Medium–high | Medium | Low | Scoped | | 4 | Keep release walkthroughs and synthetic/physical results separate | High | Ongoing | Low | Every relevant release | @@ -86,9 +86,9 @@ The queue/recovery items marked complete below were merged in #20 for 0.2.8; see The priority buckets are not a strict implementation sequence: investigate P0 reports and audit assurance wording first, use physical benchmarks for default and scheduling decisions, and validate ASC MHL chains read-only before extending them. Source-read retry depends on fault classification and source identity; destination re-copy or repair additionally requires proof of output ownership. Neither retry path depends on MHL validation. -- [x] **Close the exFAT/FAT media-file publication race in code.** The no-hard-link FAT-family path now durably creates a visible incomplete-status warning before opening each media file at its final name with exclusive create. It never renames or removes that final name. An interrupted copy leaves its partial bytes and warning in place; existing files and another writer's bytes remain untouched. Exact marker validation, independent readback and named-file identity checks guard status advancement and the safe verdict. Deterministic replacement, cleanup, interruption and recovery regressions pass on the forced-direct path; mounted exFAT marker/root sync and full Standard-to-ASC-MHL handoff pass. See the [design and limits](EXFAT_PUBLICATION_DESIGN.md). **Remaining validation:** FAT32 images could not be mounted on this Mac, and no physical drive or sudden-power-loss result is recorded. Confirm FAT32 sync and transfer behavior on another host, then test representative physical drives without treating OS flush success as a power-loss guarantee. +- [x] **Close the exFAT/FAT media-file publication race in code.** The no-hard-link FAT-family path now durably creates a visible incomplete-status warning before opening each media file at its final name with exclusive create. It never renames or removes that final name. An interrupted copy leaves its partial bytes and warning in place; existing files and another writer's bytes remain untouched. Exact marker validation, independent readback and named-file identity checks guard status advancement and the safe verdict. Deterministic replacement, cleanup, interruption and recovery regressions pass on the forced-direct path; mounted exFAT marker/root sync and full Standard-to-ASC-MHL handoff pass. See the [design and limits](EXFAT_PUBLICATION_DESIGN.md). **Validation update:** the [October 19–20 matrix](OVERNIGHT_VALIDATION_20261019.md) exercised FAT32 images, including sync, handoff, full-volume and interrupted-publication behavior. An exFAT SD-to-two-SSD matrix also completed. These cover the recorded scenarios, not sudden power loss or every bridge and drive. Keep representative hardware and field checks open; an OS flush is not a power-loss guarantee. -- [x] **Close the ASC MHL exFAT empty-directory claim race in code.** Merged in #54. FAT-family publication claims `ascmhl` once and writes through its pinned descriptor; it never renames over the claim or deletes a mutable public name. A durable status remains incomplete until a completion record binds the exact initial manifest and chain bytes. Check refuses incomplete histories or altered witnessed bytes, even if a valid report sits beside them. After #57, it accepts supported external extensions only when the original completion witness remains valid and the current inventory chain passes its read-only checks; it never rewrites or reseals the status. Deterministic adversarial regressions and a mounted-exFAT handoff/official-tool extension pass. **Remaining validation:** FAT32 and physical-drive/power-loss behavior are untested. Directory/root hashes, rename lineage and historical authenticity remain outside the chain slice. These fixes do not establish the cause of issue #10. +- [x] **Close the ASC MHL exFAT empty-directory claim race in code.** Merged in #54. FAT-family publication claims `ascmhl` once and writes through its pinned descriptor; it never renames over the claim or deletes a mutable public name. A durable status remains incomplete until a completion record binds the exact initial manifest and chain bytes. Check refuses incomplete histories or altered witnessed bytes, even if a valid report sits beside them. After #57, it accepts supported external extensions only when the original completion witness remains valid and the current inventory chain passes its read-only checks; it never rewrites or reseals the status. Deterministic adversarial regressions and a mounted-exFAT handoff/official-tool extension pass. **Validation update:** FAT32 mounted-image and exFAT physical-drive handoffs passed in the [October 19–20 matrix](OVERNIGHT_VALIDATION_20261019.md). Power loss remains untested. Directory/root hashes, rename lineage and historical authenticity remain outside the chain slice. These fixes do not establish the cause of issue #10. - [ ] **Make failed readback output unmistakable on disk.** BitMatch publishes a copied file at its normal destination name before readback; the existing corruption regression confirms that the bad file remains there even though the app verdict correctly fails closed. Investigate marking or moving only an attempt-owned, identity-checked file that fails verification. A read error does not prove the bytes are corrupt, and even bad bytes may be worth preserving. **Acceptance:** injected mismatch, read error, external replacement, interrupted marking and marking failure never overwrite, delete or relabel another writer's file; the real on-disk path and uncertainty appear in History/reports and recovery; source media stays untouched. @@ -158,20 +158,20 @@ Remote scope excludes screen sharing, arbitrary filesystem browsing, remote shel ## P4 — Convenience after reliability -- [ ] **Portable settings/workflow export and import.** The first bounded slice moves verification, ASC MHL, the global AppleDouble preference, report/thumbnail switches and basic naming options between Mac, iPad and iPhone. It requires a changes preview, rejects unsupported or oversized files, and blocks Apply during active, queued or prepared work. It omits labels, paths, per-source exclusions, private report fields, credentials, bookmarks, history and identities. **Remaining:** explicit portable folder-recipe and reusable destination configuration designs; never serialize existing job/preset records or activate imported destinations automatically. Preview changes and revalidate destinations on the receiving machine. No facility configuration-management system. +- [ ] **Portable settings/workflow export and import.** The first bounded slice is implemented and locally validated on Mac, iPad and iPhone simulator: verification, ASC MHL, the global AppleDouble preference, report/thumbnail switches and basic naming options move through Settings as a small versioned JSON file. It requires a changes preview, rejects unsupported or oversized files, and blocks Apply during active, queued or prepared work. It omits labels, paths, per-source exclusions, private report fields, credentials, bookmarks, history and identities. **Remaining:** explicit portable folder-recipe and reusable destination configuration designs; never serialize existing job/preset records or activate imported destinations automatically. Preview changes and revalidate destinations on the receiving machine. No facility configuration-management system. - [ ] **Simplified operator mode, only if users ask.** Investigate preconfigured destinations, naming and verification with an everyday source → start → review flow. Warnings stay visible and validation still runs; no facility administration or permission system. - [ ] **Organize and rename.** Extend existing project naming and folder recipes with remaining naming-pattern and organize-by-date workflows. Preserve original card contents and test destination collisions before adding new options. - [ ] **Honest byte-weighted progress.** Implemented in #56. Integration passed full engine/Mac tests, the universal iOS build and iPhone/iPad simulator suites. The shared implementation budgets selected-file bytes for source reads, writes/reuse checks, required verification and optional ASC MHL work; live chunk callbacks advance large clips, and finishing reports never reset the bar or show active 100%. Destination bars use their own byte work. Queue cards use persisted size estimates, destination counts and verification mode; a queue with unknown sizes stays indeterminate. Estimates stay separate from copy evidence and remain stable per attempt. Synthetic regressions cover tiny-plus-large files, uneven destinations, reuse, cancellation/failure, MHL mapping and 10,000 small files. **Remaining:** signed-app/physical-drive walkthroughs and finer progress through extra source-hashing passes on reused files. The current retry budget remains deliberately logical rather than an exact tally of every physical read. Reserve 100% for the authoritative result. - [x] **Speed for each drive.** Expanded transfer details show each local destination's written and read-back bytes with separate live I/O rates. These are measured write/read calls, not a durability benchmark or verification verdict. The telemetry is shared with future storage-scheduling work; this change does not alter scheduling. - [x] **Reconcile missed Mac card events.** Mount notifications and manual rescan now reconcile the mounted filesystem's identity with published cards and drives. Deterministic event-gap tests cover unmount/reinsert, same-path replacement, stale asynchronous analysis and source selection; stale discovery cannot authorize auto-queue. This is a bounded event-triggered backstop, not a new polling service. A real removable-card walkthrough remains in the release gate below. -- [ ] **Slow-cable hint.** On Mac, show a quiet hint when reliable connection information identifies a USB 2 bottleneck. Do not promise a speed multiplier without measurements. +- [x] **Slow-connection hint.** Mac Advanced now shows a quiet note for a selected backup whose USB ancestry reports USB 2. Nested selected folders resolve to their volume. The note was checked with two actual connections; faster or unknown links stay quiet. It suggests checking the cable, hub or port without assigning the cause or promising a speed multiplier. No background survey of unselected drives or scheduling change. - [ ] **Time estimate before Start.** Use card size and recent measured speeds; keep “estimating” when evidence is insufficient. - [ ] **Phone notification with no account.** Optional completion/failure push through an open service such as ntfy. Make disclosure of card names/results to that service explicit. - [ ] **Offload from Finder.** Right-click a card and choose “Offload with BitMatch.” - [ ] **Shortcuts support.** App Intents for offloads, preserving the same validation and verdict rules. - [ ] **Cloud destinations, if users ask.** A selected backup destination, not a cloud dashboard. Define the provider's staging, exclusive publication, interruption recovery and independent readback semantics before choosing an API or claiming a verified result. Keep local source provenance, locally verified backup evidence and cloud-object evidence separate; upload acknowledgment or object metadata alone is not a readback. Do not assume that a cloud destination prevents a separate local source reread. - [ ] **Persistent SFTP connection.** Replace one `ssh` process per file if SFTP becomes a headline feature. -- [ ] **macOS 15.0 minimum.** Investigate lowering 15.5 to 15.0 without breaking APIs or behavior; do not go below 15. +- [ ] **macOS 15.0 minimum.** An October 19 build-only investigation passed an unsigned Release build for Apple Silicon and Intel with a 15.0 override; compiler targets, Mach-O minimums and the app's Info.plist all recorded 15.0, with availability checking enabled. The engine already targets 15.0 and Sparkle 2.10.0 requires 12.0. The shipped minimum remains 15.5. Before changing it, run a real 15.0 walkthrough and coordinate the project, docs and `Scripts/publish_appcast.sh` minimum (currently hardcoded to 15.5; derive it from the archive). Do not go below 15. - [ ] **Liquid Glass pass.** macOS/iOS 26 toolbar, composer, rows and completion, with a material fallback. Check it in person before merging. ## Validation, accessibility and release readiness @@ -188,7 +188,7 @@ These remain ongoing gates, not optional features. Use disposable data and retai ## Developer chores - [ ] **Tests without permission prompts.** Investigate drive-access prompts that stall Mac tests; prefer owned temporary fixtures and scoped test configuration without weakening release entitlements. The documented DerivedData-outside-Desktop workaround addresses a separate loader stall. A separate close-window test-host crash was fixed in #46; that fix does not settle the permission-prompt audit. -- [ ] **Buffer benchmark first run.** Source creation and timed reads now request `F_NOCACHE`, including the first run. That request does not prove physical I/O: confirm drive read counters or cold/warm behavior on the selected hardware before using the numbers for a buffer or verification-policy decision. +- [x] **Buffer benchmark first run.** Source creation and timed reads request `F_NOCACHE`. The [October 19–20 run](OVERNIGHT_VALIDATION_20261019.md) confirmed substantial read traffic in the selected OS storage driver during the first timed read. This does not prove a cold device cache or app-only attribution. Keep the current buffer and verification defaults until controlled end-to-end measurements justify a change. - [x] **Reverse flight on Edit.** Mac Edit carries the queued card back to setup, respecting Reduce Motion. Compact mobile Edit dismisses the queue sheet. Settings, operator and notes remain snapshotted. ## Out of scope From 84445c21b8340964136af6f02593d9c2924d65e4 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 23:37:30 -0400 Subject: [PATCH 34/56] fix: keep PDF identity in the page footer --- BitMatch-iPadTests/ReportPDFLayoutTests.swift | 81 ++++++++++++++++- BitMatchTests/ReportPDFLayoutTests.swift | 89 +++++++++++++++++-- Shared/Views/ReportPDFRenderer.swift | 79 +++++++++++----- Shared/Views/ReportView.swift | 28 +++++- .../report-pdf-footer-2026-10-19.md | 38 ++++++++ 5 files changed, 282 insertions(+), 33 deletions(-) create mode 100644 docs/validation/report-pdf-footer-2026-10-19.md diff --git a/BitMatch-iPadTests/ReportPDFLayoutTests.swift b/BitMatch-iPadTests/ReportPDFLayoutTests.swift index 5301c320..271f3bd9 100644 --- a/BitMatch-iPadTests/ReportPDFLayoutTests.swift +++ b/BitMatch-iPadTests/ReportPDFLayoutTests.swift @@ -1,5 +1,8 @@ import Foundation import PDFKit +import CoreGraphics +import ImageIO +import UniformTypeIdentifiers import Testing @testable import BitMatch_iPad import BitMatchEngine @@ -64,10 +67,8 @@ struct ReportPDFLayoutTests { issues: 0, workers: 1, appVersion: "test", osVersion: "test", client: "", production: "", company: "", verificationMethod: "Standard", totalBytesProcessed: 300_000, averageSpeed: 1, clientLogoData: nil, companyLogoData: nil, photographyJob: nil, safetyState: .safeToErase) - let blocks = ReportView(s: summary, rows: rows).pdfBlocks - let heights = ReportPDFRenderer.blockHeights(blocks) - let pages = ReportPDFLayout.pages(blockHeights: heights, contentHeight: ReportPDFRenderer.contentHeight, - firstPageContentHeight: ReportPDFRenderer.firstPageContentHeight) + let layout = ReportPDFRenderer.layout(ReportView(s: summary, rows: rows)) + let (blocks, pages) = (layout.blocks, layout.pages) let document = try #require(PDFDocument(data: ReportPDFRenderer.renderPDF(summary: summary, results: rows))) #expect(pages.count > 1) #expect(document.pageCount == pages.count) @@ -103,4 +104,76 @@ struct ReportPDFLayoutTests { #expect(sliced.map(\.offset) == [0, 150, 0]) } + @Test func previewReportsKeepIdentityWithContentWithoutAnOrphanPage() throws { + try checkPreviewFooter(clipCount: 20, longIdentity: false) + try checkPreviewFooter(clipCount: 200, longIdentity: false) + try checkPreviewFooter(clipCount: 20, longIdentity: true) + } + + @Test func veryLargeIdentityPaginatesWithoutDroppingText() throws { + try checkPreviewFooter(clipCount: 1, longIdentity: true, oversizedIdentity: true) + } + + private func checkPreviewFooter(clipCount: Int, longIdentity: Bool, oversizedIdentity: Bool = false) throws { + let source = "/private/tmp/bitmatch-4k-preview-12345678-1234-1234-1234-123456789012/Card" + let backup = "/private/tmp/bitmatch-4k-preview-12345678-1234-1234-1234-123456789012/Backup" + let rows = (0..= layout.bottomMargin) + #expect(match.bounds(for: page).maxY <= ReportPDFRenderer.pageHeight - ReportPDFRenderer.verticalMargin) + } + for pageIndex in 0..= ReportPDFRenderer.horizontalMargin - 1) + #expect(bounds.maxX <= ReportPDFRenderer.pageWidth - ReportPDFRenderer.horizontalMargin + 1) + #expect(bounds.minY >= 13) + #expect(bounds.maxY <= layout.bottomMargin - 5) + } + } + if !longIdentity { #expect(document.pageCount == (clipCount == 20 ? 2 : 14)) } + if let directory = ProcessInfo.processInfo.environment["BITMATCH_PREVIEW_PDF_OUTPUT"] { + let folder = URL(fileURLWithPath: directory) + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + try pdf.write(to: folder.appendingPathComponent("footer-\(clipCount)-\(longIdentity ? "long" : "normal").pdf")) + } + } + } diff --git a/BitMatchTests/ReportPDFLayoutTests.swift b/BitMatchTests/ReportPDFLayoutTests.swift index c261b9e6..771314a0 100644 --- a/BitMatchTests/ReportPDFLayoutTests.swift +++ b/BitMatchTests/ReportPDFLayoutTests.swift @@ -1,5 +1,8 @@ import Foundation import PDFKit +import CoreGraphics +import ImageIO +import UniformTypeIdentifiers import Testing @testable import BitMatch import BitMatchEngine @@ -167,10 +170,8 @@ struct ReportPDFLayoutTests { issues: 0, workers: 1, appVersion: "test", osVersion: "test", client: "", production: "", company: "", verificationMethod: "Standard", totalBytesProcessed: 300_000, averageSpeed: 1, clientLogoData: nil, companyLogoData: nil, photographyJob: nil, safetyState: .safeToErase) - let blocks = ReportView(s: summary, rows: rows).pdfBlocks - let heights = ReportPDFRenderer.blockHeights(blocks) - let pages = ReportPDFLayout.pages(blockHeights: heights, contentHeight: ReportPDFRenderer.contentHeight, - firstPageContentHeight: ReportPDFRenderer.firstPageContentHeight) + let layout = ReportPDFRenderer.layout(ReportView(s: summary, rows: rows)) + let (blocks, pages) = (layout.blocks, layout.pages) let document = try #require(PDFDocument(data: ReportPDFRenderer.renderPDF(summary: summary, results: rows))) #expect(pages.count > 1) #expect(document.pageCount == pages.count) @@ -215,7 +216,13 @@ struct ReportPDFLayoutTests { size: 10, checksum: "fixture", destination: backup.lastPathComponent, destinationPath: backup.appendingPathComponent(name).path) } - let document = try #require(PDFDocument(data: await ReportExporter.historyPDF(record: record, includeThumbnails: false))) + let pdf = try await ReportExporter.historyPDF(record: record, includeThumbnails: false) + let document = try #require(PDFDocument(data: pdf)) + if let directory = ProcessInfo.processInfo.environment["BITMATCH_PREVIEW_PDF_OUTPUT"] { + let folder = URL(fileURLWithPath: directory) + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + try pdf.write(to: folder.appendingPathComponent("cancelled-seven-rows.pdf")) + } #expect(document.pageCount == 1) let page = try #require(document.page(at: 0)) #expect(page.string?.contains("NOT SAFE TO ERASE") == true) @@ -232,4 +239,76 @@ struct ReportPDFLayoutTests { #expect(sliced.map(\.offset) == [0, 150, 0]) } + @Test func previewReportsKeepIdentityWithContentWithoutAnOrphanPage() throws { + try checkPreviewFooter(clipCount: 20, longIdentity: false) + try checkPreviewFooter(clipCount: 200, longIdentity: false) + try checkPreviewFooter(clipCount: 20, longIdentity: true) + } + + @Test func veryLargeIdentityPaginatesWithoutDroppingText() throws { + try checkPreviewFooter(clipCount: 1, longIdentity: true, oversizedIdentity: true) + } + + private func checkPreviewFooter(clipCount: Int, longIdentity: Bool, oversizedIdentity: Bool = false) throws { + let source = "/private/tmp/bitmatch-4k-preview-12345678-1234-1234-1234-123456789012/Card" + let backup = "/private/tmp/bitmatch-4k-preview-12345678-1234-1234-1234-123456789012/Backup" + let rows = (0..= layout.bottomMargin) + #expect(match.bounds(for: page).maxY <= ReportPDFRenderer.pageHeight - ReportPDFRenderer.verticalMargin) + } + for pageIndex in 0..= ReportPDFRenderer.horizontalMargin - 1) + #expect(bounds.maxX <= ReportPDFRenderer.pageWidth - ReportPDFRenderer.horizontalMargin + 1) + #expect(bounds.minY >= 13) + #expect(bounds.maxY <= layout.bottomMargin - 5) + } + } + if !longIdentity { #expect(document.pageCount == (clipCount == 20 ? 2 : 14)) } + if let directory = ProcessInfo.processInfo.environment["BITMATCH_PREVIEW_PDF_OUTPUT"] { + let folder = URL(fileURLWithPath: directory) + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + try pdf.write(to: folder.appendingPathComponent("footer-\(clipCount)-\(longIdentity ? "long" : "normal").pdf")) + } + } + } diff --git a/Shared/Views/ReportPDFRenderer.swift b/Shared/Views/ReportPDFRenderer.swift index 87348176..bacffbf0 100644 --- a/Shared/Views/ReportPDFRenderer.swift +++ b/Shared/Views/ReportPDFRenderer.swift @@ -17,30 +17,56 @@ enum ReportPDFRenderer { static let firstPageContentHeight = pageHeight - 2 * verticalMargin static let contentHeight = pageHeight - 2 * verticalMargin - continuationHeight + struct Layout { + let blocks: [ReportPDFBlock] + let heights: [CGFloat] + let pages: [ReportPDFLayout.Page] + let bottomMargin: CGFloat + let repeatsIdentity: Bool + } + @MainActor - static func blockHeights(_ blocks: [ReportPDFBlock]) -> [CGFloat] { - blocks.map { block in - autoreleasepool { - let renderer = ImageRenderer(content: block.view - .frame(width: pageWidth - 2 * horizontalMargin, alignment: .leading) - .fixedSize(horizontal: false, vertical: true) - .environment(\.colorScheme, .light)) - renderer.proposedSize = ProposedViewSize(width: pageWidth - 2 * horizontalMargin, height: nil) - var height: CGFloat = 0 - // Layout without allocating a bitmap of the block. - renderer.render { size, _ in height = ceil(size.height) } - return height - } + private static func height(of view: AnyView) -> CGFloat { + autoreleasepool { + let renderer = ImageRenderer(content: view + .frame(width: pageWidth - 2 * horizontalMargin, alignment: .leading) + .fixedSize(horizontal: false, vertical: true) + .environment(\.colorScheme, .light)) + renderer.proposedSize = ProposedViewSize(width: pageWidth - 2 * horizontalMargin, height: nil) + var height: CGFloat = 0 + renderer.render { size, _ in height = ceil(size.height) } + return height } } + @MainActor + static func blockHeights(_ blocks: [ReportPDFBlock]) -> [CGFloat] { + blocks.map { height(of: $0.view) } + } + + @MainActor + static func layout(_ report: ReportView) -> Layout { + // Reserve enough width for any page counter before measuring wrapping. + // Actual counters are shorter, so their footer cannot need more height. + let identityHeight = height(of: AnyView(report.pdfPageFooter(page: Int.max, total: Int.max))) + let repeatsIdentity = identityHeight <= contentHeight / 3 + // Very large metadata must paginate rather than consume the page or clip. + let blocks = report.pdfBlocks + (repeatsIdentity ? [] : [report.pdfPaginatedIdentity]) + let heights = blockHeights(blocks) + let bottomMargin = repeatsIdentity ? max(verticalMargin, identityHeight + 20) : verticalMargin + let firstCapacity = pageHeight - verticalMargin - bottomMargin + let pages = ReportPDFLayout.pages(blockHeights: heights, contentHeight: firstCapacity - continuationHeight, + firstPageContentHeight: firstCapacity) + return Layout(blocks: blocks, heights: heights, pages: pages, bottomMargin: bottomMargin, + repeatsIdentity: repeatsIdentity) + } + @MainActor static func renderPDF(summary: ReportSummary, results: [ResultRow], thumbnails: ReportThumbnails = ReportThumbnails()) -> Data { let report = ReportView(s: summary, rows: results, thumbnails: thumbnails) - let blocks = report.pdfBlocks - let heights = blockHeights(blocks) - let pages = ReportPDFLayout.pages(blockHeights: heights, contentHeight: contentHeight, - firstPageContentHeight: firstPageContentHeight) + let layout = layout(report) + let (blocks, heights, pages) = (layout.blocks, layout.heights, layout.pages) + let firstCapacity = pageHeight - verticalMargin - layout.bottomMargin let pdfMetadata = [ kCGPDFContextCreator: "BitMatch", kCGPDFContextTitle: "BitMatch Verification Report" @@ -73,18 +99,25 @@ enum ReportPDFRenderer { .fixedSize(horizontal: false, vertical: true) .offset(y: -page.offset) .frame(width: pageWidth - 2 * horizontalMargin, - height: pageIndex == 0 ? firstPageContentHeight : contentHeight, alignment: .topLeading) + height: pageIndex == 0 ? firstCapacity : firstCapacity - continuationHeight, alignment: .topLeading) .clipped() Spacer(minLength: 0) } .padding(.horizontal, horizontalMargin) - .padding(.vertical, verticalMargin) + .padding(.top, verticalMargin) + .padding(.bottom, layout.bottomMargin) .frame(width: pageWidth, height: pageHeight, alignment: .topLeading) .overlay(alignment: .bottom) { - Text("Page \(pageIndex + 1) of \(pages.count)") - .font(.system(size: 9)) - .foregroundColor(.secondary) - .padding(.bottom, 14) + Group { + if layout.repeatsIdentity { + report.pdfPageFooter(page: pageIndex + 1, total: pages.count) + } else { + Text("Page \(pageIndex + 1) of \(pages.count)") + .font(.system(size: 9)).foregroundColor(.secondary) + } + } + .frame(width: pageWidth - 2 * horizontalMargin) + .padding(.bottom, 14) } .background(Color.white) .environment(\.colorScheme, .light) diff --git a/Shared/Views/ReportView.swift b/Shared/Views/ReportView.swift index 57fcc87d..fc308f85 100644 --- a/Shared/Views/ReportView.swift +++ b/Shared/Views/ReportView.swift @@ -277,10 +277,36 @@ struct ReportView: View { } } } - append(footerSection) return blocks } + /// PDF identity lives outside the manifest so it cannot create an empty last page. + func pdfPageFooter(page: Int, total: Int) -> some View { + HStack(alignment: .bottom, spacing: 12) { + pdfIdentity + .frame(maxWidth: .infinity, alignment: .leading) + Text("Page \(page) of \(total)").font(.system(size: 9)).monospacedDigit().fixedSize() + }.foregroundColor(.secondary) + } + + private var pdfIdentity: some View { + VStack(alignment: .leading, spacing: 2) { + Text("Job ID: \(s.jobID.uuidString)").font(.system(size: 8, design: .monospaced)) + if !s.appVersion.isEmpty { + Text("BitMatch v\(s.appVersion) • \(s.osVersion)").font(.system(size: 8)) + } + if !s.company.isEmpty { Text(s.company).font(.system(size: 8)) } + }.fixedSize(horizontal: false, vertical: true) + } + + /// Unusually large identity text remains ordinary paginated content. + var pdfPaginatedIdentity: ReportPDFBlock { + ReportPDFBlock(view: AnyView(VStack(alignment: .leading, spacing: 0) { + Divider().padding(.vertical, 12) + pdfIdentity.foregroundColor(.secondary) + }), continuesManifest: false) + } + private var pdfLocationsSection: some View { VStack(alignment: .leading, spacing: 5) { Text("Locations").font(.system(size: 12, weight: .semibold)) diff --git a/docs/validation/report-pdf-footer-2026-10-19.md b/docs/validation/report-pdf-footer-2026-10-19.md new file mode 100644 index 00000000..5a3cdb17 --- /dev/null +++ b/docs/validation/report-pdf-footer-2026-10-19.md @@ -0,0 +1,38 @@ +# PDF identity pagination regression + +The synthetic 4K preview probe exposed a report ending with a page containing +only Job ID/version text. A new normal regression reproduced this before the +fix: 13 tests executed, one failed. The PDF renderer had treated identity as a +final content block after the manifest. + +PDF identity now shares the page footer with the page number. The renderer +measures wrapped Job ID, version, OS and company text and reserves its space +before packing content. The screen footer is unchanged. Unusually large identity +text falls back to ordinary pagination, preserving the full text rather than +clipping it or consuming most of every page. + +Local Mac validation: + +- Of 34 focused tests discovered, 33 passed and one existing codec-fixture + opt-in was skipped: 14 layout, 16 thumbnail and three Master Report passes. + No new test was skipped; there were zero failures or xcresult runtime warnings. +- Normal regressions rendered 20 and 200 preview reports, long company/version + text, and oversized metadata. Assertions retain every filename and identity + token, check footer/text bounds and preserve the verification verdict. +- The cancelled seven-file History report still fits one page and retains its + unsafe verdict. Existing long Master Report names still wrap within the page. +- Real Standard copy/readback plus 4K preview collection was rerun at 20 and 200 + paths. Both runs passed all three probe tests; source/destination SHA-256 + snapshots matched. Independent Poppler inspection counted 20/200 image + placements across 2/14 pages, down from 3/15. Every page contains file rows + and identity; the empty final page is gone. +- Sampled process peaks in those reruns were 102/103 MiB, decoder peaks 79/82 MiB + and PDF peaks 82/98 MiB. Production preview limits stayed unchanged. These are + repeated-path synthetic H.264 probes, not proof of issue #23's cause or of + bounded memory for every codec. See the separate memory-probe note. + +All generated PDFs were rendered privately. Representative final pages, +wrapped metadata, the oversized-text fallback, cancelled report and long Master +Report were visually inspected. This is headless PDF evidence, not interactive +app or hardware validation. Shared iOS build and PDF regression gates remain +part of integration validation. From 77061e17442333489126880540658771ffecabb7 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 23:44:29 -0400 Subject: [PATCH 35/56] Record integrated PDF and mobile regression results --- CHANGELOG.md | 2 +- docs/OVERNIGHT_VALIDATION_20261019.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b5b4615..c686253f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All notable changes to this project will be documented in this file. - Check no longer reports preserved Finder metadata as missing when it is listed in the saved checksums. - Corrected physical-drive checks for selected subfolders, so two folders on one drive count as one backup. -- Cleaned up long names and page breaks in PDFs; metadata companions no longer inflate the clip-preview count. +- Cleaned up long names and page breaks in PDFs; metadata companions no longer inflate the clip-preview count. Job details now share the page footer instead of creating a nearly empty last page. - Mac: Advanced points out a selected backup using a reported USB 2 connection. - Diagnostics exports reject oversized or unreadable local records, keep the usable events, and say when the record is incomplete. - Destination progress no longer resets to Waiting while ASC MHL and transfer reports are being saved. Processed counts and measured progress stay visible without implying the transfer is complete. diff --git a/docs/OVERNIGHT_VALIDATION_20261019.md b/docs/OVERNIGHT_VALIDATION_20261019.md index 2f080019..dc45ec4a 100644 --- a/docs/OVERNIGHT_VALIDATION_20261019.md +++ b/docs/OVERNIGHT_VALIDATION_20261019.md @@ -58,7 +58,7 @@ Before that run, the benchmark harness was corrected so each FileHandle read and ## App, engine and simulator gates -The engine gates were rerun after the buffer-harness additions at `4106a48a`; the Mac and mobile rows precede the new high-resolution probe and final PDF layout follow-up. Logical test counts and parameterized case counts are kept separate. +The engine gates were rerun after the buffer-harness additions at `4106a48a`; the full Mac, iPad and iPhone suites passed again at `84445c21`, including the high-resolution probe guards and final PDF footer regressions. Logical test counts and parameterized case counts are kept separate. | Gate | Completed result | | --- | --- | @@ -66,9 +66,9 @@ The engine gates were rerun after the buffer-harness additions at `4106a48a`; th | Engine Release, XCTest | 223 scheduled: 186 passed, 37 opt-in/platform skips, zero failures | | Engine Debug, Swift Testing | 219 declarations in 39 suites: 213 passed, six opt-in skips, zero failures | | Engine Release, Swift Testing | 219 declarations in 39 suites: 213 passed, six opt-in skips, zero failures | -| Full Mac app suite | 1,286 logical tests: 1,265 passed, 21 skipped, zero failures; no reported runtime warnings | -| iPad simulator | 79 tests: 69 passed, ten skipped, zero failures; no reported runtime warnings | -| iPhone simulator | 79 tests: 69 passed, ten skipped, zero failures; no reported runtime warnings | +| Full Mac app suite | 1,291 logical tests: 1,269 passed, 22 skipped, zero failures; no reported runtime warnings | +| iPad simulator | 81 tests: 71 passed, ten skipped, zero failures; no reported runtime warnings | +| iPhone simulator | 81 tests: 71 passed, ten skipped, zero failures; no reported runtime warnings | | Shared mobile build | Universal iOS simulator build passed | | Local signed Mac archive | Built and signature verified; no release published | @@ -82,7 +82,7 @@ The diagnostics-export hardening had its own **32 passing XCTest cases**, zero s - **Large physical transfer:** the corrected 119.6 GB SSD A → SSD B case was still running at this checkpoint. Do not count it as a pass, infer its eventual verdict, or use an earlier flawed-harness attempt as a substitute. - **Application performance:** isolated repeated Standard/Thorough application trials and their counter-aligned measurements need a settled receipt. Raw buffer throughput is not application performance evidence. -- **4K report-preview memory:** the [reviewed host-only probe](validation/report-preview-memory-2026-10-19.md) passed at 20 and 200 paths, each with three executed XCTest methods, zero skips/failures and no reported runtime warnings. Sampled process footprint peaked at 103 MiB. Its fixture repeats one two-frame H.264 pattern across distinct paths; it does not cover codec diversity, long GOPs, large encoded files or distinct camera clips. All preview pixels, source/destination hashes and PDF image placements were checked. The separate orphan-page layout follow-up remains pending. +- **4K report-preview memory:** the [reviewed host-only probe](validation/report-preview-memory-2026-10-19.md) passed at 20 and 200 paths, each with three executed XCTest methods, zero skips/failures and no reported runtime warnings. Sampled process footprint peaked at 103 MiB. Its fixture repeats one two-frame H.264 pattern across distinct paths; it does not cover codec diversity, long GOPs, large encoded files or distinct camera clips. All preview pixels, source/destination hashes and PDF image placements were checked. The [separate footer fix](validation/report-pdf-footer-2026-10-19.md) passed its regressions and the integrated app suites: normal 20/200-clip reports retain all previews across 2/14 pages instead of 3/15. Oversized identity text remains fully paginated rather than truncated. - **Intel:** [translated x86_64 engine tests](validation/ROSETTA_ENGINE_20261019.md) passed: XCTest 186 passed/37 skipped, Swift Testing 213 declarations passed/six skipped, zero final failures. Initial unsigned-loader bookmark setup failures remain documented. This is Rosetta coverage, not execution on a physical Intel Mac or older macOS. - **Final revision and cleanup:** update this document after the remaining jobs settle, final combined gates complete, and owned media fixtures, images, caches and simulators are accounted for. Retain compact receipts; preserve uncertain or failed evidence rather than deleting it prematurely. From 5fb59a649d24e011921f751d4ed78bde5429e702 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Mon, 19 Oct 2026 23:50:24 -0400 Subject: [PATCH 36/56] Test transfer recovery after worker process death --- .../ProcessDeathRecoveryTests.swift | 547 ++++++++++++++++++ .../PROCESS_DEATH_RECOVERY_20261019.md | 69 +++ 2 files changed, 616 insertions(+) create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/ProcessDeathRecoveryTests.swift create mode 100644 docs/validation/PROCESS_DEATH_RECOVERY_20261019.md diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ProcessDeathRecoveryTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ProcessDeathRecoveryTests.swift new file mode 100644 index 00000000..f5333bc4 --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ProcessDeathRecoveryTests.swift @@ -0,0 +1,547 @@ +#if os(macOS) +import CryptoKit +import Darwin +import Foundation +import XCTest +@testable import BitMatchEngine + +/// Genuine process death at deterministic engine boundaries. This is not a +/// power-loss, USB-disconnect, macOS OOM or reporter-reproduction test. +/// Opt in: BITMATCH_PROCESS_DEATH_MATRIX=smoke (4 cases), or full (20 cases). +/// Use XCTest explicitly: swift test --disable-swift-testing --jobs 4 +/// --filter ProcessDeathRecoveryTests. Ordinary discovery creates no images. +final class ProcessDeathRecoveryTests: XCTestCase, @unchecked Sendable { + func testKilledWorkerRecoversAsInterrupted() async throws { + guard let selection = ProcessInfo.processInfo.environment["BITMATCH_PROCESS_DEATH_MATRIX"], + ["smoke", "full"].contains(selection), + ProcessInfo.processInfo.environment["BITMATCH_PROCESS_DEATH_WORKER"] == nil else { + throw XCTSkip("Opt in with BITMATCH_PROCESS_DEATH_MATRIX=smoke or full") + } + try await FileOperationsTestLock.shared.run { + let harness = try ProcessDeathHarness() + do { + try await harness.run(smoke: selection == "smoke", bundle: Bundle(for: Self.self).bundleURL) + try harness.cleanup(success: true) + } catch { + // Detach only owned images, preserving failure bytes and logs. + do { try harness.cleanup(success: false) } + catch { XCTFail("Owned image cleanup failed: \(error)") } + throw error + } + } + } + + /// The parent selects only this method in a fresh xctest process. Never + /// fork a Swift/Foundation process, and never run an arbitrary executable. + func testProcessDeathWorker() async throws { + guard let path = ProcessInfo.processInfo.environment["BITMATCH_PROCESS_DEATH_WORKER"] else { + throw XCTSkip("Internal explicitly selected disposable worker") + } + let control = try ProcessDeathControl.read(path: path) + try control.validate(parentPID: getppid()) + let journal = TransferJournal(fileURL: control.journal) + try deathRequire(journal.persistenceError == nil, "Worker could not open journal") + let id = try journal.enqueue(sourceURL: control.source, destinationURLs: [control.destination], + verificationMode: .standard, cameraSettings: .init(), reportSettings: .init(), generateASCMHL: true) + try journal.markRunning(id: id) + try journal.recordPlan(id: id, fileCounts: [1], selectedSourceBytes: ProcessDeathHarness.payloadSize) + let pause: @Sendable () throws -> Void = { + try control.stopAtBoundary(recordID: id) + } + let hooks = DestinationWriter.FanOutHooks(beforeWrite: { _, _, chunk in + if control.phase == .copyMidChunk && chunk == 1 { try pause() } + }) + let operation = try await TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared, + fanOutHooks: hooks).performFileOperation(sourceURL: control.source, + destinationURLs: [control.destination], verificationMode: .standard, settings: .init(), + progressCallback: { _ in }, onFileResult: nil) + try deathRequire(operation.results.count == 1 && operation.results.allSatisfy { + $0.success && $0.verificationResult?.isValid == true + }, "Worker did not complete actual copy/readback") + let plan = TransferCompletion.ascmhlPlan(results: operation.results, sourceFiles: operation.sourceManifest, + destinations: [control.destination], source: control.source, settings: .init()) + try deathRequire(plan.jobs.count == 1 && plan.issues.isEmpty, "Worker has no authoritative MHL job") + if control.phase == .pipelineDrained { try pause() } + var readHooks = ASCMHLGenerator.ReadHooks() + switch control.phase { + case .mhlClaimed: readHooks.didClaimHistory = pause + case .mhlManifestWritten: + readHooks.didWritePublicHistoryFile = { name in + if name.hasSuffix(".mhl") { try pause() } + } + case .mhlBeforeComplete: readHooks.beforePublishingCompletion = pause + case .mhlPrivateStaged: readHooks.beforeExclusiveRename = pause + default: break + } + _ = try ASCMHLGenerator.generateInitialHistory(destinationURL: plan.jobs[0].root, + files: plan.jobs[0].files, startTime: operation.startTime, sourceURL: control.source, + toolVersion: "process-death-test", readHooks: readHooks) + if control.phase == .mhlPublished { try pause() } + throw deathFailure("Worker passed its requested boundary without being killed") + } + + func testWorkerRejectsAnUnownedRoot() throws { + let root = URL(fileURLWithPath: "/private/tmp/bitmatch-process-death-\(UUID())") + let control = ProcessDeathControl(root: root, nonce: UUID(), parentPID: getpid(), + rootDevice: 0, rootInode: 0, mountDevice: 0, phase: .copyMidChunk, format: "apfs") + XCTAssertThrowsError(try control.validate(parentPID: getpid())) + } +} + +private enum ProcessDeathPhase: String, Codable, Sendable { + case copyMidChunk, pipelineDrained, mhlPrivateStaged, mhlClaimed, mhlManifestWritten, mhlBeforeComplete, mhlPublished +} + +private struct ProcessDeathControl: Codable, Sendable { + let root: URL + let nonce: UUID + let parentPID: Int32 + let rootDevice: Int32 + let rootInode: UInt64 + let mountDevice: Int32 + let phase: ProcessDeathPhase + let format: String + var source: URL { root.appendingPathComponent("Card") } + var destination: URL { root.appendingPathComponent("mount/\(phase.rawValue)/Backup") } + var journal: URL { root.appendingPathComponent("\(phase.rawValue)-journal.json") } + var breadcrumb: URL { root.appendingPathComponent("\(phase.rawValue)-boundary.json") } + var ownership: URL { root.appendingPathComponent("ownership.json") } + + static func read(path: String) throws -> Self { + let url = URL(fileURLWithPath: path) + try deathRequire(url.lastPathComponent == "control.json" && url.path.hasPrefix("/private/tmp/bitmatch-process-death-"), + "Invalid child control path") + let fd = Darwin.open(url.path, O_RDONLY | O_NOFOLLOW | O_NONBLOCK | O_CLOEXEC) + guard fd >= 0 else { throw deathFailure("Cannot open child control") } + let handle = FileHandle(fileDescriptor: fd, closeOnDealloc: true) + defer { try? handle.close() } + var info = stat() + try deathRequire(fstat(fd, &info) == 0 && info.st_mode & S_IFMT == S_IFREG && info.st_size < 8192, + "Invalid child control file") + let bytes = try handle.read(upToCount: 8193) ?? Data() + try deathRequire(bytes.count <= 8192, "Oversized child control") + return try JSONDecoder().decode(Self.self, from: bytes) + } + + func validate(parentPID actualParent: Int32) throws { + try deathRequire(root.path == "/private/tmp/bitmatch-process-death-\(nonce.uuidString)" && + parentPID == actualParent && parentPID > 1, + "Worker ownership or parent mismatch: root=\(root.path), nonce=\(nonce), expectedParent=\(parentPID), actualParent=\(actualParent)") + var rootInfo = stat(), mountInfo = stat() + try deathRequire(lstat(root.path, &rootInfo) == 0 && rootInfo.st_mode & S_IFMT == S_IFDIR && + rootInfo.st_dev == rootDevice && rootInfo.st_ino == rootInode, + "Worker root identity mismatch") + let bytes = try Data(contentsOf: ownership) + try deathRequire(bytes == Data(nonce.uuidString.utf8), "Worker nonce marker mismatch") + try deathRequire(lstat(root.appendingPathComponent("mount").path, &mountInfo) == 0 && + mountInfo.st_mode & S_IFMT == S_IFDIR && mountInfo.st_dev == mountDevice && mountDevice != rootDevice, + "Worker needs the exact disposable mounted image") + var facts = statfs() + try deathRequire(statfs(root.appendingPathComponent("mount").path, &facts) == 0, "Missing worker filesystem") + let actual = withUnsafeBytes(of: facts.f_fstypename) { String(decoding: $0.prefix { $0 != 0 }, as: UTF8.self) } + try deathRequire(actual == format, "Worker filesystem mismatch") + } + + func stopAtBoundary(recordID: UUID) throws { + try validate(parentPID: getppid()) + let disk = try JSONDecoder().decode([LocalTransferRecord].self, from: Data(contentsOf: journal)) + try deathRequire(disk.count == 1 && disk[0].id == recordID && disk[0].state == .running, + "Journal was not persisted as running before death") + let record = ProcessDeathBreadcrumb(nonce: nonce, phase: phase, pid: getpid(), recordID: recordID) + let fd = Darwin.open(breadcrumb.path, O_CREAT | O_EXCL | O_NOFOLLOW | O_WRONLY | O_CLOEXEC, 0o600) + guard fd >= 0 else { throw deathFailure("Boundary breadcrumb already exists or cannot be created") } + let handle = FileHandle(fileDescriptor: fd, closeOnDealloc: true) + defer { try? handle.close() } + try handle.write(contentsOf: JSONEncoder().encode(record)) + try handle.synchronize() + // Kernel-stopped child cannot race onward into completion while the + // parent validates its breadcrumb and sends this exact process SIGKILL. + try deathRequire(raise(SIGSTOP) == 0, "Could not stop owned worker") + throw deathFailure("Stopped worker was unexpectedly resumed") + } +} + +private struct ProcessDeathBreadcrumb: Codable { + let nonce: UUID + let phase: ProcessDeathPhase + let pid: Int32 + let recordID: UUID +} + +private final class ProcessDeathHarness { + static let payloadSize: Int64 = 12 * 1024 * 1024 + 137 + private let nonce = UUID() + private let fm = FileManager.default + private let started = ProcessInfo.processInfo.systemUptime + private let root: URL + private var mounted = false + private var worker: Process? + private var receipts: [[String: String]] = [] + private let sentinel = Data("unrelated existing destination bytes".utf8) + private var sourceBefore: [String: String] = [:] + + init() throws { + root = URL(fileURLWithPath: "/private/tmp/bitmatch-process-death-\(nonce.uuidString)", isDirectory: true) + try fm.createDirectory(at: root, withIntermediateDirectories: false, attributes: [.posixPermissions: 0o700]) + try Data(nonce.uuidString.utf8).write(to: root.appendingPathComponent("ownership.json"), options: .withoutOverwriting) + let source = root.appendingPathComponent("Card") + try fm.createDirectory(at: source, withIntermediateDirectories: false) + try Data(repeating: 0xA7, count: Int(Self.payloadSize)).write(to: source.appendingPathComponent("clip.bin"), options: .withoutOverwriting) + sourceBefore = try snapshot(source) + print("PROCESS_DEATH owned_root=\(root.path)") + } + + func run(smoke: Bool, bundle: URL) async throws { + try deathRequire(bundle.pathExtension == "xctest" && fm.fileExists(atPath: bundle.path), "No explicit XCTest bundle") + let hostLog = try command("/usr/bin/xcrun", ["--find", "xctest"]) + let host = try String(contentsOf: hostLog, encoding: .utf8).trimmingCharacters(in: .whitespacesAndNewlines) + try deathRequire(host.hasPrefix("/Applications/") && host.hasSuffix("/usr/bin/xctest"), "Unexpected XCTest host") + let formats = smoke ? [("APFS", "apfs"), ("ExFAT", "exfat")] : + [("APFS", "apfs"), ("Journaled HFS+", "hfs"), ("ExFAT", "exfat"), ("MS-DOS FAT32", "msdos")] + for (index, format) in formats.enumerated() { + let mount = root.appendingPathComponent("mount") + try fm.createDirectory(at: mount, withIntermediateDirectories: false) + let image = root.appendingPathComponent("image-\(index).sparseimage") + _ = try command("/usr/bin/hdiutil", ["create", "-quiet", "-size", "3g", "-type", "SPARSE", "-fs", format.0, + "-volname", "BMDEATH\(index)", "-o", image.path]) + mounted = true // Track even a partially successful attach. + _ = try command("/usr/bin/hdiutil", ["attach", "-quiet", "-nobrowse", "-mountpoint", mount.path, image.path]) + let fat = ["exfat", "msdos"].contains(format.1) + let phases: [ProcessDeathPhase] = smoke ? [.copyMidChunk, fat ? .mhlBeforeComplete : .mhlPrivateStaged] : + [.copyMidChunk, .pipelineDrained] + (fat ? [.mhlClaimed, .mhlManifestWritten, .mhlBeforeComplete] : [.mhlPrivateStaged]) + [.mhlPublished] + for phase in phases { + try await scenario(phase, format: format.1, host: host, bundle: bundle) + } + try detach() + try fm.removeItem(at: mount) + } + try deathRequire(receipts.count == (smoke ? 4 : 20), "Wrong exact death-scenario count") + let receipt: [String: Any] = ["scenarios": receipts, "count": receipts.count, + "elapsedSeconds": ProcessInfo.processInfo.systemUptime - started, + "claim": "SIGKILL process death only; not power loss, OOM or physical-drive evidence"] + try JSONSerialization.data(withJSONObject: receipt, options: [.prettyPrinted, .sortedKeys]) + .write(to: root.appendingPathComponent("receipt.json"), options: .withoutOverwriting) + print("PROCESS_DEATH COMPLETE count=\(receipts.count) receipt=\(root.appendingPathComponent("receipt.json").path)") + } + + private func scenario(_ phase: ProcessDeathPhase, format: String, host: String, bundle: URL) async throws { + var rootInfo = stat(), mountInfo = stat() + try deathRequire(lstat(root.path, &rootInfo) == 0 && lstat(root.appendingPathComponent("mount").path, &mountInfo) == 0, + "Cannot pin owned fixture identities") + // Names remain unique across formats because per-phase host evidence is retained. + let caseRoot = root.appendingPathComponent("mount/\(phase.rawValue)") + try fm.createDirectory(at: caseRoot.appendingPathComponent("Backup"), withIntermediateDirectories: true) + try sentinel.write(to: caseRoot.appendingPathComponent("unrelated.txt"), options: .withoutOverwriting) + let control = ProcessDeathControl(root: root, nonce: nonce, parentPID: getpid(), rootDevice: rootInfo.st_dev, + rootInode: rootInfo.st_ino, mountDevice: mountInfo.st_dev, phase: phase, format: format) + try control.validate(parentPID: getpid()) + let outputRoot = SafetyValidator.resolvedDestinationRoot(source: control.source, destination: control.destination, settings: .init()) + try fm.createDirectory(at: outputRoot, withIntermediateDirectories: true) + try sentinel.write(to: outputRoot.appendingPathComponent("existing-unrelated.txt"), options: .withoutOverwriting) + // Archive earlier same-phase evidence before creating the next worker. + for name in ["\(phase.rawValue)-journal.json", "\(phase.rawValue)-journal.json.lock", "\(phase.rawValue)-boundary.json"] { + let file = root.appendingPathComponent(name) + if fm.fileExists(atPath: file.path) { + try fm.moveItem(at: file, to: root.appendingPathComponent("prior-\(receipts.count)-\(name)")) + } + } + let config = root.appendingPathComponent("control.json") + try JSONEncoder().encode(control).write(to: config, options: .atomic) + let log = root.appendingPathComponent("child-\(format)-\(phase.rawValue).log") + try Data().write(to: log, options: .withoutOverwriting) + let output = try FileHandle(forWritingTo: log) + defer { try? output.close() } + let child = Process() + child.executableURL = URL(fileURLWithPath: host) + child.arguments = ["-XCTest", "BitMatchEngineTests.ProcessDeathRecoveryTests/testProcessDeathWorker", bundle.path] + var environment = ProcessInfo.processInfo.environment.filter { + !$0.key.hasPrefix("BITMATCH_") && !$0.key.hasPrefix("TEST_RUNNER_BITMATCH_") + } + environment["BITMATCH_PROCESS_DEATH_WORKER"] = config.path + child.environment = environment + child.standardOutput = output; child.standardError = output + let termination = ProcessDeathTermination() + child.terminationHandler = { termination.record($0) } + try child.run() + worker = child + var joined = false + defer { + if !joined && child.isRunning { + // Only our directly spawned worker, including a stopped worker. + if killAndJoin(child, termination: termination) { worker = nil } + else { XCTFail("Owned child did not settle; retain root and never detach its mount") } + } + } + let deadline = ProcessInfo.processInfo.systemUptime + 60 + while !fm.fileExists(atPath: control.breadcrumb.path) && child.isRunning && ProcessInfo.processInfo.systemUptime < deadline { + try await Task.sleep(for: .milliseconds(20)) + } + try deathRequire(child.isRunning && fm.fileExists(atPath: control.breadcrumb.path), + "Child did not reach actual \(format)/\(phase.rawValue); inspect \(log.path)") + let breadcrumb = try JSONDecoder().decode(ProcessDeathBreadcrumb.self, from: Data(contentsOf: control.breadcrumb)) + try deathRequire(breadcrumb.nonce == nonce && breadcrumb.pid == child.processIdentifier && breadcrumb.phase == phase, + "Child PID/phase/ownership proof mismatch") + let stopDeadline = ProcessInfo.processInfo.systemUptime + 3 + while try child.isRunning && !isStopped(child.processIdentifier) && ProcessInfo.processInfo.systemUptime < stopDeadline { + try await Task.sleep(for: .milliseconds(10)) + } + try deathRequire(try child.isRunning && isStopped(child.processIdentifier), "Owned child did not enter kernel SIGSTOP state") + let disk = try JSONDecoder().decode([LocalTransferRecord].self, from: Data(contentsOf: control.journal)) + try deathRequire(disk.count == 1 && disk[0].id == breadcrumb.recordID && disk[0].state == .running, + "No durable running record before SIGKILL") + try assertBoundary(control) + try deathRequire(Darwin.kill(child.processIdentifier, SIGKILL) == 0, "Could not SIGKILL owned worker") + let joinDeadline = ProcessInfo.processInfo.systemUptime + 10 + while termination.result == nil && ProcessInfo.processInfo.systemUptime < joinDeadline { + try await Task.sleep(for: .milliseconds(10)) + } + let ended = try termination.result.unwrap("SIGKILL worker failed to settle within 10 seconds") + joined = true; worker = nil + try deathRequire(ended.reason == .uncaughtSignal && ended.status == SIGKILL, + "Worker did not terminate by genuine SIGKILL") + let restored = TransferJournal(fileURL: control.journal) + try deathRequire(restored.persistenceError == nil && restored.records.count == 1 && + restored.records[0].state == .interrupted && restored.records[0].id == breadcrumb.recordID, + "Reopen did not classify abandoned running record as interrupted") + let persisted = try JSONDecoder().decode([LocalTransferRecord].self, from: Data(contentsOf: control.journal)) + try deathRequire(persisted[0].state == .interrupted, "Recovered interrupted state was not persisted") + let copied = SafetyValidator.resolvedDestinationRoot(source: control.source, destination: control.destination, settings: .init()) + if phase == .mhlPublished { try await assertCheckIntact(copied) } + else { try await assertCannotCheckGreen(copied) } + let beforeRetry = try snapshot(control.destination) + let retryReads = ProcessDeathCounter() + let retry = try await TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared, + fanOutHooks: .init(destinationDidRead: { _, _, bytes in retryReads.add(bytes) })) + .performFileOperation(sourceURL: control.source, destinationURLs: [control.destination], + verificationMode: .standard, settings: .init(), progressCallback: { _ in }, onFileResult: nil) + let afterRetry = try snapshot(control.destination) + for (path, bytes) in beforeRetry { try deathRequire(afterRetry[path] == bytes, "Retry overwrote existing \(path)") } + let fat = ["exfat", "msdos"].contains(format) + if phase == .copyMidChunk && fat { + try deathRequire(retry.results.count == 1 && !retry.results[0].success, "Partial final file was trusted on retry") + } else { + try deathRequire(retry.results.count == 1 && retry.results[0].success && retry.results[0].verificationResult?.isValid == true, + "Identical retry failed independent verification") + try deathRequire(retryReads.value >= Int(Self.payloadSize), "Retry did not reread destination media") + } + let plan = TransferCompletion.ascmhlPlan(results: retry.results, sourceFiles: retry.sourceManifest, + destinations: [control.destination], source: control.source, settings: .init()) + if phase == .copyMidChunk && fat { + try deathRequire(plan.jobs.isEmpty && !plan.issues.isEmpty, "Partial media retry produced a trusted MHL job") + } else if [.mhlClaimed, .mhlManifestWritten, .mhlBeforeComplete, .mhlPublished].contains(phase) { + let issues = try TransferCompletion.writeASCMHL(plan.jobs, planIssues: plan.issues, + startTime: retry.startTime, source: control.source, toolVersion: "process-death-retry") + try deathRequire(!issues.isEmpty, "Retry replaced existing public history") + if phase == .mhlPublished { try await assertCheckIntact(copied) } + else { try await assertCannotCheckGreen(copied) } + let afterMHL = try snapshot(control.destination) + for (path, bytes) in beforeRetry { try deathRequire(afterMHL[path] == bytes, "MHL retry overwrote \(path)") } + } else { + try deathRequire(plan.jobs.count == 1 && plan.issues.isEmpty, "Verified recovery lacks exact MHL coverage") + let issues = try TransferCompletion.writeASCMHL(plan.jobs, planIssues: plan.issues, + startTime: retry.startTime, source: control.source, toolVersion: "process-death-retry") + try deathRequire(issues.isEmpty, "Verified recovery could not create a new history") + try await assertCheckIntact(copied) + let afterMHL = try snapshot(control.destination) + for (path, bytes) in beforeRetry { try deathRequire(afterMHL[path] == bytes, "Recovery overwrote orphaned \(path)") } + } + try deathRequire(restored.records[0].state == .interrupted, "Retry rewrote the abandoned attempt as successful") + try deathRequire(try snapshot(control.source) == sourceBefore, "Original source changed") + try deathRequire(try Data(contentsOf: caseRoot.appendingPathComponent("unrelated.txt")) == sentinel, "Unrelated sentinel changed") + receipts.append(["format": format, "phase": phase.rawValue, "signal": "SIGKILL(9)", "reopened": "interrupted", + "source": "hashes unchanged", "retry": "existing bytes preserved", + "checkAfterDeath": phase == .mhlPublished ? "intact fully published history" : "not trusted"]) + print("PROCESS_DEATH PASS n=\(receipts.count) filesystem=\(format) phase=\(phase.rawValue) signal=9 recovered=interrupted") + } + + private func assertBoundary(_ control: ProcessDeathControl) throws { + let copied = SafetyValidator.resolvedDestinationRoot(source: control.source, destination: control.destination, settings: .init()) + let media = copied.appendingPathComponent("clip.bin") + if control.phase == .copyMidChunk { + let files = try CardSource.enumerateRegularFiles(base: copied) + let partial = try files.first { $0.relativePath == "clip.bin" || $0.relativePath.contains(".bitmatch.tmp.") } + .unwrap("No actual partial copy at death boundary") + try deathRequire(partial.size > 0 && partial.size < Self.payloadSize, "Copy boundary was not genuinely mid-file") + } else { + try deathRequire(try digest(media) == sourceBefore["clip.bin"], "Drained media differs from source") + } + if ["exfat", "msdos"].contains(control.format) { + let markers = try fm.contentsOfDirectory(at: copied, includingPropertiesForKeys: nil) + .filter { TransferStatusMarker.hasExactName($0.lastPathComponent) } + try deathRequire(markers.count == 1, "No unique durable transfer marker at death boundary") + let expected: TransferStatusMarker.Status = control.phase == .copyMidChunk ? .incomplete : .filesVerified + try deathRequire(TransferStatusMarker.validatedStatus(at: markers[0], relativePath: markers[0].lastPathComponent) == expected, + "FAT transfer marker does not describe actual stopped media state") + } + let history = copied.appendingPathComponent("ascmhl") + switch control.phase { + case .mhlClaimed: + try deathRequire(try fm.contentsOfDirectory(atPath: history.path).isEmpty, "Claim was not empty") + case .mhlManifestWritten, .mhlBeforeComplete: + let names = try fm.contentsOfDirectory(atPath: history.path) + try deathRequire(names.contains { $0.hasSuffix(".mhl") } && names.contains(ASCMHLGenerator.publicationStatusName), + "Partial public MHL was not actually written") + try deathRequire(!ASCMHLGenerator.publicationIsComplete(at: history.appendingPathComponent(ASCMHLGenerator.publicationStatusName)), + "Killed history already claimed complete") + try deathRequire(names.contains("ascmhl_chain.xml") == (control.phase == .mhlBeforeComplete), "Wrong public MHL boundary") + case .mhlPrivateStaged: + try deathRequire(!fm.fileExists(atPath: history.path), "Private stage already published") + let stages = try fm.contentsOfDirectory(at: copied, includingPropertiesForKeys: nil) + .filter { $0.lastPathComponent.hasPrefix(".bitmatch-ascmhl-") } + try deathRequire(stages.count == 1, "No private MHL stage") + let names = try fm.contentsOfDirectory(atPath: stages[0].path) + try deathRequire(names.count == 2 && names.contains("ascmhl_chain.xml") && names.contains { $0.hasSuffix(".mhl") }, + "Private MHL stage not fully written") + case .mhlPublished: + let names = try fm.contentsOfDirectory(atPath: history.path) + try deathRequire(names.contains("ascmhl_chain.xml") && names.contains { $0.hasSuffix(".mhl") }, "No actual published history") + if ["exfat", "msdos"].contains(control.format) { + try deathRequire(ASCMHLGenerator.publicationIsComplete(at: history.appendingPathComponent(ASCMHLGenerator.publicationStatusName)), + "Published FAT history lacks complete durable record") + } + default: + try deathRequire(!fm.fileExists(atPath: history.path), "Unexpected history before MHL") + } + } + + private func assertCannotCheckGreen(_ root: URL) async throws { + let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + do { + let discovery = try await checker.discover(under: root) + let result = try await checker.check(discovery, progress: { _ in }) + try deathRequire(!result.isIntact, "Interrupted/unpublished history produced green Check") + } catch let error as SavedChecksumCheck.CheckError { + try deathRequire([.notFound, .unreadableEvidence].contains(error), "Unexpected Check error: \(error)") + } + } + + private func assertCheckIntact(_ root: URL) async throws { + let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + let discovery = try await checker.discover(under: root) + let checked = try await checker.check(discovery, progress: { _ in }) + try deathRequire(checked.isIntact && checked.matchingPaths == ["clip.bin"] && + checked.validatedHistoryDirectories.count == 1, "Fully published history did not independently Check") + } + + private func snapshot(_ folder: URL) throws -> [String: String] { + guard fm.fileExists(atPath: folder.path) else { return [:] } + return try Dictionary(uniqueKeysWithValues: CardSource.enumerateRegularFiles(base: folder).map { + ($0.relativePath, try digest($0.url)) + }) + } + + /// Observe state without waitpid: Foundation remains the only child reaper. + private func isStopped(_ pid: Int32) throws -> Bool { + var selectors: [Int32] = [CTL_KERN, KERN_PROC, KERN_PROC_PID, pid] + var info = kinfo_proc() + var size = MemoryLayout.stride + let status = selectors.withUnsafeMutableBufferPointer { + sysctl($0.baseAddress, UInt32($0.count), &info, &size, nil, 0) + } + try deathRequire(status == 0 && size == MemoryLayout.stride, "Cannot inspect direct child stop state") + return info.kp_proc.p_stat == SSTOP + } + + private func killAndJoin(_ process: Process, termination: ProcessDeathTermination) -> Bool { + if process.isRunning { _ = Darwin.kill(process.processIdentifier, SIGKILL) } + let deadline = ProcessInfo.processInfo.systemUptime + 10 + while termination.result == nil && ProcessInfo.processInfo.systemUptime < deadline { Thread.sleep(forTimeInterval: 0.01) } + return termination.result != nil + } + + private func digest(_ url: URL) throws -> String { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + var hasher = SHA256() + while let bytes = try autoreleasepool(invoking: { try handle.read(upToCount: 1024 * 1024) }), !bytes.isEmpty { + try Task.checkCancellation(); hasher.update(data: bytes) + } + return hasher.finalize().map { String(format: "%02x", $0) }.joined() + } + + @discardableResult private func command(_ executable: String, _ arguments: [String]) throws -> URL { + let log = root.appendingPathComponent("command-\(UUID()).log") + try Data().write(to: log, options: .withoutOverwriting) + let handle = try FileHandle(forWritingTo: log) + defer { try? handle.close() } + let process = Process(); process.executableURL = URL(fileURLWithPath: executable); process.arguments = arguments + process.standardOutput = handle; process.standardError = handle + let termination = ProcessDeathTermination() + process.terminationHandler = { termination.record($0) } + try process.run() + let deadline = ProcessInfo.processInfo.systemUptime + 90 + while termination.result == nil && ProcessInfo.processInfo.systemUptime < deadline { Thread.sleep(forTimeInterval: 0.02) } + guard let ended = termination.result else { + _ = killAndJoin(process, termination: termination) + throw deathFailure("Owned command timed out; retained \(log.path)") + } + try deathRequire(ended.reason == .exit && ended.status == 0, + "Owned command failed: \(executable); inspect \(log.path)") + return log + } + + private func mountedPaths() throws -> Set { + var entries: UnsafeMutablePointer? + let count = getmntinfo(&entries, MNT_NOWAIT) + try deathRequire(count > 0 && entries != nil, "Cannot enumerate mounts for safe cleanup") + return Set((0.. Wrapped { + guard let value = self else { throw deathFailure(message) }; return value + } +} +private func deathFailure(_ text: String) -> NSError { + NSError(domain: "ProcessDeathRecovery", code: 1, userInfo: [NSLocalizedDescriptionKey: text]) +} +private func deathRequire(_ condition: Bool, _ text: String) throws { + guard condition else { throw deathFailure(text) } +} +#endif diff --git a/docs/validation/PROCESS_DEATH_RECOVERY_20261019.md b/docs/validation/PROCESS_DEATH_RECOVERY_20261019.md new file mode 100644 index 00000000..f735dec7 --- /dev/null +++ b/docs/validation/PROCESS_DEATH_RECOVERY_20261019.md @@ -0,0 +1,69 @@ +# Process-death recovery matrix + +Twenty local scenarios killed a disposable engine worker with SIGKILL on real +mounted APFS, journaled HFS+, exFAT and FAT32 sparse images. No selected physical +drive was used. The source was a synthetic 12 MiB + 137-byte file on the host. +The images and source therefore do not establish independent physical backups. + +| Boundary | APFS | HFS+ | exFAT | FAT32 | +| --- | --- | --- | --- | --- | +| Copy stopped after one chunk was written | Pass | Pass | Pass | Pass | +| Copy and verification drained, before MHL | Pass | Pass | Pass | Pass | +| Private MHL staged, before exclusive publication | Pass | Pass | Not applicable | Not applicable | +| Public MHL directory claimed, still empty | Not applicable | Not applicable | Pass | Pass | +| Public MHL manifest written, before chain | Not applicable | Not applicable | Pass | Pass | +| Manifest and chain written, before COMPLETE record | Not applicable | Not applicable | Pass | Pass | +| MHL fully published, before history settlement | Pass | Pass | Pass | Pass | + +The parent launches only this test bundle's explicit worker method through +Foundation `Process`. The child validates the owned root nonce, device/inode, +mounted filesystem and parent identity. It persists a running transfer journal, +executes the actual transfer/readback and reaches an existing engine boundary. +It synchronizes a phase breadcrumb, then stops itself with SIGSTOP. The parent +checks the breadcrumb PID and actual stopped state, examines the on-disk phase, +sends SIGKILL to that direct worker and requires a bounded termination callback +with signal 9. Foundation is the sole child reaper; there is no `fork`/`waitpid` +or cancellation-error substitute for process death. + +Every reopened journal persisted `interrupted`, without calling it cancelled or +completed. Source names and SHA-256 values stayed unchanged. Existing files, +orphaned stages, partial output and unrelated destination sentinels were +preserved through retry. The FAT transfer markers matched the actual stopped +state. Partial final media was refused; complete media was independently reread. +Empty history reservations and incomplete public histories blocked replacement. +Unpublished or incomplete histories could not produce an intact saved Check. +Recoverable retries produced a new history that passed a fresh saved Check. + +The positive boundary matters: fully published histories remained intact after +the kill and still passed a fresh read and inventory-chain check. Their transfer +journals remained interrupted because terminal settlement had not happened. +Retry preserved those histories and refused to replace them. Process death does +not automatically invalidate evidence that was already completely published. + +The final focused run scheduled three XCTest methods: two passed, the internal +worker method skipped in the parent, and there were zero failures. Its twenty +child scenarios completed in 11.790 seconds. These are scenarios inside one +parent test, not twenty additional XCTest passes. Ordinary discovery schedules +three methods, passing the ownership guard and skipping both opt-in methods. +An earlier four-case APFS/exFAT smoke run also passed. Intermediate smoke work +found a harness-only `Process.waitUntilExit` hang after a child had already +exited; the final harness uses bounded termination callbacks instead. +Those wait bounds do not preempt arbitrary synchronous filesystem or OS calls. + +Run the smoke before the full matrix: + +```sh +BITMATCH_PROCESS_DEATH_MATRIX=smoke swift test \ + --package-path Packages/BitMatchEngine --jobs 4 --disable-swift-testing \ + --filter ProcessDeathRecoveryTests +BITMATCH_PROCESS_DEATH_MATRIX=full swift test \ + --package-path Packages/BitMatchEngine --jobs 4 --disable-swift-testing \ + --filter ProcessDeathRecoveryTests +``` + +All final images were detached without force, and disposable image/source +payloads were removed. Compact journals, child logs, phase breadcrumbs and a +scenario receipt were retained privately. Failures retain their owned fixtures. +No production change was required. This tests process-death recovery, not power +loss, device removal, every possible instruction boundary, or the cause of an +operating-system memory termination. From 3c284702f703713f63d5d1284e5832258103ea76 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 00:07:20 -0400 Subject: [PATCH 37/56] Record process interruption and signed build validation --- docs/OVERNIGHT_VALIDATION_20261019.md | 9 ++++++++- docs/validation/PROCESS_DEATH_RECOVERY_20261019.md | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/OVERNIGHT_VALIDATION_20261019.md b/docs/OVERNIGHT_VALIDATION_20261019.md index dc45ec4a..4359229f 100644 --- a/docs/OVERNIGHT_VALIDATION_20261019.md +++ b/docs/OVERNIGHT_VALIDATION_20261019.md @@ -46,6 +46,12 @@ The running binary was built from `5396f6c5` plus the corrected test harness, be An earlier physical large-file attempt also had a wrong MD5-versus-SHA-256 harness assertion. Completing its byte work does not make that attempt a pass. It is excluded from the successful physical results. +## Abrupt worker termination + +The [process-death matrix](validation/PROCESS_DEATH_RECOVERY_20261019.md) passed 20 scenarios on APFS, journaled HFS+, exFAT and FAT32 images. Each directly spawned worker reached a durable, kernel-stopped boundary and then exited by SIGKILL. Its running journal reopened as interrupted; source hashes and existing bytes were preserved. Incomplete MHL could not pass Check. The positive boundary also matters: a completely published history remained intact and passed a fresh Check, while the unsettled journal stayed interrupted. This is process-death coverage, not a power-loss or OOM-cause reproduction. + +The focused run scheduled three XCTest methods: two passed and one internal helper skipped. The 20 scenarios are not 20 additional XCTest passes. Ordinary discovery passed the ownership guard and skipped the two opt-in methods. Final images and disposable source payloads were cleaned after normal detach. + ## First buffer-benchmark read The selected SSD's opt-in raw benchmark completed **one XCTest, zero failures and zero skips**: 25 hash passes and 25 raw-copy passes, each over a 2 GiB fixture. This is a raw I/O/hash benchmark, not a verified application transfer. @@ -70,7 +76,8 @@ The engine gates were rerun after the buffer-harness additions at `4106a48a`; th | iPad simulator | 81 tests: 71 passed, ten skipped, zero failures; no reported runtime warnings | | iPhone simulator | 81 tests: 71 passed, ten skipped, zero failures; no reported runtime warnings | | Shared mobile build | Universal iOS simulator build passed | -| Local signed Mac archive | Built and signature verified; no release published | +| Release builds | Universal Mac and iOS-device builds passed | +| Local signed Mac archive | `5fb59a64`: x86_64 and arm64, deep/strict signature verification passed; no install, notarization or release | Skipped opt-in tests are not counted as passes. Some were enabled in separate runs above; those runs do not turn the whole-suite skip count into zero. diff --git a/docs/validation/PROCESS_DEATH_RECOVERY_20261019.md b/docs/validation/PROCESS_DEATH_RECOVERY_20261019.md index f735dec7..99194072 100644 --- a/docs/validation/PROCESS_DEATH_RECOVERY_20261019.md +++ b/docs/validation/PROCESS_DEATH_RECOVERY_20261019.md @@ -61,6 +61,8 @@ BITMATCH_PROCESS_DEATH_MATRIX=full swift test \ --filter ProcessDeathRecoveryTests ``` +On Apple Silicon with Xcode 27, prefix the Swift command with `arch -arm64`, as the engine wrapper in `test.sh` does, if the stock helper selects an incompatible XCTest architecture. + All final images were detached without force, and disposable image/source payloads were removed. Compact journals, child logs, phase breadcrumbs and a scenario receipt were retained privately. Failures retain their owned fixtures. From ae9410178a199ac5e83224e68842abf119012a05 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 00:18:39 -0400 Subject: [PATCH 38/56] test: strengthen seeded soak evidence and worker memory guard --- .../DisposableTransferFixtureTests.swift | 20 ++ .../BitMatchEngineTests/StressTests.swift | 9 + .../DisposableTransferFixture.swift | 21 +- .../TransferSoakTests.swift | 231 ++++++++++++------ Scripts/guarded_engine_xctest.py | 196 +++++++++++++++ Scripts/run_soak_tests.sh | 94 ++++--- Scripts/test_guarded_engine_xctest.py | 45 ++++ docs/HARDWARE_TESTING.md | 10 +- docs/validation/SEEDED_SOAK_20261020.md | 64 +++++ 9 files changed, 561 insertions(+), 129 deletions(-) create mode 100755 Scripts/guarded_engine_xctest.py create mode 100644 Scripts/test_guarded_engine_xctest.py create mode 100644 docs/validation/SEEDED_SOAK_20261020.md diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DisposableTransferFixtureTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DisposableTransferFixtureTests.swift index 004ecf72..218b9bea 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DisposableTransferFixtureTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DisposableTransferFixtureTests.swift @@ -64,4 +64,24 @@ struct DisposableTransferFixtureTests { try Data().write(to: marker) fixture.cleanup() } + @Test func explicitBaseRequiresOwnershipAndKeepsParentContents() throws { + let fm = FileManager.default + let base = fm.temporaryDirectory.appendingPathComponent("bitmatch-fixture-base-\(UUID())") + try fm.createDirectory(at: base, withIntermediateDirectories: false) + defer { try? fm.removeItem(at: base) } + #expect(throws: (any Error).self) { + try DisposableTransferFixture(seed: 7, fileCount: 1, bytesPerFile: 16, baseDirectory: base) + } + let marker = base.appendingPathComponent(".bitmatch-disposable-fixture") + try Data().write(to: marker) + let sentinel = base.appendingPathComponent("unrelated.txt") + try Data("keep".utf8).write(to: sentinel) + let fixture = try DisposableTransferFixture(seed: 7, fileCount: 1, bytesPerFile: 16, baseDirectory: base) + #expect(fixture.source.deletingLastPathComponent().deletingLastPathComponent().path == base.path) + fixture.cleanup() + #expect(!fm.fileExists(atPath: fixture.source.path)) + #expect(try Data(contentsOf: sentinel) == Data("keep".utf8)) + #expect(fm.fileExists(atPath: marker.path)) + } + } diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/StressTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/StressTests.swift index 3ce34956..6933a6fb 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/StressTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/StressTests.swift @@ -15,6 +15,7 @@ final class StressTests: XCTestCase { } func testManyTinyFilesAndPathologicalNamesRemainCompleteAndVerified() async throws { + try FileHandle.standardOutput.write(contentsOf: Data("STRESS workerPID=\(getpid())\n".utf8)) let currentScale = scale try await FileOperationsTestLock.shared.run { let fixture = try StressFixture(destinationCount: 1) @@ -77,6 +78,7 @@ final class StressTests: XCTestCase { #endif XCTAssertTrue(verdict(for: operation).success, describe(operation)) + print("STRESS manyTiny files=\(expected) resultRows=\(operation.results.count) scale=\(currentScale)") } } @@ -782,6 +784,9 @@ private final class StressFixture: @unchecked Sendable { try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) } try Data().write(to: root.appendingPathComponent(".bitmatch-stress-fixture")) + if ProcessInfo.processInfo.environment["BITMATCH_STRESS_KEEP_FIXTURES"] == "1" { + try FileHandle.standardOutput.write(contentsOf: Data("STRESS fixtureRoot=\(root.path)\n".utf8)) + } } func write(_ data: Data, to relativePath: String) throws { @@ -791,6 +796,10 @@ private final class StressFixture: @unchecked Sendable { } func cleanup() { + if ProcessInfo.processInfo.environment["BITMATCH_STRESS_KEEP_FIXTURES"] == "1" { + print("STRESS retainedFixture=\(root.path)") + return + } let temporaryRoot = FileManager.default.temporaryDirectory.standardizedFileURL.path guard root.standardizedFileURL.path.hasPrefix(temporaryRoot), FileManager.default.fileExists(atPath: root.appendingPathComponent(".bitmatch-stress-fixture").path) else { return } diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TestHelpers/DisposableTransferFixture.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TestHelpers/DisposableTransferFixture.swift index 6bff52b1..e1a29845 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TestHelpers/DisposableTransferFixture.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TestHelpers/DisposableTransferFixture.swift @@ -7,10 +7,22 @@ final class DisposableTransferFixture: Sendable { let manifest: [String: String] private let root: URL + private let cleanupBase: URL - init(seed: UInt64, fileCount: Int, bytesPerFile: Int) throws { + init(seed: UInt64, fileCount: Int, bytesPerFile: Int, baseDirectory: URL? = nil) throws { let fileManager = FileManager.default - let root = fileManager.temporaryDirectory + let base = baseDirectory ?? fileManager.temporaryDirectory + if baseDirectory != nil { + let resolved = base.standardizedFileURL.resolvingSymlinksInPath().path + let temporary = fileManager.temporaryDirectory.standardizedFileURL.resolvingSymlinksInPath().path + let marker = base.appendingPathComponent(".bitmatch-disposable-fixture") + let attributes = try fileManager.attributesOfItem(atPath: marker.path) + guard (resolved.hasPrefix(temporary + "/") || resolved.hasPrefix("/private/tmp/")), + attributes[.type] as? FileAttributeType == .typeRegular else { + throw CocoaError(.fileReadNoPermission) + } + } + let root = base .appendingPathComponent("bitmatch-disposable-\(UUID().uuidString)", isDirectory: true) let source = root.appendingPathComponent("SOURCE", isDirectory: true) let mediaDirectory = source.appendingPathComponent("DCIM/100MEDIA", isDirectory: true) @@ -44,15 +56,14 @@ final class DisposableTransferFixture: Sendable { } self.root = root + self.cleanupBase = base self.source = source self.destinations = destinations self.manifest = manifest } func cleanup() { - guard root.standardizedFileURL.path.hasPrefix( - FileManager.default.temporaryDirectory.standardizedFileURL.path - ), FileManager.default.fileExists( + guard root.deletingLastPathComponent().standardizedFileURL.path == cleanupBase.standardizedFileURL.path, FileManager.default.fileExists( atPath: root.appendingPathComponent(".bitmatch-disposable-fixture").path ) else { return } diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TransferSoakTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TransferSoakTests.swift index 089f9806..671281e2 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TransferSoakTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TransferSoakTests.swift @@ -1,16 +1,39 @@ import Foundation +import CryptoKit import XCTest #if canImport(Darwin) import Darwin #endif @testable import BitMatchEngine -final class TransferSoakTests: XCTestCase { +final class TransferSoakTests: XCTestCase, @unchecked Sendable { + func testSourceGuardDetectsMutationAndInventoryChanges() throws { + let fixture = try DisposableTransferFixture(seed: 13, fileCount: 1, bytesPerFile: 16) + defer { if testRun?.totalFailureCount == 0 { fixture.cleanup() } } + let source = soakCanonicalFileURL(fixture.source) + XCTAssertEqual(try verifySoakSource(source, expected: fixture.manifest), 2) + let file = source.appendingPathComponent("DCIM/100MEDIA/MEDIA_0000.bin") + let original = try Data(contentsOf: file) + try Data(repeating: 0xFE, count: original.count).write(to: file) + XCTAssertThrowsError(try verifySoakSource(source, expected: fixture.manifest)) + try original.write(to: file) + let extra = source.appendingPathComponent("unexpected.bin") + try Data([1]).write(to: extra) + XCTAssertThrowsError(try verifySoakSource(source, expected: fixture.manifest)) + try FileManager.default.removeItem(at: extra) + let empty = source.appendingPathComponent("EMPTY_SIDECARS") + try FileManager.default.removeItem(at: empty) + XCTAssertThrowsError(try verifySoakSource(source, expected: fixture.manifest)) + try FileManager.default.createDirectory(at: empty, withIntermediateDirectories: false) + XCTAssertEqual(try verifySoakSource(source, expected: fixture.manifest), 2) + } + func testSeededStandardTransferToTwoDestinations() async throws { guard ProcessInfo.processInfo.environment["BITMATCH_RUN_SOAK"] == "1" else { throw XCTSkip("Set BITMATCH_RUN_SOAK=1 to run the seeded transfer soak test") } + try FileHandle.standardOutput.write(contentsOf: Data("SOAK workerPID=\(getpid())\n".utf8)) let environment = ProcessInfo.processInfo.environment let seed = try parseUInt64(environment["BITMATCH_SOAK_SEED"], name: "BITMATCH_SOAK_SEED", default: 20_260_711) let iterations = try parsePositiveInt( @@ -20,99 +43,137 @@ final class TransferSoakTests: XCTestCase { ) var iterationSummaries: [SoakIterationSummary] = [] - try await FileOperationsTestLock.shared.run { - for iteration in 0.. UInt64 { @@ -145,6 +206,8 @@ private enum SoakConfigurationError: LocalizedError { private enum SoakVerificationError: LocalizedError { case unexpectedResultCount(expected: Int, actual: Int, details: String) case failedOperationResults(String) + case sourceChanged + case assertionFailure case hashMismatch(relativePath: String, expected: String, actual: String) var errorDescription: String? { @@ -153,6 +216,10 @@ private enum SoakVerificationError: LocalizedError { return "Expected \(expected) soak results, received \(actual): \(details)" case let .failedOperationResults(details): return "Soak operation contained failed results: \(details)" + case .sourceChanged: + return "Source inventory or SHA-256 contents changed" + case .assertionFailure: + return "Soak assertions or final iteration coverage failed" case let .hashMismatch(relativePath, expected, actual): return "SHA-256 mismatch for \(relativePath): expected \(expected), received \(actual)" } @@ -192,3 +259,29 @@ private func describeSoakResults(_ results: [FileOperationResult]) -> String { "src=\($0.sourceURL.path), dst=\($0.destinationURL.path), success=\($0.success), status=\($0.statusDescription)" }.joined(separator: " | ") } + +/// Read the actual source inventory before and after, without checksum-cache reuse. +private func verifySoakSource(_ source: URL, expected: [String: String]) throws -> Int { + let directories = Set(try CardSource.enumerateTree(base: source).filter { $0.kind == .directory }.map(\.relativePath)) + guard directories == ["DCIM", "DCIM/100MEDIA", "EMPTY_SIDECARS"] else { throw SoakVerificationError.sourceChanged } + let files = try CardSource.enumerateRegularFiles(base: source) + guard Set(files.map(\.relativePath)) == Set(expected.keys) else { throw SoakVerificationError.sourceChanged } + for file in files { + try Task.checkCancellation() + let handle = try FileHandle(forReadingFrom: file.url) + defer { try? handle.close() } + var hasher = SHA256() + while true { + try Task.checkCancellation() + let count = try autoreleasepool { + let data = try handle.read(upToCount: 1024 * 1024) ?? Data() + hasher.update(data: data) + return data.count + } + if count == 0 { break } + } + let digest = hasher.finalize().map { String(format: "%02x", $0) }.joined() + guard digest == expected[file.relativePath] else { throw SoakVerificationError.sourceChanged } + } + return files.count +} diff --git a/Scripts/guarded_engine_xctest.py b/Scripts/guarded_engine_xctest.py new file mode 100755 index 00000000..8e638dfc --- /dev/null +++ b/Scripts/guarded_engine_xctest.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Run one exact XCTest in an owned engine bundle; sample its physical footprint. +Only SIGINT is sent to the validated direct worker. Native calls may still wait. +""" +import argparse +import ctypes +import json +import os +from pathlib import Path +import re +import signal +import subprocess +import time + + +class Usage(ctypes.Structure): + # Public Darwin rusage_info_v0 layout (sys/resource.h). + _fields_ = [('uuid', ctypes.c_uint8 * 16)] + [(name, ctypes.c_uint64) for name in + ['user', 'system', 'pkg', 'interrupt', 'pageins', 'wired', 'resident', 'footprint', 'start', 'exit']] + + +class WorkerExited(Exception): + pass + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def validate_xctest(text, selected): + require(re.fullmatch(r'BitMatchEngineTests\.[A-Za-z0-9_]+/[A-Za-z0-9_]+', selected), 'Invalid explicit XCTest selection') + class_name, method = selected.rsplit('/', 1) + rows = re.findall(r'Executed (\d+) tests?, with (?:(\d+) tests? skipped and )?(\d+) failures', text) + require(rows and tuple(int(x or 0) for x in rows[-1]) == (1, 0, 0), 'Expected one actual passing XCTest') + require(len(re.findall(r"Test Case '-\[" + re.escape(class_name) + " " + re.escape(method) + r"\]' passed", text)) == 1, + 'Selected XCTest did not pass exactly once') + + +def validate_soak(value, seed, iterations): + require(all(type(value.get(key)) is int for key in ['formatVersion', 'baseSeed', 'requestedIterations', 'completedIterations', 'destinationsPerIteration']), 'Invalid receipt numeric fields') + require(value.get('formatVersion') == 2 and value.get('outcome') == 'passed', 'Missing successful format-2 receipt') + require(value.get('baseSeed') == seed and value.get('requestedIterations') == iterations + and value.get('completedIterations') == iterations, 'Incomplete iteration coverage') + require(value.get('verificationMode') == 'standard' and value.get('destinationsPerIteration') == 2, + 'Unexpected transfer policy') + rows = value.get('iterations', []) + require(len(rows) == iterations, 'Wrong iteration receipt count') + for index, row in enumerate(rows): + require(all(type(number) is int for number in row.values()), 'Invalid iteration numeric fields') + expected_seed = (seed + index) & ((1 << 64) - 1) + require(row == {'iteration': index, 'seed': expected_seed, 'manifestFiles': 9, 'verifiedOutputs': 18, + 'sourceHashesBefore': 9, 'sourceHashesAfter': 9, 'sourceDirectoriesBeforeAfter': 3, + 'payloadPattern': expected_seed & 255}, 'Iteration inventory/hash coverage mismatch') + + +def run(args): + work = Path(args.work).resolve() + require(work.is_dir() and (work / '.bitmatch-disposable-fixture').is_file(), 'Unowned worker directory') + bundle = Path(args.bundle).resolve() + require(bundle.suffix == '.xctest' and bundle.is_dir(), 'Missing explicit engine test bundle') + host = subprocess.check_output(['/usr/bin/xcrun', '--find', 'xctest'], text=True, timeout=15).strip() + require(host.startswith('/Applications/') and host.endswith('/usr/bin/xctest'), 'Unexpected XCTest host') + architecture = subprocess.check_output(['/usr/bin/uname', '-m'], text=True, timeout=15).strip() + require(architecture in ('arm64', 'x86_64'), 'Unsupported host architecture') + environment = {key: value for key, value in os.environ.items() + if not key.startswith(('BITMATCH_', 'TEST_RUNNER_BITMATCH_'))} + explicit_environment = json.loads(args.environment) + require(isinstance(explicit_environment, dict) and all(isinstance(key, str) and isinstance(value, str) + for key, value in explicit_environment.items()), 'Worker environment must contain string fields') + require(re.fullmatch(r'BitMatchEngineTests\.[A-Za-z0-9_]+/[A-Za-z0-9_]+', args.selected), 'Invalid explicit XCTest selection') + environment.update(explicit_environment) + log = work / (args.label + '.log') + receipt = {'scope': 'Host-only selected XCTest; public proc_pid_rusage physical footprint, sampled not a leak proof', + 'limitBytes': 1024 ** 3, 'sampleIntervalSeconds': 0.01, 'samples': 0, 'peakBytes': 0, + 'guard': False, 'missingSamples': 0, 'outcome': 'failed', 'stillActive': False} + lib = ctypes.CDLL('/usr/lib/libproc.dylib', use_errno=True) + lib.proc_pid_rusage.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_void_p] + lib.proc_pid_rusage.restype = ctypes.c_int + lib.proc_pidpath.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_uint32] + lib.proc_pidpath.restype = ctypes.c_int + child = None + interrupted = None + start_identity = None + worker_seen = False + + def usage(): + item = Usage() + if lib.proc_pid_rusage(child.pid, 0, ctypes.byref(item)) != 0: + return None + return item + + def validate_worker(): + item = usage() + buffer = ctypes.create_string_buffer(4096) + path_size = lib.proc_pidpath(child.pid, buffer, len(buffer)) + if item is None or path_size <= 0: + # A successful worker may exit between poll and the read-only query. + # Join that exit before calling a vanished sample unavailable. + try: + child.wait(timeout=0.2) + raise WorkerExited() + except subprocess.TimeoutExpired: + receipt['missingSamples'] += 1 + receipt['guard'] = True + raise ValueError('Worker physical footprint or identity unavailable') + require(Path(os.fsdecode(buffer.value)).resolve() == Path(host).resolve(), 'Worker executable changed') + require(start_identity is None or item.start == start_identity, 'Worker process identity changed') + return item + + def request_interrupt(): + if child is not None and child.poll() is None: + buffer = ctypes.create_string_buffer(4096) + if lib.proc_pidpath(child.pid, buffer, len(buffer)) <= 0: + try: + child.wait(timeout=0.2) + return + except subprocess.TimeoutExpired: + raise ValueError('Cannot validate worker before interrupt') + require(Path(os.fsdecode(buffer.value)).resolve() == Path(host).resolve(), 'Interrupt target executable changed') + item = usage() + # Popen owns this direct, unreaped child; absence of a memory + # sample must not prevent interrupting that exact validated host. + require(item is None or start_identity is None or item.start == start_identity, + 'Interrupt target process identity changed') + child.send_signal(signal.SIGINT) + + def receive(signum, frame): + nonlocal interrupted + interrupted = signum + + handlers = {sig: signal.signal(sig, receive) for sig in (signal.SIGINT, signal.SIGTERM)} + started = time.monotonic() + try: + with log.open('x') as output: + child = subprocess.Popen(['/usr/bin/arch', '-' + architecture, host, '-XCTest', args.selected, str(bundle)], + env=environment, stdout=output, stderr=subprocess.STDOUT) + receipt['childPID'] = child.pid + while child.poll() is None: + text = log.read_text(errors='replace') + match = re.search(r'^(?:SOAK|STRESS) workerPID=(\d+)$', text, re.M) + if match: + require(int(match.group(1)) == child.pid, 'Breadcrumb does not identify direct worker') + try: + item = validate_worker() + except WorkerExited: + break + if start_identity is None: + start_identity = item.start + worker_seen = True + receipt['samples'] += 1 + receipt['peakBytes'] = max(receipt['peakBytes'], item.footprint) + if item.footprint >= receipt['limitBytes']: + receipt['guard'] = True + if time.monotonic() - started >= args.max_seconds or interrupted or receipt['guard']: + receipt['interruptRequested'] = True + request_interrupt() + try: + child.wait(timeout=60) + except subprocess.TimeoutExpired: + receipt['stillActive'] = True + break + time.sleep(0.01) + require(child.poll() == 0 and worker_seen and interrupted is None and not receipt.get('interruptRequested'), 'Worker failed or was interrupted') + validate_xctest(log.read_text(errors='replace'), args.selected) + receipt['outcome'] = 'passed' + finally: + if child is not None and child.poll() is None: + try: + request_interrupt() + child.wait(timeout=60) + except (OSError, ValueError, subprocess.TimeoutExpired): + receipt['stillActive'] = child.poll() is None + receipt['stillActive'] = child is not None and child.poll() is None + receipt.update(exitStatus=child.poll() if child else None, elapsedSeconds=time.monotonic() - started, + actualWorkerObserved=worker_seen, interruptedSignal=interrupted) + if interrupted is not None: + receipt['outcome'] = 'failed' + (work / (args.label + '-memory.json')).write_text(json.dumps(receipt, indent=2) + '\n') + for sig, handler in handlers.items(): + signal.signal(sig, handler) + print(json.dumps(receipt), flush=True) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--work', required=True) + parser.add_argument('--bundle', required=True) + parser.add_argument('--selected', required=True) + parser.add_argument('--environment', required=True) + parser.add_argument('--label', default='worker') + parser.add_argument('--max-seconds', type=float, default=900) + arguments = parser.parse_args() + require(0 < arguments.max_seconds <= 3600, 'Runtime limit must be 1–3600 seconds') + require(re.fullmatch('[A-Za-z0-9_-]+', arguments.label), 'Invalid log label') + run(arguments) diff --git a/Scripts/run_soak_tests.sh b/Scripts/run_soak_tests.sh index b931e345..44e60838 100755 --- a/Scripts/run_soak_tests.sh +++ b/Scripts/run_soak_tests.sh @@ -5,77 +5,67 @@ ROOT=$(cd "$(dirname "$0")/.." && pwd) source "$ROOT/Scripts/harness_evidence.sh" WORK=$(mktemp -d "${TMPDIR:-/tmp}/bitmatch-soak.XXXXXX") MARKER="$WORK/.bitmatch-disposable-fixture" -DERIVED_DATA=${BITMATCH_DERIVED_DATA_PATH:-"$WORK/DerivedData"} +ENGINE_SCRATCH=${BITMATCH_ENGINE_SCRATCH_PATH:-"$WORK/EngineBuild"} RESULT="$WORK/soak-result.json" touch "$MARKER" +mkdir "$WORK/fixtures" +touch "$WORK/fixtures/.bitmatch-disposable-fixture" cleanup() { local status=$? trap - EXIT INT TERM - remove_harness_test_run || { - echo "Unable to remove temporary test configuration: ${HARNESS_XCTESTRUN:-unknown}" >&2 - status=1 - } - local preserve_work=0 if [[ -f "$RESULT" ]]; then - if [[ -z "${EVIDENCE:-}" ]] || ! cp "$RESULT" "$EVIDENCE/soak-result.json"; then - echo "Unable to retain soak result; preserving original at: $RESULT" >&2 - preserve_work=1 - status=1 - fi + cp "$RESULT" "$EVIDENCE/soak-result.json" || status=1 fi - if [[ "$preserve_work" == 1 ]]; then - echo "Preserving harness directory: $WORK" >&2 - elif [[ -f "$MARKER" ]]; then - rm -rf "$WORK" || { - echo "Unable to remove harness directory: $WORK" >&2 - status=1 - } + if [[ "$status" == 0 && -f "$MARKER" ]]; then + rm -rf "$WORK" || status=1 else - echo "Refusing to remove unmarked soak directory: $WORK" >&2 - status=1 + echo "Preserving failed/interrupted harness directory: $WORK" >&2 fi - finish_evidence "$status" || { - echo "Unable to finalize evidence metadata: ${EVIDENCE:-unknown}; run failed" >&2 - status=1 - } + finish_evidence "$status" || status=1 exit "$status" } trap cleanup EXIT trap 'exit 130' INT trap 'exit 143' TERM - initialize_evidence soak -RESULT_BUNDLE="$EVIDENCE/results.xcresult" BITMATCH_SOAK_SEED=${BITMATCH_SOAK_SEED:-20260711} BITMATCH_SOAK_ITERATIONS=${BITMATCH_SOAK_ITERATIONS:-25} +BITMATCH_SOAK_MAX_SECONDS=${BITMATCH_SOAK_MAX_SECONDS:-900} printf 'seed=%s\niterations=%s\n' "$BITMATCH_SOAK_SEED" "$BITMATCH_SOAK_ITERATIONS" >> "$EVIDENCE/environment.txt" -[[ "$BITMATCH_SOAK_SEED" =~ ^[0-9]+$ ]] || { echo "BITMATCH_SOAK_SEED must be an unsigned integer" >&2; exit 64; } -[[ "$BITMATCH_SOAK_ITERATIONS" =~ ^[1-9][0-9]*$ ]] || { echo "BITMATCH_SOAK_ITERATIONS must be a positive integer" >&2; exit 64; } - -xcodebuild -quiet \ - -project "$ROOT/BitMatch.xcodeproj" \ - -scheme BitMatch \ - -destination 'platform=macOS' \ - -derivedDataPath "$DERIVED_DATA" \ - CODE_SIGNING_ALLOWED=NO \ - build-for-testing 2>&1 | tee "$EVIDENCE/build.log" - -prepare_harness_test_run -/usr/bin/python3 "$ROOT/Scripts/configure_harness_environment.py" "$XCTESTRUN" \ - BITMATCH_RUN_SOAK=1 \ - "BITMATCH_SOAK_SEED=$BITMATCH_SOAK_SEED" \ - "BITMATCH_SOAK_ITERATIONS=$BITMATCH_SOAK_ITERATIONS" \ - "BITMATCH_SOAK_RESULT=$RESULT" - -xcodebuild -quiet \ - test-without-building \ - -xctestrun "$XCTESTRUN" \ - -destination 'platform=macOS' \ - -resultBundlePath "$RESULT_BUNDLE" \ - -only-testing:BitMatchTests/TransferSoakTests 2>&1 | tee "$EVIDENCE/test.log" +[[ "$BITMATCH_SOAK_SEED" =~ ^[0-9]+$ ]] || { echo "BITMATCH_SOAK_SEED must be unsigned" >&2; exit 64; } +[[ "$BITMATCH_SOAK_ITERATIONS" =~ ^[1-9][0-9]*$ ]] || { echo "BITMATCH_SOAK_ITERATIONS must be positive" >&2; exit 64; } -/usr/bin/python3 -m json.tool "$RESULT" >/dev/null +# The soak suite lives in the engine package, not the app's XCTest target. +# Explicit testability supports Release @testable imports with the native builder. +/usr/bin/xcrun swift build --package-path "$ROOT/Packages/BitMatchEngine" \ + --scratch-path "$ENGINE_SCRATCH" --build-system native --jobs 4 \ + --configuration release --build-tests -Xswiftc -enable-testing 2>&1 | tee "$EVIDENCE/build.log" +BIN=$(/usr/bin/xcrun swift build --package-path "$ROOT/Packages/BitMatchEngine" \ + --scratch-path "$ENGINE_SCRATCH" --build-system native --configuration release --show-bin-path) +ENVIRONMENT=$(/usr/bin/python3 - "$BITMATCH_SOAK_SEED" "$BITMATCH_SOAK_ITERATIONS" "$RESULT" "$WORK/fixtures/" <<'PY' +import json, sys +seed, iterations = int(sys.argv[1]), int(sys.argv[2]) +assert 0 <= seed < 1 << 64 and iterations > 0 +print(json.dumps({'BITMATCH_RUN_SOAK': '1', 'BITMATCH_SOAK_SEED': str(seed), + 'BITMATCH_SOAK_ITERATIONS': str(iterations), 'BITMATCH_SOAK_RESULT': sys.argv[3], 'BITMATCH_SOAK_FIXTURE_ROOT': sys.argv[4]})) +PY +) +/usr/bin/python3 "$ROOT/Scripts/guarded_engine_xctest.py" \ + --work "$WORK" --bundle "$BIN/BitMatchEnginePackageTests.xctest" \ + --selected BitMatchEngineTests.TransferSoakTests/testSeededStandardTransferToTwoDestinations \ + --environment "$ENVIRONMENT" --max-seconds "$BITMATCH_SOAK_MAX_SECONDS" 2>&1 | tee "$EVIDENCE/guard.log" +cp "$WORK/worker.log" "$EVIDENCE/test.log" +cp "$WORK/worker-memory.json" "$EVIDENCE/memory.json" +PYTHONDONTWRITEBYTECODE=1 /usr/bin/python3 - "$ROOT/Scripts" "$RESULT" "$BITMATCH_SOAK_SEED" "$BITMATCH_SOAK_ITERATIONS" <<'PY' +import json, sys +sys.path.insert(0, sys.argv[1]) +from guarded_engine_xctest import validate_soak +value = json.load(open(sys.argv[2])) +validate_soak(value, int(sys.argv[3]), int(sys.argv[4])) +print('SOAK validated actualIterations={} sourceHashesBefore={} sourceHashesAfter={} destinationHashes={}'.format( + value['completedIterations'], sum(row['sourceHashesBefore'] for row in value['iterations']), + sum(row['sourceHashesAfter'] for row in value['iterations']), sum(row['verifiedOutputs'] for row in value['iterations']))) +PY cp "$RESULT" "$EVIDENCE/soak-result.json" -cat "$RESULT" diff --git a/Scripts/test_guarded_engine_xctest.py b/Scripts/test_guarded_engine_xctest.py new file mode 100644 index 00000000..8a10a68b --- /dev/null +++ b/Scripts/test_guarded_engine_xctest.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +import copy +import unittest +from guarded_engine_xctest import validate_soak, validate_xctest + +METHOD = 'testSeededStandardTransferToTwoDestinations' +SELECTED = 'BitMatchEngineTests.TransferSoakTests/' + METHOD +PASS = "Test Case '-[BitMatchEngineTests.TransferSoakTests " + METHOD + "]' passed (0.1 seconds).\n" + + +def result(): + return {'formatVersion': 2, 'outcome': 'passed', 'baseSeed': 20261020, 'requestedIterations': 1, + 'completedIterations': 1, 'verificationMode': 'standard', 'destinationsPerIteration': 2, + 'iterations': [{'iteration': 0, 'seed': 20261020, 'manifestFiles': 9, 'verifiedOutputs': 18, + 'sourceHashesBefore': 9, 'sourceHashesAfter': 9, 'sourceDirectoriesBeforeAfter': 3, + 'payloadPattern': 20261020 & 255}]} + + +class EvidenceTests(unittest.TestCase): + def test_exact_pass_is_accepted(self): + validate_xctest(PASS + 'Executed 1 test, with 0 failures', SELECTED) + validate_xctest('SOAK partialbuffer' + PASS + 'Executed 1 test, with 0 failures', SELECTED) + validate_soak(result(), 20261020, 1) + + def test_vacuous_skip_failure_and_duplicate_pass_are_rejected(self): + for text in ['Executed 0 tests, with 0 failures', PASS + 'Executed 1 test, with 1 test skipped and 0 failures', + PASS + 'Executed 1 test, with 1 failures', PASS + PASS + 'Executed 1 test, with 0 failures', + PASS.replace('TransferSoakTests', 'OtherTests') + 'Executed 1 test, with 0 failures']: + with self.subTest(text=text), self.assertRaises(ValueError): + validate_xctest(text, SELECTED) + + def test_partial_running_or_wrong_coverage_is_rejected(self): + for key, value in [('completedIterations', 0), ('outcome', 'running'), ('baseSeed', 1), ('formatVersion', 1)]: + receipt = result(); receipt[key] = value + with self.subTest(key=key), self.assertRaises(ValueError): + validate_soak(receipt, 20261020, 1) + for key, value in [('verifiedOutputs', 17), ('sourceHashesBefore', 0), ('sourceHashesAfter', 8), + ('sourceDirectoriesBeforeAfter', 2), ('payloadPattern', 0), ('iteration', False)]: + receipt = copy.deepcopy(result()); receipt['iterations'][0][key] = value + with self.subTest(key=key), self.assertRaises(ValueError): + validate_soak(receipt, 20261020, 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/docs/HARDWARE_TESTING.md b/docs/HARDWARE_TESTING.md index ce3e2feb..f7fac6af 100644 --- a/docs/HARDWARE_TESTING.md +++ b/docs/HARDWARE_TESTING.md @@ -58,11 +58,15 @@ BITMATCH_EVIDENCE_ROOT="$HOME/Desktop/BitMatch-evidence" \ bash Scripts/run_apfs_fault_tests.sh ``` -The APFS harness changes accessibility inside its own mounted disk image; it does **not** unplug a drive or simulate a controller losing power. The soak harness uses temporary directories on the host filesystem, not a physical-device compatibility matrix. Both require space for an Xcode build; the APFS run also needs its disk image. +The APFS harness changes accessibility inside its own mounted disk image; it does **not** unplug a drive or simulate a controller losing power. The soak harness uses temporary directories on the host filesystem, not a physical-device compatibility matrix. Both need the selected Xcode toolchain. The soak needs space for a Release engine test build; the APFS run needs an app build and its disk image. -Each invocation creates a unique evidence directory containing `environment.txt`, build/test logs, and an Xcode result bundle when test execution starts. The APFS run retains disk-image creation and attachment output in `setup.log`, including setup failures before tests start. The soak run also retains `soak-result.json` when produced. Evidence survives test failure and fixture cleanup. The default evidence root is `${TMPDIR:-/tmp}/bitmatch-evidence`; set `BITMATCH_EVIDENCE_ROOT` to a durable location for records you intend to keep. A run stopped before tests begin may have no result bundle. Consult `exit_status`, the logs, and the result bundle together; a missing result is not a pass. +Each invocation creates a unique evidence directory containing `environment.txt` and build/test logs. The APFS run also retains an Xcode result bundle when execution starts, and disk-image creation/attachment output in `setup.log`. The soak runs the engine's actual selected XCTest, with `memory.json` and a format-2 `soak-result.json` recording completed iterations, source hashes before/after, empty-folder inventory, and both backups' hashes. It requires one passing test without skips and exact iteration coverage; a missing or partial receipt is not a pass. -To reuse an existing build directory, set `BITMATCH_DERIVED_DATA_PATH` to an absolute path. Run harnesses sequentially when they share that directory. A caller-supplied build directory is retained; the default temporary build directory is removed. Do not point evidence or build paths at throwaway removable media used for faults. Open a retained bundle with `open /path/to/results.xcresult`. +The soak samples the actual worker's physical footprint at a requested 10 ms interval, verifies its PID/executable identity and requests SIGINT at 1 GiB or the runtime limit. `BITMATCH_SOAK_MAX_SECONDS` defaults to 900 (maximum 3600). Missing samples fail the run. OS calls and synchronous work may delay sampling or interruption; this is not an out-of-memory survival guarantee or a leak proof. Failed or interrupted fixtures are retained, and normal cleanup removes only the marked disposable work directory. Review raw logs before sharing them. + +The source layout and file sizes are fixed. Payload bytes use the seed's low eight bits, so patterns repeat after 256 consecutive seeds. More iterations add duration, not unlimited input diversity. + +The default evidence root is `${TMPDIR:-/tmp}/bitmatch-evidence`; set `BITMATCH_EVIDENCE_ROOT` to a durable location. For reusable builds, set `BITMATCH_ENGINE_SCRATCH_PATH` for the soak or `BITMATCH_DERIVED_DATA_PATH` for the APFS app harness. Run harnesses sequentially when sharing a build directory. Caller-supplied build directories are retained; default disposable builds are removed after success. Do not put evidence or builds on removable media used for faults. Only the APFS harness emits `results.xcresult`; open that bundle with `open /path/to/results.xcresult`. Before sharing evidence, review it for local paths, usernames, device identifiers, and private filenames. Keep full raw records privately and publish a redacted copy. Record the revision and whether the working tree contained changes; a revision alone does not identify uncommitted source changes. diff --git a/docs/validation/SEEDED_SOAK_20261020.md b/docs/validation/SEEDED_SOAK_20261020.md new file mode 100644 index 00000000..cf1786f5 --- /dev/null +++ b/docs/validation/SEEDED_SOAK_20261020.md @@ -0,0 +1,64 @@ +# Seeded host soak and many-file stress + +This extends the existing engine harness on disposable host storage. It does +not use selected physical drives, app UI, reports or MHL, and does not reproduce +issue #23. + +The soak now checks the actual source inventory and SHA-256 values before and +after each transfer, including its three directories. It requires the exact +18 destination paths, successful Standard readback and independent destination +hashes. Failed assertions, incomplete iterations or cancellation prevent a +passed receipt and preserve the current fixture. Format-2 receipts retain +partial progress. Explicit marked fixture bases keep script-owned inputs inside +the disposable work directory; setting `TMPDIR` alone did not relocate +Foundation's temporary directory on this Mac. + +The public runner previously selected an app test that had moved into the +engine package. It now builds the Release engine tests with four jobs and runs +one explicit XCTest. A read-only observer validates the direct worker's PID, +executable and process identity, samples physical footprint at a requested +10 ms interval, and requests SIGINT at 1 GiB or its deadline. It never force +kills. Sampling and OS calls can wait; brief peaks can be missed. These are +sampled guardrails, not hard latency limits or a leak proof. + +## Actual results + +- One soak XCTest completed **256/256 iterations**, seed **20261020** through + **20261275**, in 26.548 seconds. There were zero test failures or skips. + All 256 low-byte payload patterns occurred once. The fixed layout contains + eight 256 KiB files and one small hidden metadata file; this is bounded + byte-pattern coverage, not unlimited random filenames or workload diversity. +- The run checked **2,304 source hashes before**, **2,304 after**, and + **4,608 destination hashes** across two backups per iteration. Directory + inventories stayed unchanged. Sampled worker peak was **11.41 MiB** across + 1,869 samples, without a threshold or missing-sample failure. +- The first runner validator falsely rejected that successful run because + buffered Swift stdout split an XCTest pass line. The exact selected + class/method token and complete format-2 receipt were revalidated with the + corrected parser; the 256 payloads were not rerun. Original failure metadata + and the later revalidation receipt are both retained privately. +- The final complete script passed a separate two-iteration run: one actual + XCTest, no skips/failures, 18 source hashes before/after and 36 destination + hashes. Its sampled peak was 9.46 MiB. This validates the final explicit + fixture-base and cleanup path. +- The existing many-tiny-file case at scale four passed one actual XCTest in + 86.859 seconds: **20,009 verified results**, deep paths, 255-byte names, + Unicode forms, hidden files and zero-byte content. Independent post-run + checks matched **20,009 source hashes and 20,009 destination hashes** against + the declared generation bytes, with 65 matching directories. This case had + no independent pre-transfer hash pass. The source test attribute remained; + the destination correctly omitted it. Sampled peak was **144.72 MiB** across + 6,115 samples. +- A prior many-file attempt was intentionally interrupted with SIGINT to release + the shared CPU slot. It is retained as an interrupted attempt, not a passing + test or an application failure. Timing observations share the host with + other validation work and are not performance promises. +- Five fixture tests and one source-mutation XCTest passed. The new checks + reject unmarked fixture bases, preserve parent sentinels, and detect same-size + source mutation, unexpected files and a missing empty directory. Three Python + test methods also passed, including 15 negative log/receipt variants for + zero tests, skips, failures, wrong selection, duplicates and partial coverage. + +No production code changed. Successful disposable media was removed after +hash validation; failed/interrupted evidence remains private. Raw logs include +local fixture paths and should be reviewed before sharing. From 1beed21aaaa737d595855b86a74180f8d9e0582b Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 00:19:47 -0400 Subject: [PATCH 39/56] Index partial transfer results without changing row matching --- BitMatchTests/TransferRunResultsTests.swift | 52 +++++++++++++++++++ .../Core/Services/SharedAppCoordinator.swift | 20 +++++-- docs/validation/RUN_RESULT_LOOKUP_20261020.md | 30 +++++++++++ 3 files changed, 98 insertions(+), 4 deletions(-) create mode 100644 BitMatchTests/TransferRunResultsTests.swift create mode 100644 docs/validation/RUN_RESULT_LOOKUP_20261020.md diff --git a/BitMatchTests/TransferRunResultsTests.swift b/BitMatchTests/TransferRunResultsTests.swift new file mode 100644 index 00000000..ec4265f8 --- /dev/null +++ b/BitMatchTests/TransferRunResultsTests.swift @@ -0,0 +1,52 @@ +import Foundation +import Testing +@testable import BitMatch +import BitMatchEngine + +@MainActor +struct TransferRunResultsTests { + private func row(_ path: String = "/Card/clip.bin", destination: String? = "/Backup/Card/clip.bin", status: String = "copied") -> ResultRow { + ResultRow(path: path, status: status, size: 1, checksum: nil, destination: "Backup", destinationPath: destination) + } + + @Test func copyThenVerifyReplacesTheSameResultInArrivalOrder() { + let store = TransferRunResults() + store.upsert(row()) + store.upsert(row("/Card/second.bin")) + store.upsert(row(status: "verified")) + #expect(store.rows.map(\.path) == ["/Card/clip.bin", "/Card/second.bin"]) + #expect(store.rows.map(\.status) == ["verified", "copied"]) + } + + @Test func destinationsAreExactKeysIncludingNilEmptyAndCase() { + let store = TransferRunResults() + for destination in [nil, "", "/Backup/clip.bin", "/backup/clip.bin", "/Second/clip.bin"] as [String?] { + store.upsert(row(destination: destination)) + } + store.upsert(row(destination: nil, status: "verified")) + #expect(store.rows.count == 5) + #expect(store.rows.map(\.destinationPath) == [nil, "", "/Backup/clip.bin", "/backup/clip.bin", "/Second/clip.bin"]) + #expect(store.rows.map(\.status) == ["verified", "copied", "copied", "copied", "copied"]) + } + + @Test func replacementRebuildsTheIndexAndKeepsFirstDuplicateSemantics() { + let store = TransferRunResults() + store.upsert(row("/old")) + store.replace(with: [row("/second"), row(status: "first duplicate"), row(status: "second duplicate")]) + store.upsert(row(status: "verified")) + #expect(store.rows.map(\.path) == ["/second", "/Card/clip.bin", "/Card/clip.bin"]) + #expect(store.rows.map(\.status) == ["copied", "verified", "second duplicate"]) + store.upsert(row("/old", status: "appended")) + #expect(store.rows.count == 4) + #expect(store.rows.last?.status == "appended") + } + + @Test func clearingReplacementDropsEveryStaleIndex() { + let store = TransferRunResults() + store.upsert(row()) + store.replace(with: []) + store.upsert(row(status: "new attempt")) + #expect(store.rows.count == 1) + #expect(store.rows[0].status == "new attempt") + } +} diff --git a/Shared/Core/Services/SharedAppCoordinator.swift b/Shared/Core/Services/SharedAppCoordinator.swift index 1b21aeb7..be91c497 100644 --- a/Shared/Core/Services/SharedAppCoordinator.swift +++ b/Shared/Core/Services/SharedAppCoordinator.swift @@ -72,22 +72,34 @@ struct TransferRunContext { } @MainActor -private final class TransferRunResults { +final class TransferRunResults { private(set) var rows: [ResultRow] = [] + private struct Key: Hashable { + let path: String + let destinationPath: String? + } + private var indexByKey: [Key: Int] = [:] var attemptedPlanSave = false var planSaveError: Error? func upsert(_ row: ResultRow) { - if let index = rows.firstIndex(where: { - $0.path == row.path && $0.destinationPath == row.destinationPath - }) { + let key = Key(path: row.path, destinationPath: row.destinationPath) + if let index = indexByKey[key] { rows[index] = row } else { + indexByKey[key] = rows.count rows.append(row) } } func replace(with rows: [ResultRow]) { + var index: [Key: Int] = [:] + index.reserveCapacity(rows.count) + for (offset, row) in rows.enumerated() { + let key = Key(path: row.path, destinationPath: row.destinationPath) + if index[key] == nil { index[key] = offset } + } + indexByKey = index self.rows = rows } } diff --git a/docs/validation/RUN_RESULT_LOOKUP_20261020.md b/docs/validation/RUN_RESULT_LOOKUP_20261020.md new file mode 100644 index 00000000..8cb0b331 --- /dev/null +++ b/docs/validation/RUN_RESULT_LOOKUP_20261020.md @@ -0,0 +1,30 @@ +# Per-transfer result lookup + +The coordinator kept partial-attempt results by scanning the growing row array +for every copy and verification callback. For 20,000 files and two destinations, +that means 80,000 updates to 40,000 final rows and quadratic lookup work on the +main actor. + +The store now indexes the exact existing key: source path plus optional full +destination path. It preserves arrival order, replaces the first matching row, +and rebuilds the first-duplicate index when authoritative results replace the +list. Nil, empty and case-different paths remain distinct. No path normalization, +result replacement policy or verification behavior changed. + +Four normal regressions passed: copy-to-verification replacement and order; +destination/nil/empty/case distinctions; replacement with duplicate keys; and +clearing followed by reuse. They have no timing thresholds. + +An isolated optimized Swift executable used the exact production store and +`ResultRow` bodies with synthetic identifiers. The same 80,000 callbacks took +4.724–6.816 seconds with the original linear scans and 0.024–0.027 seconds with +the index. These observations were collected on a shared Mac alongside other +validation/build work. They are not app throughput benchmarks or a promised +speedup for transfers. The original body and compiled probe receipts were +retained privately. + +The live published feed was measured separately and remains unchanged. Its +80,000 updates took 33.176 seconds with a sampled peak process footprint of +about 44 MiB in the isolated executable. Published-array update cost remains a +separate performance question. Neither measurement demonstrates a memory leak, +establishes issue #23's cause, or proves the complete app has bounded memory. From 548a129f099a5a0ad1b1e76c6ea443db69d1bd6c Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 00:21:11 -0400 Subject: [PATCH 40/56] Paginate long Master Report fields without clipping --- .../MasterReportPaginationTests.swift | 108 +++++++++++++++++ .../MasterReportPaginationTests.swift | 108 +++++++++++++++++ .../SharedReportGenerationService.swift | 110 ++++++++++++------ 3 files changed, 290 insertions(+), 36 deletions(-) create mode 100644 BitMatch-iPadTests/MasterReportPaginationTests.swift create mode 100644 BitMatchTests/MasterReportPaginationTests.swift diff --git a/BitMatch-iPadTests/MasterReportPaginationTests.swift b/BitMatch-iPadTests/MasterReportPaginationTests.swift new file mode 100644 index 00000000..e2ab591f --- /dev/null +++ b/BitMatch-iPadTests/MasterReportPaginationTests.swift @@ -0,0 +1,108 @@ +import Foundation +import PDFKit +import Testing +@testable import BitMatch_iPad + +@MainActor +struct MasterReportPaginationTests { + @Test(arguments: ["production", "client", "company"]) + func oversizedMetadataPreservesAllTextAndSummary(field: String) async throws { + let value = String(repeating: "METADATAVALUE", count: 600) + " ENDMETADATA" + var configuration = SharedReportGenerationService.ReportConfiguration.default() + switch field { + case "production": configuration.production = value + case "client": configuration.client = value + default: configuration.company = value + } + let result = try await SharedReportGenerationService().generateMasterReport( + transfers: [transfer()], configuration: configuration) + let document = try #require(PDFDocument(data: result.pdfData)) + #expect(document.pageCount > 1) + #expect(compact(document.string ?? "").contains(compact(value))) + #expect(document.string?.contains("Total Transfers:") == true) + #expect(document.string?.contains("Unknown Camera") == true) + let json = try #require(JSONSerialization.jsonObject(with: result.jsonData) as? [String: Any]) + #expect(json[field] as? String == value) + try assertPrintable(document, configuration: configuration) + } + + @Test + func multilineNotesContinueWithoutDroppingTheLastLine() async throws { + var configuration = SharedReportGenerationService.ReportConfiguration.default() + configuration.productionNotes = (0..<200).map { "NOTE\($0) preserved production note" }.joined(separator: "\n") + "\nENDNOTES" + let result = try await SharedReportGenerationService().generateMasterReport( + transfers: [transfer()], configuration: configuration) + let document = try #require(PDFDocument(data: result.pdfData)) + #expect(document.pageCount > 2) + #expect(compact(document.string ?? "").contains(compact(configuration.productionNotes))) + #expect(document.string?.contains("Unknown Camera") == true) + let json = try #require(JSONSerialization.jsonObject(with: result.jsonData) as? [String: Any]) + #expect(json["productionNotes"] as? String == configuration.productionNotes) + try assertPrintable(document, configuration: configuration) + } + + @Test + func longUnicodeNotesPreserveComposedCharactersAcrossPages() async throws { + var configuration = SharedReportGenerationService.ReportConfiguration.default() + configuration.productionNotes = String(repeating: "cafe\u{301} カメラ ", count: 1_200) + "ENDNOTES" + let result = try await SharedReportGenerationService().generateMasterReport( + transfers: [transfer()], configuration: configuration) + let document = try #require(PDFDocument(data: result.pdfData)) + #expect(document.pageCount > 2) + // PDF font mappings may compose the accent; compare canonically equal + // text while requiring the JSON to preserve the exact original string. + #expect(compact(document.string ?? "").contains(compact(configuration.productionNotes))) + let json = try #require(JSONSerialization.jsonObject(with: result.jsonData) as? [String: Any]) + #expect(json["productionNotes"] as? String == configuration.productionNotes) + try assertPrintable(document, configuration: configuration) + } + + @Test + func aSingleOversizedTransferRowContinuesInsidePrintablePages() async throws { + // Inert model values deliberately exceed filesystem name limits to + // exercise a single wrapped row taller than one printable page. + let sourceName = String(repeating: "SOURCE", count: 600) + " ENDSOURCE" + let backupName = String(repeating: "BACKUP", count: 600) + " ENDBACKUP" + let configuration = SharedReportGenerationService.ReportConfiguration.default() + let result = try await SharedReportGenerationService().generateMasterReport( + transfers: [transfer(sourceName: sourceName, backupName: backupName)], configuration: configuration) + let document = try #require(PDFDocument(data: result.pdfData)) + #expect(document.pageCount > 2) + let text = compact(document.string ?? "") + // Once in the summary, then once per source / three backup row entries. + #expect(text.components(separatedBy: compact(sourceName)).count - 1 >= 2) + #expect(text.components(separatedBy: compact(backupName)).count - 1 >= 4) + try assertPrintable(document, configuration: configuration) + } + + private func transfer(sourceName: String = "Card", backupName: String = "Backup") -> TransferCard { + let source = FolderInfo(url: URL(fileURLWithPath: "/synthetic/" + sourceName), fileCount: 7, + totalSize: 1_000_000, lastModified: Date(), isInternalDrive: false) + let backups = (0..<3).map { index in + FolderInfo(url: URL(fileURLWithPath: "/synthetic/backup\(index)/" + backupName), fileCount: 7, + totalSize: 1_000_000, lastModified: Date(), isInternalDrive: false) + } + return TransferCard(source: source, destinations: backups, cameraCard: nil, metadata: nil, + progress: 1, state: .completed(.init(success: true, message: "Verified"))) + } + + private func compact(_ text: String) -> String { + text.precomposedStringWithCanonicalMapping.filter { !$0.isWhitespace } + } + + private func assertPrintable(_ document: PDFDocument, + configuration: SharedReportGenerationService.ReportConfiguration) throws { + for pageIndex in 0..= configuration.margins.left - 1) + #expect(bounds.maxX <= media.maxX - configuration.margins.right + 1) + #expect(bounds.minY >= configuration.margins.bottom - 1) + #expect(bounds.maxY <= media.maxY - configuration.margins.top + 1) + } + } + } +} diff --git a/BitMatchTests/MasterReportPaginationTests.swift b/BitMatchTests/MasterReportPaginationTests.swift new file mode 100644 index 00000000..1271aa48 --- /dev/null +++ b/BitMatchTests/MasterReportPaginationTests.swift @@ -0,0 +1,108 @@ +import Foundation +import PDFKit +import Testing +@testable import BitMatch + +@MainActor +struct MasterReportPaginationTests { + @Test(arguments: ["production", "client", "company"]) + func oversizedMetadataPreservesAllTextAndSummary(field: String) async throws { + let value = String(repeating: "METADATAVALUE", count: 600) + " ENDMETADATA" + var configuration = SharedReportGenerationService.ReportConfiguration.default() + switch field { + case "production": configuration.production = value + case "client": configuration.client = value + default: configuration.company = value + } + let result = try await SharedReportGenerationService().generateMasterReport( + transfers: [transfer()], configuration: configuration) + let document = try #require(PDFDocument(data: result.pdfData)) + #expect(document.pageCount > 1) + #expect(compact(document.string ?? "").contains(compact(value))) + #expect(document.string?.contains("Total Transfers:") == true) + #expect(document.string?.contains("Unknown Camera") == true) + let json = try #require(JSONSerialization.jsonObject(with: result.jsonData) as? [String: Any]) + #expect(json[field] as? String == value) + try assertPrintable(document, configuration: configuration) + } + + @Test + func multilineNotesContinueWithoutDroppingTheLastLine() async throws { + var configuration = SharedReportGenerationService.ReportConfiguration.default() + configuration.productionNotes = (0..<200).map { "NOTE\($0) preserved production note" }.joined(separator: "\n") + "\nENDNOTES" + let result = try await SharedReportGenerationService().generateMasterReport( + transfers: [transfer()], configuration: configuration) + let document = try #require(PDFDocument(data: result.pdfData)) + #expect(document.pageCount > 2) + #expect(compact(document.string ?? "").contains(compact(configuration.productionNotes))) + #expect(document.string?.contains("Unknown Camera") == true) + let json = try #require(JSONSerialization.jsonObject(with: result.jsonData) as? [String: Any]) + #expect(json["productionNotes"] as? String == configuration.productionNotes) + try assertPrintable(document, configuration: configuration) + } + + @Test + func longUnicodeNotesPreserveComposedCharactersAcrossPages() async throws { + var configuration = SharedReportGenerationService.ReportConfiguration.default() + configuration.productionNotes = String(repeating: "cafe\u{301} カメラ ", count: 1_200) + "ENDNOTES" + let result = try await SharedReportGenerationService().generateMasterReport( + transfers: [transfer()], configuration: configuration) + let document = try #require(PDFDocument(data: result.pdfData)) + #expect(document.pageCount > 2) + // PDF font mappings may compose the accent; compare canonically equal + // text while requiring the JSON to preserve the exact original string. + #expect(compact(document.string ?? "").contains(compact(configuration.productionNotes))) + let json = try #require(JSONSerialization.jsonObject(with: result.jsonData) as? [String: Any]) + #expect(json["productionNotes"] as? String == configuration.productionNotes) + try assertPrintable(document, configuration: configuration) + } + + @Test + func aSingleOversizedTransferRowContinuesInsidePrintablePages() async throws { + // Inert model values deliberately exceed filesystem name limits to + // exercise a single wrapped row taller than one printable page. + let sourceName = String(repeating: "SOURCE", count: 600) + " ENDSOURCE" + let backupName = String(repeating: "BACKUP", count: 600) + " ENDBACKUP" + let configuration = SharedReportGenerationService.ReportConfiguration.default() + let result = try await SharedReportGenerationService().generateMasterReport( + transfers: [transfer(sourceName: sourceName, backupName: backupName)], configuration: configuration) + let document = try #require(PDFDocument(data: result.pdfData)) + #expect(document.pageCount > 2) + let text = compact(document.string ?? "") + // Once in the summary, then once per source / three backup row entries. + #expect(text.components(separatedBy: compact(sourceName)).count - 1 >= 2) + #expect(text.components(separatedBy: compact(backupName)).count - 1 >= 4) + try assertPrintable(document, configuration: configuration) + } + + private func transfer(sourceName: String = "Card", backupName: String = "Backup") -> TransferCard { + let source = FolderInfo(url: URL(fileURLWithPath: "/synthetic/" + sourceName), fileCount: 7, + totalSize: 1_000_000, lastModified: Date(), isInternalDrive: false) + let backups = (0..<3).map { index in + FolderInfo(url: URL(fileURLWithPath: "/synthetic/backup\(index)/" + backupName), fileCount: 7, + totalSize: 1_000_000, lastModified: Date(), isInternalDrive: false) + } + return TransferCard(source: source, destinations: backups, cameraCard: nil, metadata: nil, + progress: 1, state: .completed(.init(success: true, message: "Verified"))) + } + + private func compact(_ text: String) -> String { + text.precomposedStringWithCanonicalMapping.filter { !$0.isWhitespace } + } + + private func assertPrintable(_ document: PDFDocument, + configuration: SharedReportGenerationService.ReportConfiguration) throws { + for pageIndex in 0..= configuration.margins.left - 1) + #expect(bounds.maxX <= media.maxX - configuration.margins.right + 1) + #expect(bounds.minY >= configuration.margins.bottom - 1) + #expect(bounds.maxY <= media.maxY - configuration.margins.top + 1) + } + } + } +} diff --git a/Shared/Core/Services/SharedReportGenerationService.swift b/Shared/Core/Services/SharedReportGenerationService.swift index bcdbbb79..5d449f89 100644 --- a/Shared/Core/Services/SharedReportGenerationService.swift +++ b/Shared/Core/Services/SharedReportGenerationService.swift @@ -188,21 +188,22 @@ class SharedReportGenerationService: ObservableObject { // Start first page setupNewPage() + let newPage = { + NSGraphicsContext.restoreGraphicsState() + context.endPDFPage() + setupNewPage() + } // Render sections using shared helpers - yPosition = renderHeader(in: contentRect, at: yPosition, reportData: reportData, configuration: configuration) - yPosition = renderSummary(in: contentRect, at: yPosition, reportData: reportData, configuration: configuration) + yPosition = renderHeader(in: contentRect, at: yPosition, reportData: reportData, configuration: configuration, newPage: newPage) + yPosition = renderSummary(in: contentRect, at: yPosition, reportData: reportData, configuration: configuration, newPage: newPage) if !configuration.productionNotes.isEmpty { - yPosition = renderProductionNotes(in: contentRect, at: yPosition, reportData: reportData, configuration: configuration) + yPosition = renderProductionNotes(in: contentRect, at: yPosition, reportData: reportData, configuration: configuration, newPage: newPage) } for cameraGroup in reportData.cameraGroups { - yPosition = renderCameraGroup(cameraGroup, in: contentRect, at: yPosition, configuration: configuration) { - NSGraphicsContext.restoreGraphicsState() - context.endPDFPage() - setupNewPage() - } + yPosition = renderCameraGroup(cameraGroup, in: contentRect, at: yPosition, configuration: configuration, newPage: newPage) } // End final page @@ -246,30 +247,29 @@ class SharedReportGenerationService: ObservableObject { height: max(0, pageRect.height - configuration.margins.top - configuration.margins.bottom) ) var yPosition: CGFloat = contentRect.minY + let newPage = { context.beginPage() } // Render header - yPosition = renderHeader(in: contentRect, at: yPosition, reportData: reportData, configuration: configuration) + yPosition = renderHeader(in: contentRect, at: yPosition, reportData: reportData, configuration: configuration, newPage: newPage) // Render summary - yPosition = renderSummary(in: contentRect, at: yPosition, reportData: reportData, configuration: configuration) + yPosition = renderSummary(in: contentRect, at: yPosition, reportData: reportData, configuration: configuration, newPage: newPage) // Render production notes if !configuration.productionNotes.isEmpty { - yPosition = renderProductionNotes(in: contentRect, at: yPosition, reportData: reportData, configuration: configuration) + yPosition = renderProductionNotes(in: contentRect, at: yPosition, reportData: reportData, configuration: configuration, newPage: newPage) } // Render camera groups for cameraGroup in reportData.cameraGroups { - yPosition = renderCameraGroup(cameraGroup, in: contentRect, at: yPosition, configuration: configuration) { - context.beginPage() - } + yPosition = renderCameraGroup(cameraGroup, in: contentRect, at: yPosition, configuration: configuration, newPage: newPage) } } #endif // MARK: - PDF Content Rendering (Shared Logic) - private func renderHeader(in rect: CGRect, at yPosition: CGFloat, reportData: ReportData, configuration: ReportConfiguration) -> CGFloat { + private func renderHeader(in rect: CGRect, at yPosition: CGFloat, reportData: ReportData, configuration: ReportConfiguration, newPage: () -> Void) -> CGFloat { var currentY = yPosition // Title @@ -297,14 +297,18 @@ class SharedReportGenerationService: ObservableObject { ].filter { !$0.hasSuffix(": ") } // Remove empty fields for line in metadata { - currentY += drawWrapped(line, in: rect, at: currentY, attributes: metadataAttributes) + 2 + currentY = drawPaginated(line, in: rect, at: currentY, attributes: metadataAttributes, newPage: newPage) + 2 } return currentY + 20 } - private func renderSummary(in rect: CGRect, at yPosition: CGFloat, reportData: ReportData, configuration: ReportConfiguration) -> CGFloat { + private func renderSummary(in rect: CGRect, at yPosition: CGFloat, reportData: ReportData, configuration: ReportConfiguration, newPage: () -> Void) -> CGFloat { var currentY = yPosition + if currentY + 35 + configuration.fontSize + 3 > rect.maxY { + newPage() + currentY = rect.minY + } // Summary section header let headerAttributes = [ @@ -372,14 +376,18 @@ class SharedReportGenerationService: ObservableObject { } for line in summaryLines { - currentY += drawWrapped(line, in: rect, at: currentY, attributes: summaryAttributes) + 2 + currentY = drawPaginated(line, in: rect, at: currentY, attributes: summaryAttributes, newPage: newPage) + 2 } return currentY + 20 } - private func renderProductionNotes(in rect: CGRect, at yPosition: CGFloat, reportData: ReportData, configuration: ReportConfiguration) -> CGFloat { + private func renderProductionNotes(in rect: CGRect, at yPosition: CGFloat, reportData: ReportData, configuration: ReportConfiguration, newPage: () -> Void) -> CGFloat { var currentY = yPosition + if currentY + 18 + configuration.fontSize + 3 > rect.maxY { + newPage() + currentY = rect.minY + } // Production notes header let headerAttributes = [ @@ -396,10 +404,8 @@ class SharedReportGenerationService: ObservableObject { NSAttributedString.Key.foregroundColor: platformColor(hex: "#000000") ] - let notesRect = CGRect(x: rect.minX, y: currentY, width: rect.width, height: 60) - configuration.productionNotes.draw(in: notesRect, withAttributes: notesAttributes) - - return currentY + 80 + currentY = drawPaginated(configuration.productionNotes, in: rect, at: currentY, attributes: notesAttributes, newPage: newPage) + return currentY + 20 } private func renderCameraGroup(_ group: ReportData.CameraGroup, in rect: CGRect, at yPosition: CGFloat, configuration: ReportConfiguration, newPage: () -> Void) -> CGFloat { @@ -417,7 +423,7 @@ class SharedReportGenerationService: ObservableObject { newPage() currentY = rect.minY } - currentY += drawWrapped(groupTitle, in: rect, at: currentY, attributes: headerAttributes) + 3 + currentY = drawPaginated(groupTitle, in: rect, at: currentY, attributes: headerAttributes, newPage: newPage) + 3 // Transfer details let detailAttributes = [ @@ -451,9 +457,9 @@ class SharedReportGenerationService: ObservableObject { if currentY + height > rect.maxY { newPage() currentY = rect.minY - currentY += drawWrapped(groupTitle + " (continued)", in: rect, at: currentY, attributes: headerAttributes) + 3 + currentY = drawPaginated(groupTitle + " (continued)", in: rect, at: currentY, attributes: headerAttributes, newPage: newPage) + 3 } - currentY += drawWrapped(transferLine, in: rect, at: currentY, attributes: detailAttributes) + 4 + currentY = drawPaginated(transferLine, in: rect, at: currentY, attributes: detailAttributes, newPage: newPage) + 4 } return currentY + 10 @@ -472,18 +478,50 @@ class SharedReportGenerationService: ObservableObject { } private func wrappedHeight(_ text: String, width: CGFloat, attributes: [NSAttributedString.Key: Any]) -> CGFloat { - ceil(wrappedText(text, attributes: attributes).boundingRect( - with: CGSize(width: width, height: .greatestFiniteMagnitude), - options: [.usesLineFragmentOrigin, .usesFontLeading], context: nil).height) + let storage = NSTextStorage(attributedString: wrappedText(text, attributes: attributes)) + let layout = NSLayoutManager() + let container = NSTextContainer(size: CGSize(width: width, height: .greatestFiniteMagnitude)) + container.lineFragmentPadding = 0 + layout.addTextContainer(container) + storage.addLayoutManager(layout) + layout.ensureLayout(for: container) + var height: CGFloat = 0 + layout.enumerateLineFragments(forGlyphRange: layout.glyphRange(for: container)) { line, _, _, _, _ in + height += ceil(line.height) + } + return height } - @discardableResult - private func drawWrapped(_ text: String, in rect: CGRect, at y: CGFloat, attributes: [NSAttributedString.Key: Any]) -> CGFloat { - let height = wrappedHeight(text, width: rect.width, attributes: attributes) - wrappedText(text, attributes: attributes).draw( - with: CGRect(x: rect.minX, y: y, width: rect.width, height: height), - options: [.usesLineFragmentOrigin, .usesFontLeading], context: nil) - return height + /// Lay out once and move whole line fragments to the next printable page. + /// Drawing the original composed-character ranges preserves every character + /// without truncating long fields. Use attributed drawing for each line so + /// color-emoji fallback also keeps its PDF text position on macOS. + private func drawPaginated(_ text: String, in rect: CGRect, at y: CGFloat, + attributes: [NSAttributedString.Key: Any], newPage: () -> Void) -> CGFloat { + let storage = NSTextStorage(attributedString: wrappedText(text, attributes: attributes)) + let layout = NSLayoutManager() + let container = NSTextContainer(size: rect.size) + container.size.height = .greatestFiniteMagnitude + container.lineFragmentPadding = 0 + layout.addTextContainer(container) + storage.addLayoutManager(layout) + layout.ensureLayout(for: container) + var lines: [(CGRect, NSRange)] = [] + layout.enumerateLineFragments(forGlyphRange: layout.glyphRange(for: container)) { line, _, _, glyphs, _ in + lines.append((line, glyphs)) + } + var currentY = y + for (line, glyphs) in lines { + let height = ceil(line.height) + if currentY + height > rect.maxY { + newPage() + currentY = rect.minY + } + let characters = layout.characterRange(forGlyphRange: glyphs, actualGlyphRange: nil) + storage.attributedSubstring(from: characters).draw(at: CGPoint(x: rect.minX, y: currentY)) + currentY += height + } + return currentY } // MARK: - JSON Generation From 383918a0d5f300afe888d9edeba0d797c602e0d3 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 00:28:43 -0400 Subject: [PATCH 41/56] Exercise additional 4K codecs and repeated PDF reports --- BitMatchTests/ReportPreviewMemoryTests.swift | 139 +++++++++++++----- .../report-preview-codecs-2026-10-20.md | 78 ++++++++++ 2 files changed, 184 insertions(+), 33 deletions(-) create mode 100644 docs/validation/report-preview-codecs-2026-10-20.md diff --git a/BitMatchTests/ReportPreviewMemoryTests.swift b/BitMatchTests/ReportPreviewMemoryTests.swift index 5b097a76..847f077a 100644 --- a/BitMatchTests/ReportPreviewMemoryTests.swift +++ b/BitMatchTests/ReportPreviewMemoryTests.swift @@ -40,7 +40,11 @@ final class ReportPreviewMemoryTests: XCTestCase { throw XCTSkip("Set BITMATCH_HIGH_RES_PREVIEW_MEMORY=1 for the guarded 20-clip 4K decoder probe") } let clipCount = Int(environment["BITMATCH_HIGH_RES_PREVIEW_CLIPS"] ?? "20") ?? -1 - guard (20...ReportThumbnailService.maximumPreviews).contains(clipCount) else { throw PreviewProbeError.invalidFixture } + let roundCount = Int(environment["BITMATCH_HIGH_RES_PREVIEW_ROUNDS"] ?? "1") ?? -1 + guard (20...ReportThumbnailService.maximumPreviews).contains(clipCount), (1...10).contains(roundCount), + let codec = PreviewProbeCodec(rawValue: environment["BITMATCH_HIGH_RES_PREVIEW_CODEC"] ?? "h264") else { + throw PreviewProbeError.invalidFixture + } let fm = FileManager.default let root = fm.temporaryDirectory.appendingPathComponent("bitmatch-4k-preview-\(UUID())") let artifactsBase = URL(fileURLWithPath: environment["BITMATCH_HIGH_RES_PREVIEW_ARTIFACTS"] ?? "/private/tmp") @@ -51,7 +55,7 @@ final class ReportPreviewMemoryTests: XCTestCase { let source = root.appendingPathComponent("Card") let backup = root.appendingPathComponent("Backup") for directory in [source, backup] { try fm.createDirectory(at: directory, withIntermediateDirectories: true) } - let memory = PreviewProbeMemory(clipCount: clipCount) + let memory = PreviewProbeMemory(clipCount: clipCount, codec: codec, roundCount: roundCount) _ = memory.sample() let pipeline = TransferPipeline(fileSystem: MacOSFileSystemService.shared, checksum: ChecksumEngine.shared) let worker = Task { @MainActor in @@ -71,16 +75,17 @@ final class ReportPreviewMemoryTests: XCTestCase { monitor.cancel(); await monitor.value // The monitor may have latched a failure after the prior check. try memory.requireSafe() + guard memory.snapshot().rounds.count == roundCount else { throw PreviewProbeError.invalidFixture } try save(memory, outcome: "passed", to: artifacts) try fm.removeItem(at: root) - print("PREVIEW_MEMORY_RESULT clips=\(clipCount) width=3840 height=2160 outcome=passed artifacts=\(artifacts.path)") + print("PREVIEW_MEMORY_RESULT clips=\(clipCount) codec=\(codec.rawValue) rounds=\(roundCount) width=3840 height=2160 outcome=passed artifacts=\(artifacts.path)") } catch { worker.cancel(); pipeline.cancelOperation() // run() awaits its owned seed task before escaping; collection // cancellation requests native AVFoundation cancellation as usual. monitor.cancel(); await monitor.value _ = memory.sample() - try? save(memory, outcome: memory.snapshot().guardTriggered ? "guarded" : "failed", to: artifacts) + try? save(memory, outcome: memory.snapshot().guardTriggered ? "guarded" : (error is XCTSkip ? "unsupported" : "failed"), to: artifacts) print("PREVIEW_MEMORY_FAILURE retainedFixture=\(root.path) artifacts=\(artifacts.path)") throw error } @@ -91,8 +96,11 @@ final class ReportPreviewMemoryTests: XCTestCase { memory.mark("fixture") try memory.requireSafe() let clipCount = memory.snapshot().sourceClips - let seed = Task.detached { try await Self.seed(source, clipCount: clipCount) } - let original = try await withTaskCancellationHandler { try await seed.value } onCancel: { seed.cancel() } + let codec = try XCTUnwrap(PreviewProbeCodec(rawValue: memory.snapshot().requestedCodec)) + let seed = Task.detached { try await Self.seed(source, clipCount: clipCount, codec: codec) } + let seeded = try await withTaskCancellationHandler { try await seed.value } onCancel: { seed.cancel() } + let original = seeded.manifest + memory.state.withLock { $0.actualFourCC = seeded.fourCC } try Task.checkCancellation() let encoder = JSONEncoder(); encoder.outputFormatting = [.prettyPrinted, .sortedKeys] try encoder.encode(original).write(to: artifacts.appendingPathComponent("source-sha256.json"), options: .atomic) @@ -105,7 +113,44 @@ final class ReportPreviewMemoryTests: XCTestCase { guard rows.count == clipCount, rows.allSatisfy(\.isVerifiedStatus) else { throw PreviewProbeError.incompleteVerification } memory.state.withLock { $0.verifiedRows = rows.count } try Task.checkCancellation() - memory.mark("decoding") + let roundCount = memory.snapshot().roundCount + for round in 0.. String = { roundCount == 1 ? $0 : String(format: "round-%02d.", round + 1) + $0 } + let roundArtifacts = roundCount == 1 ? artifacts : artifacts.appendingPathComponent(String(format: "round-%02d", round + 1)) + if roundCount > 1 { try FileManager.default.createDirectory(at: roundArtifacts, withIntermediateDirectories: false) } + try await collectAndRender(source: source, backup: backup, artifacts: roundArtifacts, original: original, + operation: operation, rows: rows, memory: memory, phase: phase) + memory.mark(phase("sourceAndDestinationChecks")) + let checked = Task.detached { + let after = try Self.hashes(source) + let copied = try Self.hashes(backup.appendingPathComponent(source.lastPathComponent), ignoreTransferStatus: true) + guard after == original, copied == original else { throw PreviewProbeError.changedMedia } + } + try await withTaskCancellationHandler { try await checked.value } onCancel: { checked.cancel() } + try Task.checkCancellation() + if roundCount > 1 { + memory.mark(phase("settling")) + try await Task.sleep(for: .seconds(2)) + } + try memory.requireSafe() + let measured = memory.snapshot() + memory.state.withLock { state in + state.rounds.append(.init(number: round + 1, settledMiB: measured.afterMiB, + decoderPeakMiB: measured.phases[phase("decoding")]?.peakMiB ?? 0, + pdfPeakMiB: measured.phases[phase("pdf")]?.peakMiB ?? 0, + previews: measured.collected, pdfPages: measured.pdfPages, + sourceHashesAfter: original.count, destinationHashesAfter: original.count)) + } + } + } + + /// All per-report images/native PDF objects leave scope before settling. + private func collectAndRender(source: URL, backup: URL, artifacts: URL, original: [String: String], + operation: FileOperation, rows: [ResultRow], memory: PreviewProbeMemory, + phase: (String) -> String) async throws { + let clipCount = original.count + memory.mark(phase("decoding")) try memory.requireSafe() let started = ProcessInfo.processInfo.systemUptime let thumbnails = try await ReportThumbnailService.collect(results: rows, enabled: true) @@ -118,7 +163,7 @@ final class ReportPreviewMemoryTests: XCTestCase { try data.write(to: artifacts.appendingPathComponent(String(format: "preview-%02d.jpg", index))) try Self.validateJPEG(data) } - memory.mark("pdf") + memory.mark(phase("pdf")) try memory.requireSafe() let summary = ReportSummary(jobID: UUID(), started: operation.startTime, finished: Date(), mode: .copyAndVerify, source: source.path, destinations: [backup.path], totalFiles: rows.count, matched: rows.count, issues: 0, @@ -127,24 +172,19 @@ final class ReportPreviewMemoryTests: XCTestCase { totalBytesProcessed: rows.reduce(0) { $0 + $1.size }, averageSpeed: 0, clientLogoData: nil, companyLogoData: nil, photographyJob: nil, operatorName: "BitMatch validation", notes: "Synthetic 4K preview memory probe", safetyState: .safeToErase) - let pdf = ReportPDFRenderer.renderPDF(summary: summary, results: rows, thumbnails: thumbnails) - // Preserve the generated bytes even if validation or the guard fails. - try pdf.write(to: artifacts.appendingPathComponent("verified-4k-previews.pdf")) - try memory.requireSafe() - guard let document = PDFDocument(data: pdf), let text = document.string, - text.contains("Clip previews: \(clipCount)/\(clipCount)"), text.contains("VERIFICATION SUCCESSFUL") else { throw PreviewProbeError.invalidPDF } - for name in original.keys { guard text.contains(name) else { throw PreviewProbeError.invalidPDF } } - let embedded = try Self.embeddedPreviews(pdf) - guard embedded > 0 else { throw PreviewProbeError.invalidPDF } - memory.state.withLock { $0.pdfPages = document.pageCount; $0.embeddedImageResources = embedded } - memory.mark("sourceAndDestinationChecks") - let checked = Task.detached { - let after = try Self.hashes(source) - let copied = try Self.hashes(backup.appendingPathComponent(source.lastPathComponent), ignoreTransferStatus: true) - guard after == original, copied == original else { throw PreviewProbeError.changedMedia } + let (pageCount, embedded) = try autoreleasepool { () throws -> (Int, Int) in + let pdf = ReportPDFRenderer.renderPDF(summary: summary, results: rows, thumbnails: thumbnails) + // Preserve the generated bytes even if validation or the guard fails. + try pdf.write(to: artifacts.appendingPathComponent("verified-4k-previews.pdf")) + try memory.requireSafe() + guard let document = PDFDocument(data: pdf), let text = document.string, + text.contains("Clip previews: \(clipCount)/\(clipCount)"), text.contains("VERIFICATION SUCCESSFUL") else { throw PreviewProbeError.invalidPDF } + for name in original.keys { guard text.contains(name) else { throw PreviewProbeError.invalidPDF } } + let embedded = try Self.embeddedPreviews(pdf) + guard embedded > 0 else { throw PreviewProbeError.invalidPDF } + return (document.pageCount, embedded) } - try await withTaskCancellationHandler { try await checked.value } onCancel: { checked.cancel() } - try Task.checkCancellation() + memory.state.withLock { $0.pdfPages = pageCount; $0.embeddedImageResources = embedded } } private func save(_ memory: PreviewProbeMemory, outcome: String, to artifacts: URL) throws { @@ -153,14 +193,22 @@ final class ReportPreviewMemoryTests: XCTestCase { try encoder.encode(snapshot).write(to: artifacts.appendingPathComponent("memory.json"), options: .atomic) } - nonisolated private static func seed(_ source: URL, clipCount: Int) async throws -> [String: String] { + nonisolated private static func seed(_ source: URL, clipCount: Int, codec: PreviewProbeCodec) async throws -> PreviewProbeSeed { let first = source.appendingPathComponent("clip-00.mov") let writer = try AVAssetWriter(outputURL: first, fileType: .mov) defer { if writer.status != .completed { writer.cancelWriting() } } - let input = AVAssetWriterInput(mediaType: .video, outputSettings: [AVVideoCodecKey: AVVideoCodecType.h264, - AVVideoWidthKey: 3_840, AVVideoHeightKey: 2_160, - AVVideoCompressionPropertiesKey: [AVVideoProfileLevelKey: AVVideoProfileLevelH264HighAutoLevel, - AVVideoAverageBitRateKey: 2_000_000, AVVideoAllowFrameReorderingKey: false]]) + var settings: [String: Any] = [AVVideoCodecKey: codec.videoType, AVVideoWidthKey: 3_840, AVVideoHeightKey: 2_160] + if codec == .h264 { + settings[AVVideoCompressionPropertiesKey] = [AVVideoProfileLevelKey: AVVideoProfileLevelH264HighAutoLevel, + AVVideoAverageBitRateKey: 2_000_000, AVVideoAllowFrameReorderingKey: false] + } else if codec == .hevc { + settings[AVVideoCompressionPropertiesKey] = [AVVideoAverageBitRateKey: 2_000_000, AVVideoAllowFrameReorderingKey: false] + } + guard writer.canApply(outputSettings: settings, forMediaType: .video) else { + throw XCTSkip("Requested 4K \(codec.rawValue) encoder is unsupported; no fallback") + } + let input = AVAssetWriterInput(mediaType: .video, outputSettings: settings) + guard writer.canAdd(input) else { throw XCTSkip("Cannot add requested \(codec.rawValue) encoder; no fallback") } let adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input, sourcePixelBufferAttributes: [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32ARGB]) writer.add(input) @@ -199,11 +247,17 @@ final class ReportPreviewMemoryTests: XCTestCase { guard let track = tracks.first, try await track.load(.naturalSize) == CGSize(width: 3_840, height: 2_160) else { throw PreviewProbeError.invalidFixture } + let descriptions = try await track.load(.formatDescriptions) + guard let description = descriptions.first else { throw PreviewProbeError.invalidFixture } + let type = CMFormatDescriptionGetMediaSubType(description) + let fourCC = String(bytes: [UInt8(truncatingIfNeeded: type >> 24), UInt8(truncatingIfNeeded: type >> 16), + UInt8(truncatingIfNeeded: type >> 8), UInt8(truncatingIfNeeded: type)], encoding: .ascii) ?? "" + guard codec.fourCCs.contains(fourCC) else { throw PreviewProbeError.invalidFixture } for index in 1.. [String: String] { @@ -279,6 +333,17 @@ final class ReportPreviewMemoryTests: XCTestCase { } } +private enum PreviewProbeCodec: String, Sendable { + case h264, hevc, prores422 + var videoType: AVVideoCodecType { + switch self { case .h264: .h264; case .hevc: .hevc; case .prores422: .proRes422 } + } + var fourCCs: Set { + switch self { case .h264: ["avc1"]; case .hevc: ["hvc1", "hev1"]; case .prores422: ["apcn"] } + } +} +private struct PreviewProbeSeed: Sendable { let manifest: [String: String]; let fourCC: String } + private final class PreviewProbeImages { var count = 0; var invalid = false } private enum PreviewProbeError: Error { case invalidFixture, incompleteVerification, incompletePreviews, invalidPreview, invalidPDF @@ -286,6 +351,10 @@ private enum PreviewProbeError: Error { } private final class PreviewProbeMemory: Sendable { struct Phase: Encodable, Sendable { var initialMiB: Int; var peakMiB: Int; var lastMiB: Int } + struct Round: Encodable, Sendable { + let number: Int; let settledMiB: Int; let decoderPeakMiB: Int; let pdfPeakMiB: Int + let previews: Int; let pdfPages: Int; let sourceHashesAfter: Int; let destinationHashesAfter: Int + } struct State: Encodable, Sendable { var outcome = "running" var phase = "initial" @@ -296,13 +365,17 @@ private final class PreviewProbeMemory: Sendable { var pdfPages = 0; var embeddedImageResources = 0 let guardMiB = 1_024 let sourceWidth = 3_840; let sourceHeight = 2_160; let sourceClips: Int + let requestedCodec: String; let roundCount: Int + var actualFourCC = ""; var rounds: [Round] = [] let maximumPreviews = ReportThumbnailService.maximumPreviews let collectionBudgetSeconds = ReportThumbnailService.maximumSeconds let perClipSeconds = ReportThumbnailService.perClipSeconds - let scope = "Synthetic host-only 4K H264 decoding/report probe. Sampled process physical footprint; not a reproduction or proof of issue #23. Native calls are not forcibly interrupted." + let scope = "Synthetic host-only 4K explicit-codec decoding/report probe. Sampled process physical footprint; not a reproduction or proof of issue #23. Native calls are not forcibly interrupted." } let state: Mutex - init(clipCount: Int = 20) { state = Mutex(State(sourceClips: clipCount)) } + init(clipCount: Int = 20, codec: PreviewProbeCodec = .h264, roundCount: Int = 1) { + state = Mutex(State(sourceClips: clipCount, requestedCodec: codec.rawValue, roundCount: roundCount)) + } func snapshot() -> State { state.withLock { $0 } } func requireSafe() throws { if sample() { throw PreviewProbeError.memoryGuard } diff --git a/docs/validation/report-preview-codecs-2026-10-20.md b/docs/validation/report-preview-codecs-2026-10-20.md new file mode 100644 index 00000000..7a10791f --- /dev/null +++ b/docs/validation/report-preview-codecs-2026-10-20.md @@ -0,0 +1,78 @@ +# Explicit-codec and repeated-report preview probe + +October 20, 2026. This extends the [4K preview memory probe](report-preview-memory-2026-10-19.md) +with two additional encoders and repeated reports in one process. It changes +test code only. All media lived in disposable Mac host scratch; no physical +card, backup drive, MHL handoff or durable History workflow is claimed here. + +Each codec used one original, locally generated two-frame 3840×2160 quadrant +pattern, copied to 20 distinct source paths. The fixture requires the requested +encoder, the actual track dimensions and the actual FourCC. It never falls back +to a different codec. Unsupported `canApply`/`canAdd` results are explicit skips +with an `unsupported` receipt; actual encoding failures remain failures. Both +additional codecs were supported on this Mac. + +| Probe | Actual FourCC | Report rounds | Initial / overall peak / final settled | Highest decoder / PDF phase | Probe test time | +| --- | --- | --- | --- | --- | --- | +| HEVC, 20 paths | `hvc1` | 1 | 46 / 103 / 83 MiB | 80 / 83 MiB | 3.388 s | +| ProRes 422, 20 paths | `apcn` | 1 | 45 / 143 / 85 MiB | 81 / 84 MiB | 3.819 s | +| H.264, 20 paths | `avc1` | 10 in the same process | 45 / 102 / 59 MiB | 93 / 93 MiB | 28.680 s | + +The normal invocation passed the two small guard tests and intentionally skipped +the opt-in probe. Each of the three opt-in invocations passed those two tests +and the real probe: four invocations, 11 passed test executions, one intentional +skip, zero failures and zero xcresult runtime warnings. The real probes produced +60 independently verified pipeline rows across three transfers, followed by +240 preview extractions and 12 PDF reports. Repeated H.264 rounds reuse the same +verified backup; they are not ten additional transfers. + +All 240 JPEGs passed dimension, byte-size and quadrant-pixel checks. Independent +Poppler inspection counted 20 JPEG placements in every PDF, each 256×144. All +24 pages were rendered; a contact sheet and representative full-size pages +were inspected. The reports retained every clip caption, verdict and footer. +Identical images may share a PDF resource, so resource count is not presented +as a count of distinct pictures. + +Source SHA-256 inventories were captured before copying: 60 files in total. +Every report round rechecked the source and backup inventories independently: +240 source hashes and 240 destination hashes afterward. Every comparison +matched. Successful scratch media was removed only after these checks and the +final latched memory guard passed. PDFs, JPEGs, logs and memory receipts remain +private because their generated location fields contain local paths. + +The ten-round run waits two seconds after each report's images and native PDF +objects leave scope. Its per-round physical-footprint samples were +83, 83, 83, 83, 83, 92, 92, 93, 59 and 59 MiB. The additional final settling +sample was 59 MiB. These observations show the measured trajectory; they do +not assert a leak-free trend or a permanent upper bound. Single-round probes +also include the original final two-second settling period. + +The original limits remain unchanged: 200 previews, 30 seconds per collection, +two seconds per clip and a sampled 1,024 MiB process-footprint guard. Sampling +was requested every 10 ms; brief peaks can be missed. No probe hit the guard or +lost a sample. Cancellation is cooperative; synchronous filesystem, decoder +and PDF calls cannot be forcibly preempted by this harness. Per-report native +PDF objects are scoped inside an autorelease pool, rather than retained by the +test across rounds. Fixture and hash tasks are joined before returning. + +This adds decoder diversity, not camera-format diversity: the three codecs all +encode the same tiny two-frame pattern. It does not cover long-GOP footage, +large encoded clips, every decoder, an interactive UI, or the cause of issue +#23. Timings are shared-host observations, not performance guarantees. + +With the Mac test host reserved, build once and select the existing test class: + +```sh +TEST_RUNNER_BITMATCH_HIGH_RES_PREVIEW_MEMORY=1 \ +TEST_RUNNER_BITMATCH_HIGH_RES_PREVIEW_CLIPS=20 \ +TEST_RUNNER_BITMATCH_HIGH_RES_PREVIEW_CODEC=hevc \ +TEST_RUNNER_BITMATCH_HIGH_RES_PREVIEW_ROUNDS=1 \ +TEST_RUNNER_BITMATCH_HIGH_RES_PREVIEW_ARTIFACTS=/private/tmp/bitmatch-preview-evidence \ +xcodebuild -project BitMatch.xcodeproj -scheme BitMatch \ + -destination 'platform=macOS' -jobs 4 -parallel-testing-enabled NO \ + -only-testing:BitMatchTests/ReportPreviewMemoryTests test +``` + +Codec accepts `h264` (default), `hevc` or `prores422`. Rounds accepts 1–10 +(default 1); clip count remains 20–200. Increase only after the smaller measured +case is safe. The checked-in defaults still skip the real probe. From 79f07486889e887c0fe8de808ff330931428d684 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 00:38:01 -0400 Subject: [PATCH 42/56] Isolate diagnostic assertions from concurrent app runs --- BitMatchTests/SharedCompareFlowTests.swift | 38 ++++---- .../Sources/BitMatchEngine/SharedLogger.swift | 27 +++--- .../ComparisonDiagnosticTests.swift | 41 ++++++--- .../DiagnosticStoreScopeTests.swift | 92 +++++++++++++++++++ .../TestHelpers/FileOperationsTestLock.swift | 12 ++- .../TransferDiagnosticStoreTests.swift | 34 ++++--- .../VerificationMemoryTests.swift | 3 +- 7 files changed, 190 insertions(+), 57 deletions(-) create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/DiagnosticStoreScopeTests.swift diff --git a/BitMatchTests/SharedCompareFlowTests.swift b/BitMatchTests/SharedCompareFlowTests.swift index 9e2dee81..817e8f9b 100644 --- a/BitMatchTests/SharedCompareFlowTests.swift +++ b/BitMatchTests/SharedCompareFlowTests.swift @@ -2,7 +2,7 @@ import Foundation import Testing @testable import BitMatch -import BitMatchEngine +@testable import BitMatchEngine struct SharedCompareFlowTests { @@ -512,6 +512,10 @@ struct SharedCompareFlowTests { /// `lastCompareEnd = .cancelled` from the `CancellationError` branch. @Test func testCancelledCompareRecordsCancelledOutcome() async throws { + let diagnosticRoot = FileManager.default.temporaryDirectory + .appendingPathComponent("bitmatch-cancel-compare-diagnostics-\(UUID())") + defer { try? FileManager.default.removeItem(at: diagnosticRoot) } + let diagnosticStore = TransferDiagnosticStore(directory: diagnosticRoot) let left = URL(fileURLWithPath: "/cancel-outcome/left") let right = URL(fileURLWithPath: "/cancel-outcome/right") let checksum = CancellingChecksumService() @@ -524,21 +528,23 @@ struct SharedCompareFlowTests { } checksum.onVerify = { await MainActor.run { - let run = coordinator.stateService.currentOperationId - coordinator.cancelOperation() - // Plant: log only activeStartID in cancelOperation. Compare has no - // copy start ID, so its user-cancel event would lose this run. - do { - let data = try TransferDiagnosticStore.shared.exportData() - let root = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) - let events = try #require(root["events"] as? [[String: Any]]) - let id = try #require(run) - #expect(events.contains { - $0["run"] as? String == id.uuidString - && $0["event"] as? String == "cancelOrigin" - && $0["origin"] as? String == "user" - }) - } catch { Issue.record(error) } + TransferDiagnostics.$recordingStoreOverride.withValue(diagnosticStore) { + let run = coordinator.stateService.currentOperationId + coordinator.cancelOperation() + // Plant: log only activeStartID in cancelOperation. Compare has no + // copy start ID, so its user-cancel event would lose this run. + do { + let data = try diagnosticStore.exportData() + let root = try #require(JSONSerialization.jsonObject(with: data) as? [String: Any]) + let events = try #require(root["events"] as? [[String: Any]]) + let id = try #require(run) + #expect(events.contains { + $0["run"] as? String == id.uuidString + && $0["event"] as? String == "cancelOrigin" + && $0["origin"] as? String == "user" + }) + } catch { Issue.record(error) } + } } } await MainActor.run { diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/SharedLogger.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/SharedLogger.swift index 624496bb..329c4444 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/SharedLogger.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/SharedLogger.swift @@ -51,7 +51,7 @@ extension SharedLogger { } public static func transferEvent(_ event: TransferEvent, run: UUID?, code: Int = 0, taskCancelled: Bool = false, explicit: Bool = false) { - TransferDiagnosticStore.shared.record(event, run: run) { + TransferDiagnostics.recordingStore.record(event, run: run) { $0.code = code; $0.taskCancelled = taskCancelled; $0.explicit = explicit } logger(for: .transfer).notice("event=\(event.rawValue, privacy: .public) run=\(run?.uuidString ?? "none", privacy: .public) code=\(code, privacy: .public) task_cancelled=\(taskCancelled, privacy: .public) explicit=\(explicit, privacy: .public)") @@ -63,11 +63,11 @@ extension SharedLogger { /// or bytes read so far, and the destination index. Never a name or path. public enum VerifyOutcome: String, Codable, Sendable { case matched, mismatched, failed, cancelled } public static func verifyConfiguration(run: UUID?, concurrency: Int) { - TransferDiagnosticStore.shared.record(.verifyConfiguration, run: run) { $0.verifyConcurrency = concurrency } + TransferDiagnostics.recordingStore.record(.verifyConfiguration, run: run) { $0.verifyConcurrency = concurrency } } public static func verifyEvent(_ event: TransferEvent, run: UUID?, ordinal: Int?, destinationIndex: Int, bytes: Int64?, totalBytes: Int64? = nil, outcome: VerifyOutcome? = nil) { - TransferDiagnosticStore.shared.record(event, run: run) { + TransferDiagnostics.recordingStore.record(event, run: run) { $0.ordinal = ordinal; $0.destinationIndex = destinationIndex; $0.bytesProcessed = bytes; $0.totalBytes = totalBytes; $0.verifyOutcome = outcome } logger(for: .transfer).notice("event=\(event.rawValue, privacy: .public) run=\(run?.uuidString ?? "none", privacy: .public) ordinal=\(ordinal ?? -1, privacy: .public) destination=\(destinationIndex, privacy: .public) bytes=\(bytes ?? -1, privacy: .public) outcome=\(outcome?.rawValue ?? "pending", privacy: .public)") @@ -76,7 +76,7 @@ extension SharedLogger { extension SharedLogger { public static func transferPhase(_ phase: ProgressStage, run: UUID?) { - TransferDiagnosticStore.shared.record(.phase, run: run) { $0.phase = phase } + TransferDiagnostics.recordingStore.record(.phase, run: run) { $0.phase = phase } logger(for: .transfer).notice("event=phase run=\(run?.uuidString ?? "none", privacy: .public) phase=\(phase.displayName, privacy: .public)") } } @@ -91,7 +91,7 @@ public enum TransferCancelOrigin: String, Codable, Sendable { extension SharedLogger { public static func cancelEvent(_ origin: TransferCancelOrigin, run: UUID?) { - TransferDiagnosticStore.shared.record(.cancelOrigin, run: run) { $0.origin = origin } + TransferDiagnostics.recordingStore.record(.cancelOrigin, run: run) { $0.origin = origin } logger(for: .transfer).notice("event=explicit_cancel run=\(run?.uuidString ?? "none", privacy: .public) origin=\(origin.rawValue, privacy: .public)") } } @@ -103,13 +103,13 @@ extension SharedLogger { // expose only this fixed classification, never localized descriptions. let kind: TransferDiagnosticStore.ErrorKind = error is CancellationError ? .cancellation : error is TransferInterruption ? .identityGuard : ns.domain == NSPOSIXErrorDomain ? .posix : ns.domain == NSCocoaErrorDomain ? .cocoa : .other - TransferDiagnosticStore.shared.record(.error, run: run) { + TransferDiagnostics.recordingStore.record(.error, run: run) { $0.kind = kind; $0.code = ns.code; $0.taskCancelled = Task.isCancelled; $0.explicit = explicit } logger(for: .transfer).error("event=error run=\(run?.uuidString ?? "none", privacy: .public) kind=\(kind.rawValue, privacy: .public) code=\(ns.code, privacy: .public) task_cancelled=\(Task.isCancelled, privacy: .public) explicit=\(explicit, privacy: .public)") } public static func correlatePipeline(run: UUID, pipeline: UUID) { - TransferDiagnosticStore.shared.record(.pipelineResult, run: run) { $0.pipeline = pipeline } + TransferDiagnostics.recordingStore.record(.pipelineResult, run: run) { $0.pipeline = pipeline } logger(for: .transfer).notice("event=pipeline_result run=\(run.uuidString, privacy: .public) pipeline=\(pipeline.uuidString, privacy: .public)") } public static func transferConfiguration(run: UUID, source: URL, destinations: [URL], mhl: Bool, report: Bool, mode: VerificationMode? = nil) { @@ -122,11 +122,11 @@ extension SharedLogger { return TransferDiagnosticStore.Filesystem(rawValue: value) ?? .other } let sourceType = fsType(source) - TransferDiagnosticStore.shared.record(.config, run: run) { $0.filesystem = sourceType; $0.mhl = mhl; $0.report = report; $0.verificationMode = mode } + TransferDiagnostics.recordingStore.record(.config, run: run) { $0.filesystem = sourceType; $0.mhl = mhl; $0.report = report; $0.verificationMode = mode } logger(for: .transfer).notice("event=config run=\(run.uuidString, privacy: .public) source_fs=\(sourceType.rawValue, privacy: .public) mhl=\(mhl, privacy: .public) report=\(report, privacy: .public)") for (index, url) in destinations.enumerated() { let type = fsType(url) - TransferDiagnosticStore.shared.record(.destination, run: run) { $0.destinationIndex = index; $0.filesystem = type } + TransferDiagnostics.recordingStore.record(.destination, run: run) { $0.destinationIndex = index; $0.filesystem = type } logger(for: .transfer).notice("event=destination run=\(run.uuidString, privacy: .public) index=\(index, privacy: .public) fs=\(type.rawValue, privacy: .public)") } } @@ -135,6 +135,11 @@ extension SharedLogger { /// Inherited by the engine's scoped/unstructured run tasks; detached handoff /// tasks receive the same UUID explicitly. Never carries source metadata. public enum TransferDiagnostics { + /// Internal isolation seam for deterministic engine tests. Production uses + /// the shared bounded store; scoped child tasks inherit an override, while + /// detached tasks and separate processes must not be assumed to inherit it. + @TaskLocal static var recordingStoreOverride: TransferDiagnosticStore? + static var recordingStore: TransferDiagnosticStore { recordingStoreOverride ?? .shared } @TaskLocal public static var runID: UUID? /// The pipelined verify job running in this task, for read breadcrumbs. @TaskLocal public static var verifyOrdinal: Int? @@ -143,7 +148,7 @@ public enum TransferDiagnostics { extension SharedLogger { /// Select only numerical progress. `currentFile` is deliberately excluded. public static func transferProgress(_ progress: OperationProgress, run: UUID?) { - TransferDiagnosticStore.shared.record(.progress, run: run) { + TransferDiagnostics.recordingStore.record(.progress, run: run) { $0.phase = progress.currentStage $0.filesProcessed = progress.filesProcessed; $0.totalFiles = progress.totalFiles $0.bytesProcessed = progress.bytesProcessed; $0.totalBytes = progress.totalBytes @@ -165,7 +170,7 @@ extension SharedLogger { mode: VerificationMode? = nil, phase: ComparisonPhase? = nil, outcome: ComparisonOutcome? = nil, stats: CompareStats? = nil, checked: Int? = nil, total: Int? = nil) { - TransferDiagnosticStore.shared.record(event, run: run) { + TransferDiagnostics.recordingStore.record(event, run: run) { $0.verificationMode = mode; $0.comparisonPhase = phase; $0.comparisonOutcome = outcome $0.filesProcessed = checked; $0.totalFiles = total; $0.taskCancelled = Task.isCancelled $0.onlyInSourceCount = stats?.onlyInLeftCount diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ComparisonDiagnosticTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ComparisonDiagnosticTests.swift index 218e9b37..4e562f6d 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ComparisonDiagnosticTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ComparisonDiagnosticTests.swift @@ -3,8 +3,8 @@ import XCTest @testable import BitMatchEngine final class ComparisonDiagnosticTests: XCTestCase { - private func events(_ run: UUID) throws -> [[String: Any]] { - let data = try TransferDiagnosticStore.shared.exportData() + private func events(_ run: UUID, store: TransferDiagnosticStore) throws -> [[String: Any]] { + let data = try store.exportData() let root = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any]) return try XCTUnwrap(root["events"] as? [[String: Any]]).filter { $0["run"] as? String == run.uuidString } } @@ -20,12 +20,15 @@ final class ComparisonDiagnosticTests: XCTestCase { try Data().write(to: left.appendingPathComponent("missing")) try Data().write(to: right.appendingPathComponent("extra")) let run = UUID() - let result = try await TransferDiagnostics.$runID.withValue(run) { - try await FolderComparer(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) - .compare(left: left, right: right, verificationMode: .standard, progress: { _ in }) + let store = TransferDiagnosticStore(directory: root.appendingPathComponent("Diagnostics")) + let result = try await TransferDiagnostics.$recordingStoreOverride.withValue(store) { + try await TransferDiagnostics.$runID.withValue(run) { + try await FolderComparer(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + .compare(left: left, right: right, verificationMode: .standard, progress: { _ in }) + } } XCTAssertEqual(result.mismatchedCount, 1) - let records = try events(run) + let records = try events(run, store: store) XCTAssertEqual(records.first?["event"] as? String, "compareStarted") XCTAssertEqual(records.last?["comparisonOutcome"] as? String, "completed") XCTAssertEqual(records.last?["onlyInSourceCount"] as? Int, 1) @@ -36,14 +39,19 @@ final class ComparisonDiagnosticTests: XCTestCase { func testReadFailureRetainsPhaseAndDoesNotLogSuccessfulCompletion() async throws { let run = UUID() let marker = "PRIVATE_PATH_\(UUID())" + let diagnosticRoot = FileManager.default.temporaryDirectory.appendingPathComponent("bitmatch-comparison-diagnostics-\(UUID())") + defer { try? FileManager.default.removeItem(at: diagnosticRoot) } + let store = TransferDiagnosticStore(directory: diagnosticRoot) do { - _ = try await TransferDiagnostics.$runID.withValue(run) { - try await FolderComparer(fileAccess: FailingListing(), checksum: ChecksumEngine.shared) - .compare(left: URL(fileURLWithPath: "/\(marker)"), right: URL(fileURLWithPath: "/backup"), verificationMode: .standard, progress: { _ in }) + _ = try await TransferDiagnostics.$recordingStoreOverride.withValue(store) { + try await TransferDiagnostics.$runID.withValue(run) { + try await FolderComparer(fileAccess: FailingListing(), checksum: ChecksumEngine.shared) + .compare(left: URL(fileURLWithPath: "/\(marker)"), right: URL(fileURLWithPath: "/backup"), verificationMode: .standard, progress: { _ in }) + } } XCTFail("Expected unreadable source") } catch { XCTAssertEqual((error as NSError).code, 13) } - let records = try events(run) + let records = try events(run, store: store) XCTAssertEqual(records.first { $0["event"] as? String == "comparePhase" }?["comparisonPhase"] as? String, "listingSource") XCTAssertEqual(records.last?["comparisonOutcome"] as? String, "failed") XCTAssertNil(records.last?["matchingCount"]) @@ -51,15 +59,20 @@ final class ComparisonDiagnosticTests: XCTestCase { } func testCancellationStillThrowsAndHasNoCompletedSummary() async throws { let run = UUID() + let diagnosticRoot = FileManager.default.temporaryDirectory.appendingPathComponent("bitmatch-comparison-diagnostics-\(UUID())") + defer { try? FileManager.default.removeItem(at: diagnosticRoot) } + let store = TransferDiagnosticStore(directory: diagnosticRoot) let task = Task { - try await TransferDiagnostics.$runID.withValue(run) { - try await FolderComparer(fileAccess: CancelledListing(), checksum: ChecksumEngine.shared) - .compare(left: URL(fileURLWithPath: "/source"), right: URL(fileURLWithPath: "/backup"), verificationMode: .standard, progress: { _ in }) + try await TransferDiagnostics.$recordingStoreOverride.withValue(store) { + try await TransferDiagnostics.$runID.withValue(run) { + try await FolderComparer(fileAccess: CancelledListing(), checksum: ChecksumEngine.shared) + .compare(left: URL(fileURLWithPath: "/source"), right: URL(fileURLWithPath: "/backup"), verificationMode: .standard, progress: { _ in }) + } } } do { _ = try await task.value; XCTFail("Cancellation must propagate") } catch { XCTAssertTrue(error is CancellationError) } - XCTAssertEqual(try events(run).last?["comparisonOutcome"] as? String, "cancelled") + XCTAssertEqual(try events(run, store: store).last?["comparisonOutcome"] as? String, "cancelled") } } private final class FailingListing: FakeFileAccess, @unchecked Sendable { diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DiagnosticStoreScopeTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DiagnosticStoreScopeTests.swift new file mode 100644 index 00000000..284ce738 --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DiagnosticStoreScopeTests.swift @@ -0,0 +1,92 @@ +import Foundation +import Testing +@testable import BitMatchEngine + +struct DiagnosticStoreScopeTests { + @Test func unstructuredAndGroupChildrenInheritTheRecordingStore() async throws { + let root = directory() + defer { try? FileManager.default.removeItem(at: root) } + let store = TransferDiagnosticStore(directory: root) + let run = UUID() + await TransferDiagnostics.$recordingStoreOverride.withValue(store) { + SharedLogger.verifyEvent(.verifyStarted, run: run, ordinal: 1, destinationIndex: 0, bytes: 0) + let child = Task { + SharedLogger.verifyEvent(.verifyRead, run: run, ordinal: 1, destinationIndex: 0, bytes: 4) + } + await child.value + await withTaskGroup(of: Void.self) { group in + group.addTask { + SharedLogger.verifyEvent(.verifyDigestFinished, run: run, ordinal: 1, destinationIndex: 0, bytes: 4) + } + await group.waitForAll() + } + SharedLogger.verifyEvent(.verifyFinished, run: run, ordinal: 1, destinationIndex: 0, bytes: 4, outcome: .matched) + } + let records = try events(store) + #expect(records.compactMap { $0["event"] as? String } == ["verifyStarted", "verifyRead", "verifyDigestFinished", "verifyFinished"]) + #expect(records.allSatisfy { $0["run"] as? String == run.uuidString }) + #expect(TransferDiagnostics.recordingStoreOverride == nil) + } + + @Test func nestedThrowingScopeRestoresItsParentAndFixturePreservesIt() async throws { + let root = directory() + defer { try? FileManager.default.removeItem(at: root) } + let outer = TransferDiagnosticStore(directory: root.appendingPathComponent("outer")) + let inner = TransferDiagnosticStore(directory: root.appendingPathComponent("inner")) + await TransferDiagnostics.$recordingStoreOverride.withValue(outer) { + SharedLogger.transferEvent(.admitted, run: nil, code: 1) + do { + try TransferDiagnostics.$recordingStoreOverride.withValue(inner) { + SharedLogger.transferEvent(.phaseChanged, run: nil, code: 2) + throw ScopeProbe.expected + } + Issue.record("Nested scope must throw") + } catch { #expect(error is ScopeProbe) } + await FileOperationsTestLock.shared.run { + #expect(TransferDiagnostics.recordingStore === outer) + SharedLogger.transferEvent(.pipelineDrained, run: nil, code: 3) + } + } + #expect(try events(outer).compactMap { $0["code"] as? Int } == [1, 3]) + #expect(try events(inner).compactMap { $0["code"] as? Int } == [2]) + #expect(TransferDiagnostics.recordingStoreOverride == nil) + } + + @Test func concurrentScopesKeepEveryEventInTheirOwnFiles() async throws { + let root = directory() + defer { try? FileManager.default.removeItem(at: root) } + let stores = ["first", "second"].map { TransferDiagnosticStore(directory: root.appendingPathComponent($0)) } + let runs = [UUID(), UUID()] + await withTaskGroup(of: Void.self) { group in + for index in stores.indices { + group.addTask { + await TransferDiagnostics.$recordingStoreOverride.withValue(stores[index]) { + for ordinal in 0..<200 { + await Task.yield() + SharedLogger.verifyEvent(.verifyRead, run: runs[index], ordinal: ordinal, + destinationIndex: index, bytes: Int64(ordinal)) + } + } + } + } + await group.waitForAll() + } + for index in stores.indices { + let records = try events(stores[index]) + #expect(records.count == 200) + #expect(records.allSatisfy { $0["run"] as? String == runs[index].uuidString }) + #expect(Set(records.compactMap { $0["ordinal"] as? Int }) == Set(0..<200)) + } + #expect(TransferDiagnostics.recordingStoreOverride == nil) + } + + private func directory() -> URL { + FileManager.default.temporaryDirectory.appendingPathComponent("bitmatch-diagnostic-scope-\(UUID())") + } + private func events(_ store: TransferDiagnosticStore) throws -> [[String: Any]] { + let object = try #require(JSONSerialization.jsonObject(with: store.exportData()) as? [String: Any]) + #expect(object["recordingHadErrors"] as? Bool == false) + return try #require(object["events"] as? [[String: Any]]) + } +} +private enum ScopeProbe: Error { case expected } diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TestHelpers/FileOperationsTestLock.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TestHelpers/FileOperationsTestLock.swift index f20cdce6..17f18d10 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TestHelpers/FileOperationsTestLock.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TestHelpers/FileOperationsTestLock.swift @@ -1,9 +1,19 @@ import Foundation +@testable import BitMatchEngine actor FileOperationsTestLock { static let shared = FileOperationsTestLock() func run(_ operation: () async throws -> T) async rethrows -> T { - try await operation() + // Ordinary synthetic transfers must not write into the user's real + // diagnostic ring. Preserve any explicitly supplied outer test store. + if TransferDiagnostics.recordingStoreOverride != nil { return try await operation() } + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("bitmatch-engine-diagnostics-\(UUID())") + let store = TransferDiagnosticStore(directory: directory) + defer { try? FileManager.default.removeItem(at: directory) } + return try await TransferDiagnostics.$recordingStoreOverride.withValue(store) { + try await operation() + } } } diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TransferDiagnosticStoreTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TransferDiagnosticStoreTests.swift index ed61bdba..7dfb8d28 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TransferDiagnosticStoreTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/TransferDiagnosticStoreTests.swift @@ -199,17 +199,20 @@ final class TransferDiagnosticStoreTests: XCTestCase { func testLoggerExcludesFootageNamesPathsAndErrorDescriptions() throws { let marker = "SECRET_FOOTAGE_" + UUID().uuidString let run = UUID() - SharedLogger.transferError(NSError(domain: marker, code: 45, - userInfo: [NSLocalizedDescriptionKey: marker]), run: run) - SharedLogger.transferConfiguration(run: run, - source: URL(fileURLWithPath: "/" + marker), - destinations: [URL(fileURLWithPath: "/" + marker + "/backup")], - mhl: true, report: true, mode: .standard) - SharedLogger.transferProgress(OperationProgress(overallProgress: 0.5, - currentFile: marker, filesProcessed: 2, totalFiles: 4, - currentStage: .verifying, speed: nil), run: run) - SharedLogger.info(marker) // Free-form logs must never enter the export. - let data = try TransferDiagnosticStore.shared.exportData() + let store = TransferDiagnosticStore(directory: try fixture()) + TransferDiagnostics.$recordingStoreOverride.withValue(store) { + SharedLogger.transferError(NSError(domain: marker, code: 45, + userInfo: [NSLocalizedDescriptionKey: marker]), run: run) + SharedLogger.transferConfiguration(run: run, + source: URL(fileURLWithPath: "/" + marker), + destinations: [URL(fileURLWithPath: "/" + marker + "/backup")], + mhl: true, report: true, mode: .standard) + SharedLogger.transferProgress(OperationProgress(overallProgress: 0.5, + currentFile: marker, filesProcessed: 2, totalFiles: 4, + currentStage: .verifying, speed: nil), run: run) + SharedLogger.info(marker) // Free-form logs must never enter the export. + } + let data = try store.exportData() XCTAssertFalse(String(decoding: data, as: UTF8.self).contains(marker)) let object = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any]) let events = try XCTUnwrap(object["events"] as? [[String: Any]]) @@ -220,9 +223,12 @@ final class TransferDiagnosticStoreTests: XCTestCase { } func testVerifyBreadcrumbsCarryNumbersAndMemoryFootprintOnly() throws { let run = UUID() - SharedLogger.verifyEvent(.verifyStarted, run: run, ordinal: 8, destinationIndex: 0, bytes: 45_344_117_808) - SharedLogger.verifyEvent(.verifyRead, run: run, ordinal: 8, destinationIndex: 0, bytes: 8 * 1024 * 1024 * 1024) - let object = try XCTUnwrap(JSONSerialization.jsonObject(with: TransferDiagnosticStore.shared.exportData()) as? [String: Any]) + let store = TransferDiagnosticStore(directory: try fixture()) + TransferDiagnostics.$recordingStoreOverride.withValue(store) { + SharedLogger.verifyEvent(.verifyStarted, run: run, ordinal: 8, destinationIndex: 0, bytes: 45_344_117_808) + SharedLogger.verifyEvent(.verifyRead, run: run, ordinal: 8, destinationIndex: 0, bytes: 8 * 1024 * 1024 * 1024) + } + let object = try XCTUnwrap(JSONSerialization.jsonObject(with: store.exportData()) as? [String: Any]) let events = try XCTUnwrap(object["events"] as? [[String: Any]]) .filter { ($0["run"] as? String) == run.uuidString } XCTAssertEqual(events.map { $0["event"] as? String }, ["verifyStarted", "verifyRead"]) diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/VerificationMemoryTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/VerificationMemoryTests.swift index 315395a7..c4bddb68 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/VerificationMemoryTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/VerificationMemoryTests.swift @@ -111,6 +111,7 @@ struct VerificationMemoryTests { @Test func verifyBreadcrumbsHaveConsistentRunAndTerminalOutcomes() async throws { try await FileOperationsTestLock.shared.run { + let diagnosticStore = TransferDiagnostics.recordingStore for pipelined in [true, false] { for outcome in ["matched", "failed", "mismatched", "cancelled"] { let cancelled = outcome == "cancelled" @@ -142,7 +143,7 @@ struct VerificationMemoryTests { #expect(!cancelled) #expect(operation.results.allSatisfy { $0.success == (outcome == "matched") }) } catch is CancellationError { #expect(cancelled) } - let object = try #require(JSONSerialization.jsonObject(with: TransferDiagnosticStore.shared.exportData()) as? [String: Any]) + let object = try #require(JSONSerialization.jsonObject(with: diagnosticStore.exportData()) as? [String: Any]) let events = try #require(object["events"] as? [[String: Any]]).filter { $0["run"] as? String == run.uuidString } #expect(events.first { $0["event"] as? String == "verifyConfiguration" }?["verifyConcurrency"] as? Int == 1) let verify = events.filter { ($0["event"] as? String)?.hasPrefix("verify") == true && $0["event"] as? String != "verifyConfiguration" } From 039e64e4806b31c0a13b19bd69e3202e3d2f6747 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 00:43:14 -0400 Subject: [PATCH 43/56] Exercise live coordinator memory with large file inventories --- .../CoordinatorManyFilesMemoryTests.swift | 357 ++++++++++++++++++ .../COORDINATOR_MANY_FILES_MEMORY_20261020.md | 117 ++++++ 2 files changed, 474 insertions(+) create mode 100644 BitMatchTests/CoordinatorManyFilesMemoryTests.swift create mode 100644 docs/validation/COORDINATOR_MANY_FILES_MEMORY_20261020.md diff --git a/BitMatchTests/CoordinatorManyFilesMemoryTests.swift b/BitMatchTests/CoordinatorManyFilesMemoryTests.swift new file mode 100644 index 00000000..2154a88a --- /dev/null +++ b/BitMatchTests/CoordinatorManyFilesMemoryTests.swift @@ -0,0 +1,357 @@ +#if os(macOS) +import AppKit +import Combine +import CryptoKit +import Darwin +import Foundation +import PDFKit +import SwiftUI +import Synchronization +import XCTest +@testable import BitMatch +@testable import BitMatchEngine + +/// Real coordinator, pipeline and observed SwiftUI views; host scratch only. +/// Row/history/PDF storage is O(N), not a constant-memory promise or #23 repro. +@MainActor +final class CoordinatorManyFilesMemoryTests: XCTestCase { + func testMemoryGuardCancelsOwnedWork() async throws { + let guardrail = ManyFilesMemory() + XCTAssertTrue(guardrail.record(1_024)) + let worker = Task { try await Task.sleep(for: .seconds(60)) } + let monitor = guardrail.monitor { worker.cancel() } + do { try await worker.value; XCTFail("Guard did not cancel owned work") } + catch { XCTAssertTrue(error is CancellationError) } + monitor.cancel(); await monitor.value + XCTAssertThrowsError(try guardrail.requireSafe()) + } + + func testMissingMemorySampleFailsClosed() { + let guardrail = ManyFilesMemory() + XCTAssertTrue(guardrail.record(nil)) + XCTAssertThrowsError(try guardrail.requireSafe()) + } + + func testActualCoordinatorManyFilesWithLiveViews() async throws { + let env = ProcessInfo.processInfo.environment + guard env["BITMATCH_APP_MANY_FILES_MEMORY"] == "1" else { + throw XCTSkip("Set BITMATCH_APP_MANY_FILES_MEMORY=1 for the guarded coordinator probe") + } + let count = Int(env["BITMATCH_APP_MANY_FILES_COUNT"] ?? "20000") ?? 0 + let reports = env["BITMATCH_APP_MANY_FILES_REPORTS"] ?? "off" + let maxSeconds = Double(env["BITMATCH_APP_MANY_FILES_MAX_SECONDS"] ?? "600") ?? 0 + guard (100...20000).contains(count), ["off", "both"].contains(reports), + maxSeconds.isFinite, (60...1200).contains(maxSeconds) else { throw ManyFilesError.invalidFixture } + try FileHandle.standardOutput.write(contentsOf: Data("APP_MEMORY workerPID=\(getpid()) files=\(count) reports=\(reports)\n".utf8)) + let root = URL(fileURLWithPath: "/private/tmp/bitmatch-app-many-files-\(UUID())", isDirectory: true) + let artifacts = root.appendingPathComponent("evidence") + let source = root.appendingPathComponent("Card") + let fm = FileManager.default + try fm.createDirectory(at: artifacts, withIntermediateDirectories: true) + try fm.createDirectory(at: source, withIntermediateDirectories: true) + try Data("owned many-files app fixture".utf8).write(to: root.appendingPathComponent(".bitmatch-owned"), options: .withoutOverwriting) + let suite = "BitMatchTests.ManyFiles.\(UUID())" + let defaults = try XCTUnwrap(UserDefaults(suiteName: suite)) + defer { defaults.removePersistentDomain(forName: suite) } + defaults.set(true, forKey: GeneralSettings.notificationPromptAskedKey) + defaults.set(false, forKey: GeneralSettings.notifyFinishKey) + defaults.set(false, forKey: GeneralSettings.notifyAttentionKey) + let memory = ManyFilesMemory(maxSeconds: maxSeconds) + let pipeline = TransferPipeline(fileSystem: MacOSFileSystemService.shared, checksum: ChecksumEngine.shared) + let coordinator = SharedAppCoordinator(platformManager: RecordingPlatformManager(fileOperations: pipeline), + transferJournal: LocalTransferJournal(fileURL: artifacts.appendingPathComponent("history.json")), + projectStore: InMemoryPhotographerJobStore(), defaults: defaults, + physicalDiskIdentityProvider: ManyFilesAbstractDisks()) + let renders = ManyFilesRenders() + let host = NSHostingView(rootView: ManyFilesLiveView(coordinator: coordinator, renders: renders) + .frame(width: 900, height: 1000).background(Color(nsColor: .windowBackgroundColor))) + let window = NSWindow(contentRect: NSRect(x: -2000, y: -2000, width: 900, height: 1000), + styleMask: [.titled], backing: .buffered, defer: false) + window.contentView = host; window.isReleasedWhenClosed = false + host.layoutSubtreeIfNeeded() + defer { window.close() } + let worker = Task { @MainActor in + try await run(root: root, source: source, artifacts: artifacts, count: count, withReports: reports == "both", + coordinator: coordinator, memory: memory, host: host, renders: renders) + } + let monitor = memory.monitor { + worker.cancel(); pipeline.cancelOperation() + Task { @MainActor in coordinator.cancelOperation() } + } + let layout = Task { @MainActor in + while !Task.isCancelled { + host.layoutSubtreeIfNeeded() + do { try await Task.sleep(for: .milliseconds(100)) } catch { return } + } + } + do { + try await withTaskCancellationHandler { try await worker.value } onCancel: { worker.cancel(); pipeline.cancelOperation() } + try memory.requireSafe() + guard testRun?.totalFailureCount == 0 else { throw ManyFilesError.assertionFailure } + memory.mark("settling") + try await Task.sleep(for: .seconds(2)) + monitor.cancel(); await monitor.value + layout.cancel(); await layout.value + try memory.requireSafe() + try save(memory, artifacts: artifacts, outcome: "passed") + // Keep compact evidence/history; remove only owned media after success. + let ownedMedia = Set(["Card", "BackupA-0", "BackupB-0", "BackupA-1", "BackupB-1"]) + for entry in try fm.contentsOfDirectory(at: root, includingPropertiesForKeys: nil) where ownedMedia.contains(entry.lastPathComponent) { + try fm.removeItem(at: entry) + } + print("APP_MANY_FILES PASSED count=\(count) backups=2 reports=\(reports) artifacts=\(artifacts.path)") + } catch { + worker.cancel(); pipeline.cancelOperation() + if coordinator.isOperationInProgress { coordinator.cancelOperation() } + _ = await worker.result + monitor.cancel(); await monitor.value + layout.cancel(); await layout.value + try? save(memory, artifacts: artifacts, outcome: "failed") + print("APP_MANY_FILES FAILED retainedFixture=\(root.path)") + throw error + } + } + + private func run(root: URL, source: URL, artifacts: URL, count: Int, withReports: Bool, + coordinator: SharedAppCoordinator, memory: ManyFilesMemory, + host: NSView, renders: ManyFilesRenders) async throws { + memory.mark("fixture") + let seed = Task.detached { try Self.seed(source, count: count) } + let original = try await withTaskCancellationHandler { try await seed.value } onCancel: { seed.cancel() } + try memory.requireSafe() + try JSONEncoder().encode(original).write(to: artifacts.appendingPathComponent("source-sha256.json"), options: .atomic) + for pass in 0..<(withReports ? 2 : 1) { + let report = pass == 1 + let destinations = ["BackupA-\(pass)", "BackupB-\(pass)"].map { root.appendingPathComponent($0) } + for destination in destinations { try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: false) } + coordinator.resetForNewOperation() + coordinator.cameraLabelSettings.label = "" + coordinator.reportSettings.makeReport = report + coordinator.reportSettings.includeThumbnails = false + coordinator.verificationMode = .standard + coordinator.generateASCMHL = true + coordinator.destinationURLs = destinations + coordinator.sourceURL = source + memory.mark("selection-\(pass)") + let deadline = ProcessInfo.processInfo.systemUptime + 60 + while !coordinator.canStartOperation && ProcessInfo.processInfo.systemUptime < deadline { + try memory.requireSafe(); try await Task.sleep(for: .milliseconds(25)) + } + guard coordinator.canStartOperation else { throw ManyFilesError.notReady } + let firstRender = renders.active + var updates = 0 + var finalLiveRows = 0 + var lastFraction = 0.0 + var rewound = false + let live = coordinator.liveResults.$rows.sink { rows in finalLiveRows = rows.count } + let progress = coordinator.liveProgress.$progress.compactMap { $0 }.sink { value in + updates += 1 + if value.overallProgress < lastFraction || value.overallProgress >= 1 { rewound = true } + lastFraction = value.overallProgress + memory.mark("run-\(pass)-\(value.currentStage.rawValue)") + } + let started = ProcessInfo.processInfo.systemUptime + memory.mark("run-\(pass)") + await coordinator.startOperation() + live.cancel(); progress.cancel() + host.layoutSubtreeIfNeeded() + try memory.requireSafe() + guard !rewound, updates > 1, finalLiveRows == count * 2, + renders.active > firstRender, + coordinator.results.count == count * 2, coordinator.results.allSatisfy(\.isVerifiedStatus), + let record = coordinator.outcomeRecord, record.state == .completed, + record.plannedFileCounts == [count, count], record.results.count == count * 2, + record.results.allSatisfy(\.isVerifiedStatus), record.independentDestinationCount == 2, + coordinator.transferJournal.records.count == pass + 1 else { throw ManyFilesError.incomplete } + let operation = try XCTUnwrap(coordinator.currentOperation) + guard operation.results.count == count * 2, operation.results.allSatisfy({ $0.success && $0.verificationResult?.isValid == true }), + Set(record.results.map(\.id)) == Set(coordinator.results.map(\.id)) else { throw ManyFilesError.incomplete } + memory.mark("hashes-\(pass)") + let checks = Task.detached { + guard try Self.hashes(source) == original else { throw ManyFilesError.sourceChanged } + for destination in destinations { + let output = SafetyValidator.resolvedDestinationRoot(source: source, destination: destination, settings: operation.settings) + let actual = try Self.hashes(output, mediaOnly: true) + guard actual == original else { throw ManyFilesError.destinationChanged } + } + } + try await withTaskCancellationHandler { try await checks.value } onCancel: { checks.cancel() } + try memory.requireSafe() + var pdfPages = 0 + if report { + memory.mark("pdfCheck-\(pass)") + let reports = destinations[0].appendingPathComponent("Reports") + let pdfs = try FileManager.default.contentsOfDirectory(at: reports, includingPropertiesForKeys: nil) + .filter { $0.pathExtension.lowercased() == "pdf" } + pdfPages = try autoreleasepool { + guard pdfs.count == 1, let pdf = PDFDocument(url: pdfs[0]), pdf.pageCount > 0, + let text = pdf.string, text.contains("VERIFICATION SUCCESSFUL") else { throw ManyFilesError.incomplete } + // Each selected source path must appear once per backup in the actual PDF manifest. + // PDFKit may split text runs after underscores even when the page is visually unwrapped. + // These fixed fixture names contain no whitespace; this is not a general path parser. + let pattern = try NSRegularExpression(pattern: #"Folder[0-9]+/file_\s*[0-9]+\.bin"#) + var occurrences: [String: Int] = [:] + pattern.enumerateMatches(in: text, range: NSRange(text.startIndex..., in: text)) { match, _, _ in + if let match, let range = Range(match.range, in: text) { + occurrences[String(text[range].filter { !$0.isWhitespace }), default: 0] += 1 + } + } + guard Set(occurrences.keys) == Set(original.keys), occurrences.values.allSatisfy({ $0 == 2 }) else { + throw ManyFilesError.incomplete + } + return pdf.pageCount + } + try FileManager.default.copyItem(at: pdfs[0], to: artifacts.appendingPathComponent("report-\(pass).pdf")) + } else if FileManager.default.fileExists(atPath: destinations[0].appendingPathComponent("Reports").path) { + throw ManyFilesError.incomplete + } + try memory.requireSafe() + memory.completed(pass: pass, files: count, rows: count * 2, updates: updates, + renders: renders.active - firstRender, report: report, pdfPages: pdfPages, seconds: ProcessInfo.processInfo.systemUptime - started) + try save(memory, artifacts: artifacts, outcome: "running") + } + } + + private nonisolated static func seed(_ source: URL, count: Int) throws -> [String: String] { + for folder in 0..<32 { try FileManager.default.createDirectory(at: source.appendingPathComponent("Folder\(folder)"), withIntermediateDirectories: false) } + for index in 0.. [String: String] { + var output: [String: String] = [:] + let files = try autoreleasepool { try CardSource.enumerateRegularFiles(base: root) } + for file in files { + try Task.checkCancellation() + if mediaOnly && (file.relativePath.hasPrefix("ascmhl/") || file.relativePath.hasPrefix("Reports/")) { continue } + if mediaOnly && TransferStatusMarker.hasExactName(file.relativePath) { + guard TransferStatusMarker.validatedStatus(at: file.url, relativePath: file.relativePath) == .filesVerified else { + throw ManyFilesError.destinationChanged + } + continue + } + output[file.relativePath] = try autoreleasepool { + let handle = try FileHandle(forReadingFrom: file.url) + defer { try? handle.close() } + var hasher = SHA256() + while true { + try Task.checkCancellation() + let count = try autoreleasepool { () throws -> Int in + let bytes = try handle.read(upToCount: 1024 * 1024) ?? Data(); hasher.update(data: bytes); return bytes.count + } + if count == 0 { break } + } + return hasher.finalize().map { String(format: "%02x", $0) }.joined() + } + } + return output + } + + private func save(_ memory: ManyFilesMemory, artifacts: URL, outcome: String) throws { + var state = memory.snapshot(); state.outcome = outcome + let encoder = JSONEncoder(); encoder.outputFormatting = [.sortedKeys, .prettyPrinted] + try encoder.encode(state).write(to: artifacts.appendingPathComponent("memory.json"), options: .atomic) + } +} + +private struct ManyFilesAbstractDisks: PhysicalDiskIdentityProviding { + func physicalDiskIdentity(for url: URL) -> String? { "synthetic-abstract-" + url.lastPathComponent } +} + +@MainActor private final class ManyFilesRenders { + var active = 0 + func render(_ running: Bool) { if running { active += 1 } } +} + +private struct ManyFilesLiveView: View { + @ObservedObject var coordinator: SharedAppCoordinator + @ObservedObject var progress: LiveProgressFeed + let renders: ManyFilesRenders + init(coordinator: SharedAppCoordinator, renders: ManyFilesRenders) { + self.coordinator = coordinator; self.progress = coordinator.liveProgress; self.renders = renders + } + var body: some View { + let _ = renders.render(coordinator.isOperationInProgress) + return ScrollView { + VStack { + TransferQueueSection(coordinator: coordinator) + if coordinator.isOperationInProgress { + ProgressScreen(presentation: .make(coordinator: coordinator), + actions: ProgressActions(pause: { Task { await coordinator.pauseOperation() } }, + resume: { Task { await coordinator.resumeOperation() } }, + cancel: { coordinator.cancelOperation() }), confirmingCancel: .constant(false)) + } + }.padding() + } + } +} + +private enum ManyFilesError: Error { case invalidFixture, memoryLimit, deadline, assertionFailure, notReady, incomplete, sourceChanged, destinationChanged } + +private final class ManyFilesMemory: Sendable { + struct Phase: Codable { var peakMiB: Int; var lastMiB: Int } + struct State: Codable { + var outcome = "running"; var phase = "initial"; var samples = 0 + var initialMiB = 0; var peakMiB = 0; var lastMiB = 0 + var guardTriggered = false; var missingSample = false + var phases: [String: Phase] = [:] + var runs: [[String: String]] = [] + let limitMiB = 1024 + let maxSeconds: TimeInterval + let scope = "Actual app-test process, coordinator/pipeline/live NSHostingView; host scratch only, abstract disk identities, expected O(N) rows/history/PDF storage, not issue23 or constant-memory proof" + } + private let state: Mutex + private let started = ProcessInfo.processInfo.systemUptime + private let maxSeconds: TimeInterval + init(maxSeconds: TimeInterval = 600) { + self.maxSeconds = maxSeconds + state = Mutex(State(maxSeconds: maxSeconds)) + } + func snapshot() -> State { state.withLock { $0 } } + func mark(_ phase: String) { state.withLock { $0.phase = phase }; _ = sample() } + func completed(pass: Int, files: Int, rows: Int, updates: Int, renders: Int, report: Bool, pdfPages: Int, seconds: TimeInterval) { + state.withLock { $0.runs.append(["pass": "\(pass)", "files": "\(files)", "verifiedRows": "\(rows)", + "progressUpdates": "\(updates)", "activeViewRenders": "\(renders)", "report": "\(report)", "pdfPages": "\(pdfPages)", "seconds": "\(seconds)"] ) } + } + func requireSafe() throws { + if sample() { throw ManyFilesError.memoryLimit } + if ProcessInfo.processInfo.systemUptime - started >= maxSeconds { throw ManyFilesError.deadline } + try Task.checkCancellation() + } + func monitor(cancel: @escaping @Sendable () -> Void) -> Task { + Task.detached { + while !Task.isCancelled { + if self.sample() || ProcessInfo.processInfo.systemUptime - self.started >= self.maxSeconds { cancel(); return } + do { try await Task.sleep(for: .milliseconds(10)) } catch { return } + } + } + } + @discardableResult func sample() -> Bool { + var info = task_vm_info_data_t() + var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) + let result = withUnsafeMutablePointer(to: &info) { p in + p.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count) } + } + return record(result == KERN_SUCCESS ? Int(info.phys_footprint / 1_048_576) : nil) + } + @discardableResult func record(_ value: Int?) -> Bool { + state.withLock { s in + guard let value else { s.guardTriggered = true; s.missingSample = true; return true } + if s.samples == 0 { s.initialMiB = value } + s.samples += 1; s.lastMiB = value; s.peakMiB = max(s.peakMiB, value) + let old = s.phases[s.phase]?.peakMiB ?? 0 + s.phases[s.phase] = Phase(peakMiB: max(old, value), lastMiB: value) + if value >= s.limitMiB { s.guardTriggered = true } + return s.guardTriggered + } + } +} +#endif diff --git a/docs/validation/COORDINATOR_MANY_FILES_MEMORY_20261020.md b/docs/validation/COORDINATOR_MANY_FILES_MEMORY_20261020.md new file mode 100644 index 00000000..3f520837 --- /dev/null +++ b/docs/validation/COORDINATOR_MANY_FILES_MEMORY_20261020.md @@ -0,0 +1,117 @@ +# Many-file coordinator memory probe + +This opt-in Mac app test exercises the real coordinator, transfer pipeline and +observed production progress/queue views with 20,000 distinct 1 KiB files and two +backups. Standard verification and ASC MHL stay enabled. Synthetic disk identities +make the workflow eligible for two-backup policy; every folder is actually on the +same host filesystem. This is not independent physical-backup coverage. + +The test checks 40,000 verified live/final/journal/engine rows, matching row IDs, +completed journal state, exact planned counts, active view renders and monotonic +progress below 100% while active. It independently hashes every original and both +copies after the transfer, preserving the original manifest. Report-on runs also +require an actual readable PDF whose manifest contains every relative source +path exactly twice, once per backup, with no missing or extra selected paths. +Its page count and the PDF are retained with the receipt. The fixture-specific +text check tolerates PDFKit line breaks inside these short generated names; it +is not a general parser for footage paths. Thumbnails are disabled in this case. + +The actual app-test process samples its physical footprint every 10 ms. Missing +samples or a 1 GiB observation fail closed and request cancellation of owned work. +The coordinator awaits the pipeline; the test joins its own worker, monitor and +layout task, then records settled memory before a passing receipt. Filesystem +calls and synchronous PDF rendering remain cooperative; +this guard is not an OS-enforced memory cap or an absolute runtime guarantee. +Successful fixtures remove only explicitly named owned media children; failures +retain their media and evidence. Each file's independent hash validation, including +read/open/hex formatting/close, is inside an autorelease pool so the test helper does +not accumulate Foundation temporaries across the entire inventory. + +Two ordinary regressions check sticky memory-limit cancellation and unavailable +memory samples. The expensive coordinator case skips unless explicitly enabled. +Use these environment values in the app test target's xctestrun environment: + +```text +BITMATCH_APP_MANY_FILES_MEMORY=1 +BITMATCH_APP_MANY_FILES_COUNT=20000 +BITMATCH_APP_MANY_FILES_REPORTS=off +BITMATCH_APP_MANY_FILES_MAX_SECONDS=1200 +``` + +The count accepts 100–20,000; reports accepts `off` or `both` (one pass without +reports, then a fresh pair of backups with reports); the cooperative deadline +accepts 60–1,200 seconds and defaults to 600. Run only +`CoordinatorManyFilesMemoryTests` with app-host parallel testing disabled. + +## Results and limits + +The corrected 300-file Debug smoke passed seven tests: three XCTest methods and +four normal result-index semantic tests, with no failures or skips. The actual +coordinator case produced 600 verified rows, 51 progress callbacks and 56 active +view renders. Its 499 memory samples peaked at 142 MiB and settled at 93 MiB. + +The 20,000-file Debug report-off case passed: 40,000 verified rows, 3,014 progress +callbacks and 3,690 active view renders, with 24,541 samples. XCTest wall time was +248.983 seconds. Peak process footprint was 874 MiB, including the earlier +validation helper; the transfer phase peaked at 640 MiB and the post-run direct +hash check at 874 MiB. It settled at 257 MiB. This retained result predates the +whole-file validation-helper autorelease-pool correction and is qualified as such. + +An earlier 20,000-file attempt was deliberately interrupted to repair a harness +cleanup-path comparison; it is not counted as a pass or a product failure. That +comparison affected only the test's own evidence directory, and cleanup now uses +an explicit child-name allowlist. + +The corrected-helper Release 20,000-file report-off case passed one XCTest in +210.093 seconds, with no skips or failures. It produced 40,000 verified rows, +3,069 progress callbacks and 3,661 active view renders. Across 21,300 samples, +process footprint was 48 MiB initially, 438 MiB at peak and 393 MiB after settling +with the completed coordinator/results/history still loaded. +The observed transfer-phase peak was 380 MiB; the post-run direct hashes peaked at +438 MiB. The memory guard stayed clear and no samples were unavailable. This run changed +both the build configuration and the helper pools; the peak difference is not +a controlled attribution to either change alone. + +Xcode's structured runtime-warning summary was empty, but the captured native log +contains one SwiftUI `onChange(of: Array)` multiple-updates-per-frame +warning during the MHL handoff. It is retained as a presentation follow-up, not +silently reported as a warning-free run. + +The first 2,000-file report-on attempt completed both transfers, independent +source/copy hashes and a 144-page PDF, then failed the test's exact-path assertion. +PDFKit inserted a newline after each underscore in its text extraction, while +Poppler extracted all 4,000 visible manifest paths correctly. The corrected +fixture-specific extractor independently revalidated that retained PDF: exactly +2,000 selected paths, each present twice. This was a harness assertion defect, +not missing PDF evidence. That failed fixture/receipt is retained, and a fresh +coordinator rerun validates the corrected harness rather than relabeling the +failed XCTest as passed. + +The corrected Release 2,000-file two-pass case then passed with the two normal +memory guards and four result-index regressions: seven tests, no skips or +failures. Each pass produced 4,000 verified results and passed all direct source +and backup hash checks. The report-on pass produced a readable 144-page PDF with +exactly 4,000 manifest entries: every one of the 2,000 source paths twice. +Across 11,927 footprint samples, the process started at 72 MiB, peaked at 327 MiB +(in the test's PDF extraction), and settled at 119 MiB. The memory guard stayed +clear. The native queue-observer warning remained; Xcode's structured warning +summary was empty. The first/report-off pass took 20.856 seconds and the second, +with the previous completed record still displayed and reporting enabled, took +105.937 seconds. These differ in history and reporting work, so they are not an +isolated report-performance comparison. + +This establishes the 20,000-file report-off and 2,000-file report-on scopes +separately. A 20,000-file PDF was not run. The queue presentation currently groups +completed records' result rows on repeated renders; caching that immutable +summary is a separate possible performance improvement, not part of this patch. + +The Release test build uses normal optimizations plus the command-line +`ENABLE_TESTABILITY=YES` override; it is an app-test host, not the signed shipping +app. Xcode log capture, XCTest, live views and the test's own inventories all share +the measured process. Build work elsewhere on the Mac makes elapsed times +observations rather than throughput benchmarks. + +Row arrays, manifests, history and PDF pages retain expected O(N) evidence. This +probe does not establish constant memory for unlimited files/history, prove +native picker/device/drive behavior, or identify the cause of issue #23. Separate +large-file, thumbnail, filesystem and process-death tests cover other workloads. From 058d6bd805d30529c14b486cb8d399d3e37fbe44 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 00:43:24 -0400 Subject: [PATCH 44/56] Observe queue completion transitions without progress churn --- .../QueueSessionPresentationTests.swift | 76 +++++++++++++++++++ .../Models/QueueSessionPresentation.swift | 26 ++++++- Shared/Views/TransferQueueSection.swift | 11 ++- 3 files changed, 106 insertions(+), 7 deletions(-) diff --git a/BitMatchTests/QueueSessionPresentationTests.swift b/BitMatchTests/QueueSessionPresentationTests.swift index c2bb8814..b9973c7c 100644 --- a/BitMatchTests/QueueSessionPresentationTests.swift +++ b/BitMatchTests/QueueSessionPresentationTests.swift @@ -368,6 +368,82 @@ struct QueueSessionPresentationTests { #expect(!QueueHeroPolicy.shouldExpand(row: safeRow, previousRows: [], currentRows: runningRows)) } + @Test func transitionObservationIgnoresProgressAndActivePhaseChanges() throws { + let fixture = try QueuePresentationFixture() + defer { fixture.cleanup() } + let running = try fixture.record(name: "A001", state: .running, at: 1) + let stages: [ProgressStage] = [.preparing, .copying, .copying, .verifying, .generating, .completed] + let rows = try stages.enumerated().map { index, stage in + let progress = OperationProgress(overallProgress: Double(index) / 6, + currentFile: "clip-\(index).mov", filesProcessed: index, totalFiles: 6, + currentStage: stage, speed: Double(index + 1)) + return try #require(QueueSessionPresentation.make( + records: [running], sessionIDs: [running.id], progress: progress, mountedSourceIDs: [] + ).rows.first) + } + let first = try #require(rows.first) + #expect(rows.dropFirst().allSatisfy { $0 != first }) + #expect(rows.allSatisfy { $0.isRunning && !$0.showsSafeHero }) + #expect(rows.map(QueueRowTransitionState.init).allSatisfy { $0 == QueueRowTransitionState(first) }) + } + + @Test func transitionObservationKeepsIdentityOrderRunningAndSafeVerdicts() throws { + let fixture = try QueuePresentationFixture() + defer { fixture.cleanup() } + let waiting = try fixture.record(name: "A001", state: .queued, at: 1) + let safe = try fixture.record(name: "A002", state: .completed, outcome: .verified, at: 2) + func rows(_ records: [LocalTransferRecord]) -> [QueueSessionRow] { + QueueSessionPresentation.make(records: records, sessionIDs: Set(records.map(\.id)), + progress: nil, mountedSourceIDs: [safe.id]).rows + } + let initial = rows([waiting, safe]).map(QueueRowTransitionState.init) + #expect(initial != Array(initial.reversed())) + #expect(initial != rows([waiting]).map(QueueRowTransitionState.init)) + #expect(initial != rows([waiting, safe, try fixture.record(name: "A003", state: .queued, at: 3)]) + .map(QueueRowTransitionState.init)) + var running = waiting + running.state = .running + #expect(rows([waiting]).map(QueueRowTransitionState.init) != rows([running]).map(QueueRowTransitionState.init)) + var noVerdict = safe + noVerdict.state = .issues + noVerdict.results = [] + #expect(rows([noVerdict]).map(QueueRowTransitionState.init) != rows([safe]).map(QueueRowTransitionState.init)) + var retry = safe + retry.state = .running + #expect(rows([safe]).map(QueueRowTransitionState.init) != rows([retry]).map(QueueRowTransitionState.init)) + } + + @Test func projectedHeroInputsPreserveSafetyAndPreviousRowPolicy() throws { + let fixture = try QueuePresentationFixture() + defer { fixture.cleanup() } + let safe = try fixture.record(name: "A001", state: .completed, outcome: .verified, at: 1) + let running = try fixture.record(name: "A002", state: .running, at: 2) + let failed = try fixture.record(name: "A003", state: .failed, outcome: .failed, at: 3) + let quick = try fixture.record(name: "A004", state: .issues, mode: .quick, + outcome: .copiedUnverified, quickOnlyUnverified: true, at: 4) + let excludedOnly = try fixture.record(name: "A005", state: .issues, outcome: .excludedAppleDouble, at: 5) + let records = [safe, running, failed, quick, excludedOnly] + let rows = QueueSessionPresentation.make(records: records, sessionIDs: Set(records.map(\.id)), + progress: nil, mountedSourceIDs: Set(records.map(\.id))).rows + let safeRow = try #require(rows.first { $0.id == safe.id }) + let runningRow = try #require(rows.first { $0.id == running.id }) + #expect(rows.filter { [failed.id, quick.id, excludedOnly.id].contains($0.id) }.allSatisfy { !$0.showsSafeHero }) + for row in rows { + for previous in [[], [row], [safeRow]] { + for current in [[row], [row, runningRow], [row, safeRow]] { + let oldSemantics = row.showsSafeHero + && previous.first(where: { $0.id == row.id })?.showsSafeHero != true + && !current.contains { $0.id != row.id && $0.isRunning } + let prior = previous.map(QueueRowTransitionState.init) + #expect(QueueHeroPolicy.shouldExpand(row: row, previousRows: previous, currentRows: current) == oldSemantics) + #expect(QueueHeroPolicy.shouldExpand(row: row, + wasSafe: prior.first(where: { $0.id == row.id })?.showsSafeHero == true, + otherCardIsRunning: current.contains { $0.id != row.id && $0.isRunning }) == oldSemantics) + } + } + } + } + @Test func accessibilityStatusCombinesCardStateCauseAndSafetyWarning() throws { let fixture = try QueuePresentationFixture() defer { fixture.cleanup() } diff --git a/Shared/Core/Models/QueueSessionPresentation.swift b/Shared/Core/Models/QueueSessionPresentation.swift index a40a743c..79f748c6 100644 --- a/Shared/Core/Models/QueueSessionPresentation.swift +++ b/Shared/Core/Models/QueueSessionPresentation.swift @@ -69,6 +69,20 @@ struct QueueSessionRow: Identifiable, Equatable, Sendable { } } +/// Row transitions used by completion heroes and automatic eject. +/// Progress and phase text still render, without rechecking those actions. +struct QueueRowTransitionState: Equatable, Sendable { + let id: UUID + let isRunning: Bool + let showsSafeHero: Bool + + init(_ row: QueueSessionRow) { + id = row.id + isRunning = row.isRunning + showsSafeHero = row.showsSafeHero + } +} + struct QueueTally: Equatable, Sendable { var safeToErase = 0 var copiedNotVerified = 0 @@ -444,9 +458,15 @@ enum QueueHeroPolicy { previousRows: [QueueSessionRow], currentRows: [QueueSessionRow] ) -> Bool { - guard row.showsSafeHero, - previousRows.first(where: { $0.id == row.id })?.showsSafeHero != true else { return false } - return !currentRows.contains { $0.id != row.id && $0.isRunning } + shouldExpand( + row: row, + wasSafe: previousRows.first(where: { $0.id == row.id })?.showsSafeHero == true, + otherCardIsRunning: currentRows.contains { $0.id != row.id && $0.isRunning } + ) + } + + static func shouldExpand(row: QueueSessionRow, wasSafe: Bool, otherCardIsRunning: Bool) -> Bool { + row.showsSafeHero && !wasSafe && !otherCardIsRunning } } diff --git a/Shared/Views/TransferQueueSection.swift b/Shared/Views/TransferQueueSection.swift index a1ca2834..df248d92 100644 --- a/Shared/Views/TransferQueueSection.swift +++ b/Shared/Views/TransferQueueSection.swift @@ -89,8 +89,8 @@ struct TransferQueueSection: View { reduceMotion ? .easeInOut(duration: 0.18) : .spring(duration: 0.4, bounce: 0.18), value: presentation.rows.map(\.id) ) - .onChange(of: presentation.rows) { oldRows, newRows in - handleRowChanges(from: oldRows, to: newRows) + .onChange(of: presentation.rows.map(QueueRowTransitionState.init)) { oldRows, _ in + handleRowChanges(from: oldRows, to: presentation.rows) } .onDisappear { heroDismissTask?.cancel() @@ -772,7 +772,7 @@ struct TransferQueueSection: View { else { withAnimation(.easeInOut(duration: 0.22)) { expandedIDs.insert(id) } } } - private func handleRowChanges(from oldRows: [QueueSessionRow], to newRows: [QueueSessionRow]) { + private func handleRowChanges(from oldRows: [QueueRowTransitionState], to newRows: [QueueSessionRow]) { if let heroID, newRows.contains(where: { $0.id != heroID && $0.isRunning }) { settleHero(heroID) } @@ -786,7 +786,10 @@ struct TransferQueueSection: View { ) { eject(row.id) } - if QueueHeroPolicy.shouldExpand(row: row, previousRows: oldRows, currentRows: newRows) { + if QueueHeroPolicy.shouldExpand( + row: row, wasSafe: !isNewlySafe, + otherCardIsRunning: newRows.contains { $0.id != row.id && $0.isRunning } + ) { showHero(for: row) } } From 89187c5e4e2b8002a14d4195a7ac75edb0d2a1e2 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 00:49:09 -0400 Subject: [PATCH 45/56] Keep memory-probe cancellation scoped to active work --- .../CoordinatorManyFilesMemoryTests.swift | 2 +- .../COORDINATOR_MANY_FILES_MEMORY_20261020.md | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/BitMatchTests/CoordinatorManyFilesMemoryTests.swift b/BitMatchTests/CoordinatorManyFilesMemoryTests.swift index 2154a88a..890ec0e7 100644 --- a/BitMatchTests/CoordinatorManyFilesMemoryTests.swift +++ b/BitMatchTests/CoordinatorManyFilesMemoryTests.swift @@ -76,7 +76,7 @@ final class CoordinatorManyFilesMemoryTests: XCTestCase { } let monitor = memory.monitor { worker.cancel(); pipeline.cancelOperation() - Task { @MainActor in coordinator.cancelOperation() } + Task { @MainActor in if coordinator.isOperationInProgress { coordinator.cancelOperation() } } } let layout = Task { @MainActor in while !Task.isCancelled { diff --git a/docs/validation/COORDINATOR_MANY_FILES_MEMORY_20261020.md b/docs/validation/COORDINATOR_MANY_FILES_MEMORY_20261020.md index 3f520837..f4953658 100644 --- a/docs/validation/COORDINATOR_MANY_FILES_MEMORY_20261020.md +++ b/docs/validation/COORDINATOR_MANY_FILES_MEMORY_20261020.md @@ -105,6 +105,24 @@ separately. A 20,000-file PDF was not run. The queue presentation currently grou completed records' result rows on repeated renders; caching that immutable summary is a separate possible performance improvement, not part of this patch. +A matched 2,000-file two-pass run after the queue's transition-only observer +change passed again: one XCTest, no skips or failures; 8,000 persisted verified +rows across two completed attempts; every original/copy hash preserved; and the +same 144-page PDF with exactly 4,000 expected manifest entries. Across 10,417 +samples, physical footprint was 47 MiB initially, 303 MiB at peak and 95 MiB after +settling. Both the structured Xcode warning list and a full raw-log scan were +clear. The raw scan covered any `onChange`, multiple-updates-per-frame, +modifying/publishing-during-view-update and AttributeGraph cycle messages, so it +does not merely rely on the old warning's array type disappearing. This confirms +the observed warning is absent for the matched workload, not every possible UI +path. The report-off/on passes took 19.921/89.934 seconds on the shared host; +those observations are not a controlled performance benchmark. + +The final test-only cleanup also avoids sending a user-cancel action after the +coordinator has already completed if its independent memory guard trips during +post-run hash/PDF validation. Existing successful receipts had no guard trigger; +this preserves their assertions while keeping future failure breadcrumbs honest. + The Release test build uses normal optimizations plus the command-line `ENABLE_TESTABILITY=YES` override; it is an app-test host, not the signed shipping app. Xcode log capture, XCTest, live views and the test's own inventories all share From 2c3ac9093aa6433f0ccdece3b258c20ae11d77af Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 00:53:30 -0400 Subject: [PATCH 46/56] Track repeated completed-queue summary work --- docs/TODO.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/TODO.md b/docs/TODO.md index 3315f5fa..5ec67869 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -187,6 +187,7 @@ These remain ongoing gates, not optional features. Use disposable data and retai ## Developer chores +- [ ] **Avoid repeating completed-card summaries on every queue render.** `QueueSessionPresentation.makeRow` still regroups each completed record's results: repeated O(N) work in the result count, not a demonstrated cause of issue #23. Consider a bounded cache of derived summaries, invalidated by result and verdict revisions; keep live mount/eject state and active progress current. **Acceptance:** amended results, changed verdicts, retry and removal cannot leave stale summaries or promote incomplete evidence to safe. Check many completed cards alongside an active transfer on Mac, iPad and iPhone; preserve the existing fail-closed verdict rules. - [ ] **Tests without permission prompts.** Investigate drive-access prompts that stall Mac tests; prefer owned temporary fixtures and scoped test configuration without weakening release entitlements. The documented DerivedData-outside-Desktop workaround addresses a separate loader stall. A separate close-window test-host crash was fixed in #46; that fix does not settle the permission-prompt audit. - [x] **Buffer benchmark first run.** Source creation and timed reads request `F_NOCACHE`. The [October 19–20 run](OVERNIGHT_VALIDATION_20261019.md) confirmed substantial read traffic in the selected OS storage driver during the first timed read. This does not prove a cold device cache or app-only attribution. Keep the current buffer and verification defaults until controlled end-to-end measurements justify a change. - [x] **Reverse flight on Edit.** Mac Edit carries the queued card back to setup, respecting Reduce Motion. Compact mobile Edit dismisses the queue sheet. Settings, operator and notes remain snapshotted. From 7c7c326f6b85aeee29c4ed72a06faabd1b987a49 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 00:59:18 -0400 Subject: [PATCH 47/56] Record integrated app and memory validation --- CHANGELOG.md | 2 ++ docs/OVERNIGHT_VALIDATION_20261019.md | 42 +++++++++++++++++++-------- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c686253f..4531257a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ All notable changes to this project will be documented in this file. - Check no longer reports preserved Finder metadata as missing when it is listed in the saved checksums. - Corrected physical-drive checks for selected subfolders, so two folders on one drive count as one backup. - Cleaned up long names and page breaks in PDFs; metadata companions no longer inflate the clip-preview count. Job details now share the page footer instead of creating a nearly empty last page. +- Long Master Report details and notes now continue onto another page instead of being cut off. +- Reduced extra work when tracking transfers with lots of files. - Mac: Advanced points out a selected backup using a reported USB 2 connection. - Diagnostics exports reject oversized or unreadable local records, keep the usable events, and say when the record is incomplete. - Destination progress no longer resets to Waiting while ASC MHL and transfer reports are being saved. Processed counts and measured progress stay visible without implying the transfer is complete. diff --git a/docs/OVERNIGHT_VALIDATION_20261019.md b/docs/OVERNIGHT_VALIDATION_20261019.md index 4359229f..8e0903d4 100644 --- a/docs/OVERNIGHT_VALIDATION_20261019.md +++ b/docs/OVERNIGHT_VALIDATION_20261019.md @@ -1,6 +1,6 @@ # Overnight validation — October 19–20, 2026 -**Draft checkpoint.** This records local validation and its limits, not a release approval or proof that a field issue is fixed. The work started from `d6096442`; integration was at `4106a48a` when this draft was prepared. Several long-running tests were compiled earlier, as qualified below. Final integration and cleanup results still need to be recorded. +**Draft checkpoint.** This records local validation and its limits, not a release approval or proof that a field issue is fixed. The work started from `d6096442`; individual gate revisions and earlier long-running binaries are qualified below. Final integration and cleanup results still need to be recorded. [Issue #10](https://github.com/BitmatchApp/Bitmatch/issues/10) and [issue #23](https://github.com/BitmatchApp/Bitmatch/issues/23) remain open. These runs do not establish the reporters' exact failure mechanism. @@ -52,6 +52,18 @@ The [process-death matrix](validation/PROCESS_DEATH_RECOVERY_20261019.md) passed The focused run scheduled three XCTest methods: two passed and one internal helper skipped. The 20 scenarios are not 20 additional XCTest passes. Ordinary discovery passed the ownership guard and skipped the two opt-in methods. Final images and disposable source payloads were cleaned after normal detach. +## Repeated transfers and many-file workloads + +The [seeded host soak](validation/SEEDED_SOAK_20261020.md) passed 256 iterations using all 256 low-byte payload patterns, with 2,304 source hashes before, 2,304 after and 4,608 destination hashes. Its corrected runner requires one actual passing XCTest and retains partial or failed evidence. A separate engine case passed with 20,009 files; independent post-run source and destination hashes matched the declared fixture bytes. That case had no independent pre-transfer hash pass. These are host-filesystem engine tests, without MHL or app UI. + +A separate production-coordinator/SwiftUI-host probe completed 20,000 files to two backups through MHL and journal settlement. Debug passed with 40,000 verified rows and matching source/destination hashes; its 874 MiB sampled peak included the test's post-transfer hash helper. After pooling that helper's per-file Foundation work, Release also passed: 40,000 verified rows, a 438 MiB sampled process peak and 393 MiB settled footprint. These figures include the test host and retained history, not just the shipping app's transfer buffers. The Release test host enables test visibility; it is not an identical shipping binary. + +The [PDF-enabled follow-up](validation/COORDINATOR_MANY_FILES_MEMORY_20261020.md) passed two sequential 2,000-file transfers in one process, including all 4,000 entries across both destinations in each 144-page PDF. Source and destination hashes and persisted results matched. The initial corrected run passed seven tests without skips or failures and peaked at 327 MiB. Its raw log contained a SwiftUI queue `onChange` warning even though structured runtime warnings were empty. The queue observer now tracks completion transitions rather than every progress update; 24 focused presentation tests passed. A matched real-workflow rerun passed, peaked at 303 MiB and had no matches in either structured warnings or the broader raw-console scan. That establishes warning removal for this case, not every possible UI workload. Neither run proves constant memory use or the cause of issue #23. + +Long Master Report summaries, notes and oversized file rows now paginate instead of clipping. Seven actual-service controls passed; six stress cases failed against the earlier renderer. The normal one-page case stayed one page, and Unicode notes retained their text across page breaks. These checks are separate from the transfer-PDF preview cases. + +The probe exposed a separate [quadratic result lookup](validation/RUN_RESULT_LOOKUP_20261020.md). Its indexed replacement preserves arrival order, exact source/destination matching and first-duplicate behavior. Four semantic tests passed. This removes measured bookkeeping work; it does not establish a memory leak or explain issue #23. + ## First buffer-benchmark read The selected SSD's opt-in raw benchmark completed **one XCTest, zero failures and zero skips**: 25 hash passes and 25 raw-copy passes, each over a 2 GiB fixture. This is a raw I/O/hash benchmark, not a verified application transfer. @@ -64,20 +76,20 @@ Before that run, the benchmark harness was corrected so each FileHandle read and ## App, engine and simulator gates -The engine gates were rerun after the buffer-harness additions at `4106a48a`; the full Mac, iPad and iPhone suites passed again at `84445c21`, including the high-resolution probe guards and final PDF footer regressions. Logical test counts and parameterized case counts are kept separate. +The engine gates passed again at `79f07486`, including the soak and private diagnostic-scope additions. Fresh integrated Mac, iPad and iPhone suites passed at `2c3ac909`; the only commit during that app run changed the TODO document. Logical test counts and parameterized case counts are kept separate. | Gate | Completed result | | --- | --- | -| Engine Debug, XCTest | 223 scheduled: 186 passed, 37 opt-in/platform skips, zero failures | -| Engine Release, XCTest | 223 scheduled: 186 passed, 37 opt-in/platform skips, zero failures | -| Engine Debug, Swift Testing | 219 declarations in 39 suites: 213 passed, six opt-in skips, zero failures | -| Engine Release, Swift Testing | 219 declarations in 39 suites: 213 passed, six opt-in skips, zero failures | -| Full Mac app suite | 1,291 logical tests: 1,269 passed, 22 skipped, zero failures; no reported runtime warnings | -| iPad simulator | 81 tests: 71 passed, ten skipped, zero failures; no reported runtime warnings | -| iPhone simulator | 81 tests: 71 passed, ten skipped, zero failures; no reported runtime warnings | +| Engine Debug, XCTest | 227 scheduled: 188 passed, 39 opt-in/platform skips, zero failures | +| Engine Release, XCTest | 227 scheduled: 188 passed, 39 opt-in/platform skips, zero failures | +| Engine Debug, Swift Testing | 223 declarations in 40 suites: 217 passed, six opt-in skips, zero failures | +| Engine Release, Swift Testing | 223 declarations in 40 suites: 217 passed, six opt-in skips, zero failures | +| Full Mac app suite | 1,305 logical tests: 1,282 passed, 23 skipped, zero failures; structured and raw UI-warning scans clear | +| iPad simulator | 85 tests: 75 passed, ten skipped, zero failures; structured and raw UI-warning scans clear | +| iPhone simulator | 85 tests: 75 passed, ten skipped, zero failures; structured and raw UI-warning scans clear | | Shared mobile build | Universal iOS simulator build passed | -| Release builds | Universal Mac and iOS-device builds passed | -| Local signed Mac archive | `5fb59a64`: x86_64 and arm64, deep/strict signature verification passed; no install, notarization or release | +| Release builds | `2c3ac909`: universal Mac and iOS-device builds passed with signing disabled | +| Local signed Mac archive | `2c3ac909`: x86_64 and arm64, deep/strict signature verification passed; no install, notarization or release | Skipped opt-in tests are not counted as passes. Some were enabled in separate runs above; those runs do not turn the whole-suite skip count into zero. @@ -85,11 +97,17 @@ A separate signed-app walkthrough completed a Standard transfer to two physical The diagnostics-export hardening had its own **32 passing XCTest cases**, zero skips and a universal mobile build. An oversized-input regression was observed failing before the fix. The tests cover malformed, oversized, unreadable, symlink and FIFO inputs, preserving valid events from the other input and reporting incomplete evidence honestly. This does not identify the cause of issue #23. +Separate Thread Sanitizer and Address Sanitizer builds at `3c284702` exercised the selected diagnostics, recorded-metadata, disk-resolution and publication tests: each scheduled 24 XCTest methods, with 22 passed, two opt-in physical tests skipped and zero failures. Each sanitizer also passed 13 Swift Testing declarations covering the pipeline and byte-weighted progress. The test binaries contained the respective sanitizer initialization symbols, and neither run reported sanitizer findings. An initial receipt parser rejected the successful Swift Testing summary because it included a suite count; the retained logs were revalidated after correcting the parser. These checks cover the executed paths, not every possible race, memory leak or resource termination. + +The later diagnostic-scope changes also passed Thread Sanitizer: 15 XCTest methods and six Swift Testing declarations, with four declarations passed and two opt-in large-file cases skipped. Instrumentation was confirmed and no findings were reported. Scope tests cover child-task inheritance, nested throwing scopes and concurrent isolation; the override does not propagate to detached tasks or other processes. + +A preceding post-soak Release run failed two diagnostic-ordering assertions while reading the shared ring. A separate private-directory reproduction confirmed silent record loss between independent store instances; it does not establish which process affected the original test. The regression fixtures now use private diagnostic stores, and the fresh full Debug/Release engine gates above pass. This fixes test isolation, not independent-process writes to one diagnostics file; the failed logs remain retained. + ## Pending finalization - **Large physical transfer:** the corrected 119.6 GB SSD A → SSD B case was still running at this checkpoint. Do not count it as a pass, infer its eventual verdict, or use an earlier flawed-harness attempt as a substitute. - **Application performance:** isolated repeated Standard/Thorough application trials and their counter-aligned measurements need a settled receipt. Raw buffer throughput is not application performance evidence. -- **4K report-preview memory:** the [reviewed host-only probe](validation/report-preview-memory-2026-10-19.md) passed at 20 and 200 paths, each with three executed XCTest methods, zero skips/failures and no reported runtime warnings. Sampled process footprint peaked at 103 MiB. Its fixture repeats one two-frame H.264 pattern across distinct paths; it does not cover codec diversity, long GOPs, large encoded files or distinct camera clips. All preview pixels, source/destination hashes and PDF image placements were checked. The [separate footer fix](validation/report-pdf-footer-2026-10-19.md) passed its regressions and the integrated app suites: normal 20/200-clip reports retain all previews across 2/14 pages instead of 3/15. Oversized identity text remains fully paginated rather than truncated. +- **4K report-preview memory:** the [initial host-only probe](validation/report-preview-memory-2026-10-19.md) passed at 20 and 200 H.264 paths, peaking at 103 MiB. The [explicit-codec extension](validation/report-preview-codecs-2026-10-20.md) also passed actual 4K HEVC and ProRes 422 at 20 paths, plus ten H.264 reports in one process. Those three invocations each passed three methods without skips/failures; their peaks were 103, 143 and 102 MiB. All preview pixels, source/destination hashes and PDF placements were checked. Each codec still repeats one short generated pattern; there is no claim of long-GOP, large encoded-file or camera-footage coverage. The [footer fix](validation/report-pdf-footer-2026-10-19.md) retains normal 20/200-clip reports across 2/14 pages instead of 3/15; oversized identity text remains paginated rather than truncated. - **Intel:** [translated x86_64 engine tests](validation/ROSETTA_ENGINE_20261019.md) passed: XCTest 186 passed/37 skipped, Swift Testing 213 declarations passed/six skipped, zero final failures. Initial unsigned-loader bookmark setup failures remain documented. This is Rosetta coverage, not execution on a physical Intel Mac or older macOS. - **Final revision and cleanup:** update this document after the remaining jobs settle, final combined gates complete, and owned media fixtures, images, caches and simulators are accounted for. Retain compact receipts; preserve uncertain or failed evidence rather than deleting it prematurely. From 0d62cbc5fa35da9190c8a7ca17c215bc266386b0 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 01:24:17 -0400 Subject: [PATCH 48/56] test: cover exact FAT32 single-file boundaries --- .../MountedFAT32BoundaryTests.swift | 285 ++++++++++++++++++ docs/FAT32_BOUNDARY_VALIDATION_20261020.md | 26 ++ 2 files changed, 311 insertions(+) create mode 100644 Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedFAT32BoundaryTests.swift create mode 100644 docs/FAT32_BOUNDARY_VALIDATION_20261020.md diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedFAT32BoundaryTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedFAT32BoundaryTests.swift new file mode 100644 index 00000000..6a6efd8e --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedFAT32BoundaryTests.swift @@ -0,0 +1,285 @@ +#if os(macOS) +import CryptoKit +import Darwin +import Foundation +import XCTest +@testable import BitMatchEngine + +/// Logical-size boundary evidence on a real mounted FAT32 driver. The source +/// is sparse APFS data with sentinels, not representative camera throughput. +final class MountedFAT32BoundaryTests: XCTestCase { + func testLargestFAT32FileCompletesStandardMHLAndSavedCheck() async throws { + try await boundaryCase(oversize: false) + } + + func testFirstOversizeFileFailsClosedAndPreservesPriorHistoryOnRetry() async throws { + try await boundaryCase(oversize: true) + } + + private func boundaryCase(oversize: Bool) async throws { + guard ProcessInfo.processInfo.environment["BITMATCH_FAT32_BOUNDARY"] == "1" else { + throw XCTSkip("Opt in with BITMATCH_FAT32_BOUNDARY=1; writes about 4 GiB per disposable image") + } + try await FileOperationsTestLock.shared.run { + let fixture = try FAT32BoundaryFixture() + do { + try await fixture.exercise(oversize: oversize) + try fixture.detach() + try fixture.removePassedFixture() + } catch { + // Detach normally but retain the image, source and logs for failures. + try? fixture.detach() + print("FAT32_BOUNDARY retained=\(fixture.root.path) error=\(error)") + throw error + } + } + } +} + +private final class FAT32BoundaryFixture { + let root: URL + private let fm = FileManager.default + private let token = UUID().uuidString + private let mount: URL + private let image: URL + private var attached = false + private var rootIdentity: (dev_t, ino_t)? + private var mountDevice: String? + + init() throws { + root = URL(fileURLWithPath: "/private/tmp/bitmatch-fat32-boundary-\(UUID())", isDirectory: true) + mount = root.appendingPathComponent("mount", isDirectory: true) + image = root.appendingPathComponent("boundary.sparseimage") + try fm.createDirectory(at: root, withIntermediateDirectories: false) + var owned = stat() + try require(lstat(root.path, &owned) == 0 && owned.st_mode & S_IFMT == S_IFDIR, "Owned root is not a real directory") + rootIdentity = (owned.st_dev, owned.st_ino) + try Data(token.utf8).write(to: root.appendingPathComponent(".owned"), options: .withoutOverwriting) + var filesystem = statfs() + try require(statfs(root.path, &filesystem) == 0, "Source statfs failed") + let type = withUnsafeBytes(of: filesystem.f_fstypename) { String(decoding: $0.prefix { $0 != 0 }, as: UTF8.self) } + try require(type == "apfs", "Sparse source requires actual APFS") + try fm.createDirectory(at: mount, withIntermediateDirectories: false) + try command(["create", "-quiet", "-size", "7g", "-type", "SPARSE", "-fs", "MS-DOS FAT32", "-volname", "BMFAT", "-o", image.path]) + // Track before attach so a partial attach is never recursively removed. + attached = true + do { + try command(["attach", "-quiet", "-nobrowse", "-mountpoint", mount.path, image.path]) + try require(statfs(mount.path, &filesystem) == 0, "Destination statfs failed") + let actual = withUnsafeBytes(of: filesystem.f_fstypename) { String(decoding: $0.prefix { $0 != 0 }, as: UTF8.self) } + try require(actual == "msdos", "Expected actual mounted FAT32") + mountDevice = withUnsafeBytes(of: filesystem.f_mntfromname) { String(decoding: $0.prefix { $0 != 0 }, as: UTF8.self) } + } catch { + try? detach() + print("FAT32_BOUNDARY setup_retained=\(root.path)") + throw error + } + } + + func exercise(oversize: Bool) async throws { + let card = root.appendingPathComponent("Card") + let target = mount.appendingPathComponent("Backup") + try fm.createDirectory(at: card, withIntermediateDirectories: false) + try fm.createDirectory(at: target, withIntermediateDirectories: false) + let tiny = Data("prior media must survive".utf8) + try tiny.write(to: card.appendingPathComponent("keep.bin")) + let baseline = try await transfer(card, target) + let copied = SafetyValidator.resolvedDestinationRoot(source: card, destination: target, settings: .init()) + try require(baseline.results.count == 1 && baseline.results.allSatisfy(\.success), "Tiny baseline failed") + if oversize { try handoff(baseline, card, target) } + let histories = oversize ? try historyBytes(copied) : [:] + try require(!oversize || !histories.isEmpty, "No baseline history") + let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + if oversize { + let baselineCheck = try await checker.check(checker.discover(under: copied)) { _ in } + try require(baselineCheck.isIntact && baselineCheck.recordedFileCount == 1, "Baseline Saved Check failed") + } + + let size: Int64 = 4_294_967_296 - (oversize ? 0 : 1) + let source = card.appendingPathComponent("limit.bin") + try sparseSource(source, size: size) + let sourceBefore = try digest(source) + let sourceAttributes = try fm.attributesOfItem(atPath: source.path) + if oversize { + let probe = mount.appendingPathComponent("native-probe") + let fd = open(probe.path, O_CREAT | O_EXCL | O_RDWR | O_NOFOLLOW, 0o600) + try require(fd >= 0, "Native probe open failed") + let result = ftruncate(fd, off_t(size)); let observedErrno = errno + close(fd) + try require(result == -1 && observedErrno == EFBIG, "Native FAT32 truncate did not reject 4 GiB with EFBIG; result=\(result) errno=\(observedErrno)") + try fm.removeItem(at: probe) + print("FAT32_BOUNDARY native_size=\(size) errno=\(observedErrno)") + } + let operation = try await transfer(card, target) + try require(operation.results.count == 2, "Boundary result coverage") + let large = try unwrap(operation.results.first { $0.sourceURL.lastPathComponent == "limit.bin" }, "Missing boundary row") + try require(try Data(contentsOf: copied.appendingPathComponent("keep.bin")) == tiny, "Existing media overwritten") + if oversize { try require(try historyBytes(copied) == histories, "Prior history overwritten before handoff") } + if oversize { + let error = try unwrap(large.error, "Missing oversized error") as NSError + try require(!large.success && error.domain == NSPOSIXErrorDomain && error.code == Int(EFBIG), "Engine oversized failure was not EFBIG: \(error)") + let plan = TransferCompletion.ascmhlPlan(results: operation.results, sourceFiles: operation.sourceManifest, destinations: [target], source: card, settings: .init()) + try require(plan.jobs.isEmpty && !plan.issues.isEmpty, "Oversized transfer planned complete MHL") + let verdict = TransferCompletion.verdict(rows: TransferCompletion.rows(from: operation), sourceFiles: operation.sourceManifest, destinations: [target], source: card, settings: .init(), mode: .standard, generateASCMHL: true, handoffIssues: plan.issues, reportIssue: nil, project: .init(didPersist: true, locallySafe: nil)) + try require(!verdict.success, "Oversized transfer claimed a green verdict") + let partial = copied.appendingPathComponent("limit.bin") + let partialBefore = fm.fileExists(atPath: partial.path) ? try digest(partial) : nil + let retry = try await transfer(card, target) + try require(!retry.results.allSatisfy(\.success), "Oversized retry claimed success") + let retryPlan = TransferCompletion.ascmhlPlan(results: retry.results, sourceFiles: retry.sourceManifest, destinations: [target], source: card, settings: .init()) + try require(retryPlan.jobs.isEmpty, "Oversized retry planned complete history") + try require(try historyBytes(copied) == histories, "Oversized retry changed prior history") + if let partialBefore { try require(try digest(partial) == partialBefore, "Retry replaced partial destination bytes") } + let markers = try fm.contentsOfDirectory(at: copied, includingPropertiesForKeys: nil).filter { TransferStatusMarker.hasExactName($0.lastPathComponent) } + try require(markers.contains { TransferStatusMarker.validatedStatus(at: $0, relativePath: $0.lastPathComponent) == .incomplete }, "No durable incomplete marker") + // Saved Check may accurately validate the older tiny history; it must + // not present the unrecorded partial large file as recorded evidence. + let checked = try await checker.check(checker.discover(under: copied)) { _ in } + try require(checked.recordedFileCount == 1 && checked.matchingPaths == ["keep.bin"], "Partial file became trusted history") + if partialBefore != nil { try require(checked.newPaths.contains("limit.bin"), "Partial file absent from unrecorded inventory") } + } else { + try require(large.success && large.verificationResult?.isValid == true, "Exact maximum failed: \(String(describing: large.error))") + try require((try fm.attributesOfItem(atPath: large.destinationURL.path)[.size] as? Int64) == size, "Exact-max destination size changed") + try require(try digest(large.destinationURL) == sourceBefore, "Exact-max destination content differs") + try handoff(operation, card, target) + let checked = try await checker.check(checker.discover(under: copied)) { _ in } + try require(checked.isIntact && checked.recordedFileCount == 2, "Exact-max Saved Check failed") + let published = try historyBytes(copied) + let retry = try await transfer(card, target) + try require(retry.results.count == 2 && retry.results.allSatisfy { $0.success && $0.verificationResult?.isValid == true }, "Identical exact-max retry failed") + try require(try historyBytes(copied) == published, "Identical retry replaced history") + try require(try digest(large.destinationURL) == sourceBefore, "Identical retry changed destination") + } + let sourceAfter = try digest(source) + try require(sourceAfter == sourceBefore, "Sparse source content changed") + let sentinelHandle = try FileHandle(forReadingFrom: source) + defer { try? sentinelHandle.close() } + for (offset, expected): (Int64, UInt8) in [(0, 0x13), (size / 2, 0x59), (size - 1, 0xE7)] { + try sentinelHandle.seek(toOffset: UInt64(offset)) + try require(try sentinelHandle.read(upToCount: 1) == Data([expected]), "Source sentinel changed") + } + // Release before normal image detachment; the source is outside it. + try sentinelHandle.close() + print("FAT32_BOUNDARY SOURCE bytes=\(size) shaBefore=\(sourceBefore) shaAfter=\(sourceAfter) sentinels=13,59,e7") + let finalAttributes = try fm.attributesOfItem(atPath: source.path) + try require(finalAttributes[.size] as? Int64 == size && finalAttributes[.modificationDate] as? Date == sourceAttributes[.modificationDate] as? Date, "Source size or modification date changed") + try require(try Data(contentsOf: card.appendingPathComponent("keep.bin")) == tiny, "Tiny source changed") + try require(Set(try fm.contentsOfDirectory(atPath: card.path)) == ["keep.bin", "limit.bin"], "Source inventory changed") + print("FAT32_BOUNDARY PASS bytes=\(size) oversized=\(oversize) sourcePreserved=true priorHistoryPreserved=true") + } + + private func transfer(_ card: URL, _ target: URL) async throws -> FileOperation { + try await TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared) + .performFileOperation(sourceURL: card, destinationURLs: [target], verificationMode: .standard, + settings: .init(), progressCallback: { _ in }, onFileResult: nil) + } + private func handoff(_ operation: FileOperation, _ card: URL, _ target: URL) throws { + let plan = TransferCompletion.ascmhlPlan(results: operation.results, sourceFiles: operation.sourceManifest, destinations: [target], source: card, settings: .init()) + try require(plan.jobs.count == 1 && plan.issues.isEmpty, "Missing complete handoff") + let issues = try TransferCompletion.writeASCMHL(plan.jobs, planIssues: plan.issues, startTime: operation.startTime, source: card, toolVersion: "fat32-boundary-test") + try require(issues.isEmpty, "MHL publication failed") + } + private func sparseSource(_ url: URL, size: Int64) throws { + let fd = open(url.path, O_CREAT | O_EXCL | O_RDWR | O_NOFOLLOW, 0o600) + try require(fd >= 0, "Sparse source open failed") + defer { close(fd) } + try require(ftruncate(fd, off_t(size)) == 0, "Sparse source truncate failed") + for (offset, value): (Int64, UInt8) in [(0, 0x13), (size / 2, 0x59), (size - 1, 0xE7)] { + var byte = value + try require(pwrite(fd, &byte, 1, off_t(offset)) == 1, "Sparse sentinel write failed") + } + try require(fsync(fd) == 0, "Source fsync failed") + } + private func digest(_ url: URL) throws -> String { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + var hash = SHA256() + while true { + try Task.checkCancellation() + let done = try autoreleasepool { () throws -> Bool in + let data = try handle.read(upToCount: 4 * 1024 * 1024) ?? Data() + if data.isEmpty { return true } + hash.update(data: data) + return false + } + if done { break } + } + return hash.finalize().map { String(format: "%02x", $0) }.joined() + } + private func historyBytes(_ copied: URL) throws -> [String: Data] { + let history = copied.appendingPathComponent("ascmhl") + return try Dictionary(uniqueKeysWithValues: fm.contentsOfDirectory(at: history, includingPropertiesForKeys: nil).map { ($0.lastPathComponent, try Data(contentsOf: $0)) }) + } + func detach() throws { + if mountedPaths().contains(mount.path) { + var current = statfs() + try require(statfs(mount.path, ¤t) == 0, "Cannot identify owned mount") + let device = withUnsafeBytes(of: current.f_mntfromname) { String(decoding: $0.prefix { $0 != 0 }, as: UTF8.self) } + if let mountDevice { try require(device == mountDevice, "Owned mount was replaced; retaining it") } + // Confirm this exact mount still belongs to this exact image, rather + // than detaching another disk which appeared at the same path. + let data = try commandOutput(["info", "-plist"]) + let plist = try PropertyListSerialization.propertyList(from: data, format: nil) as? [String: Any] + let images = plist?["images"] as? [[String: Any]] ?? [] + let owns = images.contains { item in + guard (item["image-path"] as? String) == image.path else { return false } + return (item["system-entities"] as? [[String: Any]] ?? []).contains { $0["mount-point"] as? String == mount.path } + } + try require(owns, "Mount backing image changed; retaining fixture") + try command(["detach", "-quiet", mount.path]) + } + try require(!mountedPaths().contains(mount.path), "Owned image remains mounted") + attached = false + } + func removePassedFixture() throws { + try require(!attached && !mountedPaths().contains { $0 == root.path || $0.hasPrefix(root.path + "/") }, "Mounted fixture must be retained") + var info = stat() + try require(lstat(root.path, &info) == 0 && info.st_mode & S_IFMT == S_IFDIR && rootIdentity?.0 == info.st_dev && rootIdentity?.1 == info.st_ino, "Owned root replaced") + let marker = root.appendingPathComponent(".owned") + var markerInfo = stat() + try require(lstat(marker.path, &markerInfo) == 0 && markerInfo.st_mode & S_IFMT == S_IFREG, "Ownership marker is not a regular file") + try require(try Data(contentsOf: marker) == Data(token.utf8), "Ownership marker replaced") + try fm.removeItem(at: root) + print("FAT32_BOUNDARY cleanup=normal_detach_and_owned_fixture_removed") + } + private func mountedPaths() -> Set { + var entries: UnsafeMutablePointer? + let count = getmntinfo(&entries, MNT_NOWAIT) + guard count > 0, let entries else { return [root.path] } + return Set((0.. Data { + let output = root.appendingPathComponent("hdiutil-info-\(UUID()).plist") + try Data().write(to: output) + let handle = try FileHandle(forWritingTo: output) + defer { try? handle.close() } + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/hdiutil") + process.arguments = arguments + process.standardOutput = handle + try process.run(); process.waitUntilExit() + try require(process.terminationStatus == 0, "Cannot validate mounted backing image") + try handle.synchronize() + return try Data(contentsOf: output) + } + private func command(_ arguments: [String]) throws { + let log = root.appendingPathComponent("hdiutil-\(UUID()).log") + try Data().write(to: log) + let output = try FileHandle(forWritingTo: log) + defer { try? output.close() } + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/hdiutil") + process.arguments = arguments + process.standardOutput = output; process.standardError = output + try process.run(); process.waitUntilExit() + try require(process.terminationStatus == 0, "hdiutil failed; retained log \(log.lastPathComponent)") + } + private func require(_ value: Bool, _ message: String) throws { + if !value { throw NSError(domain: "MountedFAT32Boundary", code: 1, userInfo: [NSLocalizedDescriptionKey: message]) } + } + private func unwrap(_ value: T?, _ message: String) throws -> T { + guard let value else { throw NSError(domain: "MountedFAT32Boundary", code: 2, userInfo: [NSLocalizedDescriptionKey: message]) } + return value + } +} +#endif diff --git a/docs/FAT32_BOUNDARY_VALIDATION_20261020.md b/docs/FAT32_BOUNDARY_VALIDATION_20261020.md new file mode 100644 index 00000000..9448ba61 --- /dev/null +++ b/docs/FAT32_BOUNDARY_VALIDATION_20261020.md @@ -0,0 +1,26 @@ +# FAT32 single-file boundary validation + +These opt-in engine tests exercise the actual macOS FAT32 driver through fresh disposable sparse disk images. They fill a gap left by `testFAT32LargeFileLimitKeepsOtherBackupVerified`, which already checks a 4 GiB + 1 byte file against FAT32 and a successful second backup. + +Run on an APFS host with at least 8 GiB free: + +```sh +BITMATCH_FAT32_BOUNDARY=1 swift test \ + --package-path Packages/BitMatchEngine --jobs 4 \ + --disable-swift-testing --filter MountedFAT32BoundaryTests +``` + +The two cases use separate 7 GiB FAT32 images, mounted under a new marked temporary directory. They never format, eject or write to selected physical drives. An ordinary run without the environment variable skips both tests. + +- **4,294,967,295 bytes:** Standard copy and destination verification, independent source/destination SHA-256 equality, ASC MHL publication, Saved Check for both recorded files, and an identical retry which preserves the published history and destination content. +- **4,294,967,296 bytes:** a native filesystem `ftruncate` probe must reject the size with `EFBIG`; the actual transfer must also fail with `EFBIG`, refuse a complete MHL plan and a green verdict, retain an incomplete transfer marker, and preserve earlier media and history through a retry. If partial destination bytes remain, the retry must preserve them. Saved Check may still verify the earlier, smaller history; the partial file must remain outside that recorded inventory. + +The source files are sparse APFS fixtures with distinct bytes at the beginning, middle and final byte. Full SHA-256, size and modification-date checks establish that the source remains unchanged after the operations. The test deliberately adds the large file only while preparing the owned fixture, before capturing its source baseline. This is logical-size coverage, not evidence of camera-media throughput, real HDD or dock behavior, sudden power loss, or every FAT32 implementation. + +Cleanup normally detaches only the image proven to back the exact owned mount. Successful fixtures are removed after ownership and mount checks. Failed fixtures and command logs are retained, with normal detachment attempted. No force detachment is used. + +## Local result, October 20 + +The final native Apple Silicon Debug run passed both cases with zero failures and zero skips in 68.983 seconds. A preceding corrected run also passed both cases in 71.273 seconds; these are two repeated runs of two unique cases. The default opt-out run skipped exactly two tests with zero failures. Actual source hashes matched before and after each case, all three sentinels survived, and the mount table confirmed no owned image remained mounted. Successful fixtures were removed after normal detachment. + +The initial attempt failed both cases during tiny-fixture setup because the selected backup folder had not been created. Those failures are retained separately and are not passing coverage or product failures. The corrected harness creates that folder before invoking the engine. No production change was needed. From 43cd675e357b75b4598554fb2fab97e39d92b1a8 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 01:37:29 -0400 Subject: [PATCH 49/56] Record large physical transfer and FAT32 boundary results --- docs/HARDWARE_COMPATIBILITY.md | 1 + docs/OVERNIGHT_VALIDATION_20261019.md | 15 ++++++++++++--- docs/TODO.md | 1 + 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/HARDWARE_COMPATIBILITY.md b/docs/HARDWARE_COMPATIBILITY.md index 2c43148d..e6cc0744 100644 --- a/docs/HARDWARE_COMPATIBILITY.md +++ b/docs/HARDWARE_COMPATIBILITY.md @@ -40,5 +40,6 @@ What the code on `main` does for these cases, with where it lives. These are cod | Report | Date | App revision / OS | Source and destinations / filesystems | Connection | Scenario | Outcome | | --- | --- | --- | --- | --- | --- | --- | | [Local SD / two-SSD matrix](OVERNIGHT_VALIDATION_20261019.md#physical-sd-to-two-ssds) | 2026-10-19–20 | Engine `5396f6c5` plus corrected test harness; macOS 27.0.1 | exFAT microSD in an adapter → two separate exFAT SSDs | Built-in SD reader; SSD A reports USB 10 Gb/s, SSD B USB 480 Mb/s | Four modes × pipelining off/on, plus Standard reuse; 134 selected files / 2.15 GB; ASC MHL and Saved Check for verified modes | **Pass:** nine scenarios, source inventory and content hashes unchanged; Quick/exclusion safety limits retained. Concurrent large workload makes timings unsuitable for mode comparisons. Not a power-loss or cable-pull test. | +| [Large SSD transfer](OVERNIGHT_VALIDATION_20261019.md#large-physical-ssd-transfer) | 2026-10-20 | Same earlier engine/test-harness qualification; macOS 27.0.1 | exFAT SSD A → separate exFAT SSD B | Reported USB 10 Gb/s → USB 480 Mb/s | Eight synthetic MOV containers / 119.6 GB; Standard, two requested workers; MHL and Saved Check | **Pass:** one XCTest, zero failures/skips; two large readers observed, 32 MiB sampled engine peak, source inventory/size/content hashes preserved. Earlier overlap prevents a controlled performance comparison. No HDD/dock, GUI-memory or power-loss claim. | Use **Pass**, **Fail**, **Inconclusive**, or **Not tested**. A pass requires the scenario's expected behavior and independent hashes for all outputs reported as successful. Document failures and incomplete runs as well as passes. Use the [report template](HARDWARE_REPORT_TEMPLATE.md) and link the evidence when adding a row. diff --git a/docs/OVERNIGHT_VALIDATION_20261019.md b/docs/OVERNIGHT_VALIDATION_20261019.md index 8e0903d4..3462a927 100644 --- a/docs/OVERNIGHT_VALIDATION_20261019.md +++ b/docs/OVERNIGHT_VALIDATION_20261019.md @@ -46,6 +46,16 @@ The running binary was built from `5396f6c5` plus the corrected test harness, be An earlier physical large-file attempt also had a wrong MD5-versus-SHA-256 harness assertion. Completing its byte work does not make that attempt a pass. It is excluded from the successful physical results. +## Large physical SSD transfer + +The corrected SSD A → SSD B run passed one actual XCTest, zero failures and zero skips, in 14,566.320 seconds. Eight generated MOV containers contained **119,600,000,000 selected media bytes**, including 44.5 GB and 45.3 GB files. Standard verification used two requested workers; two large destination readers were observed. ASC MHL and Saved Check completed, and the final source inventory, sizes and content hashes matched their baseline. The sampled engine transfer/MHL/Check footprint peaked at **32 MiB**, from 11 MiB initially. It is not a GUI-process measurement or a leak proof. + +Copy/verification took approximately 7,682 seconds, MHL 3,264 seconds and Saved Check 3,228 seconds. MHL covered all 119.6 GB of selected media; exFAT's deliberate reread policy remained enabled. Earlier phases overlapped the SD matrix, and the slower connection dominated I/O, so these are observations rather than a controlled performance benchmark. Exclusions still cannot earn a whole-card safety claim. + +Like the SD matrix, this long-running binary predates the final harness guards. Its older receipt says `sourceUnchanged`, meaning the asserted inventory/size/content-hash comparison, not every filesystem attribute. The follow-up controller initially rejected that field because it expected the later `sourceSnapshotCompared` spelling; that controller failure occurred before app trials or HFS+ provisioning and does not invalidate the native passing transfer. Its original failed receipt is retained separately. + +The [FAT32 boundary follow-up](FAT32_BOUNDARY_VALIDATION_20261020.md) also passed two actual XCTest cases without skips or failures: the largest allowed file completed Standard → MHL → Saved Check and retry, while exactly 4 GiB returned `EFBIG` and preserved prior media/history and incomplete evidence. These used sparse APFS sources and fresh host disk images, not physical FAT32 media or materialized camera footage. Initial setup-only failures remain documented. + ## Abrupt worker termination The [process-death matrix](validation/PROCESS_DEATH_RECOVERY_20261019.md) passed 20 scenarios on APFS, journaled HFS+, exFAT and FAT32 images. Each directly spawned worker reached a durable, kernel-stopped boundary and then exited by SIGKILL. Its running journal reopened as interrupted; source hashes and existing bytes were preserved. Incomplete MHL could not pass Check. The positive boundary also matters: a completely published history remained intact and passed a fresh Check, while the unsettled journal stayed interrupted. This is process-death coverage, not a power-loss or OOM-cause reproduction. @@ -80,8 +90,8 @@ The engine gates passed again at `79f07486`, including the soak and private diag | Gate | Completed result | | --- | --- | -| Engine Debug, XCTest | 227 scheduled: 188 passed, 39 opt-in/platform skips, zero failures | -| Engine Release, XCTest | 227 scheduled: 188 passed, 39 opt-in/platform skips, zero failures | +| Engine Debug, XCTest | `0d62cbc5`: 229 scheduled: 188 passed, 41 opt-in/platform skips, zero failures | +| Engine Release, XCTest | `0d62cbc5`: 229 scheduled: 188 passed, 41 opt-in/platform skips, zero failures | | Engine Debug, Swift Testing | 223 declarations in 40 suites: 217 passed, six opt-in skips, zero failures | | Engine Release, Swift Testing | 223 declarations in 40 suites: 217 passed, six opt-in skips, zero failures | | Full Mac app suite | 1,305 logical tests: 1,282 passed, 23 skipped, zero failures; structured and raw UI-warning scans clear | @@ -105,7 +115,6 @@ A preceding post-soak Release run failed two diagnostic-ordering assertions whil ## Pending finalization -- **Large physical transfer:** the corrected 119.6 GB SSD A → SSD B case was still running at this checkpoint. Do not count it as a pass, infer its eventual verdict, or use an earlier flawed-harness attempt as a substitute. - **Application performance:** isolated repeated Standard/Thorough application trials and their counter-aligned measurements need a settled receipt. Raw buffer throughput is not application performance evidence. - **4K report-preview memory:** the [initial host-only probe](validation/report-preview-memory-2026-10-19.md) passed at 20 and 200 H.264 paths, peaking at 103 MiB. The [explicit-codec extension](validation/report-preview-codecs-2026-10-20.md) also passed actual 4K HEVC and ProRes 422 at 20 paths, plus ten H.264 reports in one process. Those three invocations each passed three methods without skips/failures; their peaks were 103, 143 and 102 MiB. All preview pixels, source/destination hashes and PDF placements were checked. Each codec still repeats one short generated pattern; there is no claim of long-GOP, large encoded-file or camera-footage coverage. The [footer fix](validation/report-pdf-footer-2026-10-19.md) retains normal 20/200-clip reports across 2/14 pages instead of 3/15; oversized identity text remains paginated rather than truncated. - **Intel:** [translated x86_64 engine tests](validation/ROSETTA_ENGINE_20261019.md) passed: XCTest 186 passed/37 skipped, Swift Testing 213 declarations passed/six skipped, zero final failures. Initial unsigned-loader bookmark setup failures remain documented. This is Rosetta coverage, not execution on a physical Intel Mac or older macOS. diff --git a/docs/TODO.md b/docs/TODO.md index 5ec67869..0feb5d5a 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -187,6 +187,7 @@ These remain ongoing gates, not optional features. Use disposable data and retai ## Developer chores +- [ ] **Preserve diagnostics when separate writers share a folder.** A private two-store reproduction lost one of 401 records without marking the export incomplete. Production uses one shared store per process; tests now use separate folders. Define ownership or safe merging across writers, then test concurrent recording, rotation and export without exposing names or paths. Test isolation does not fix shared-folder writes, and this reproduction does not establish the cause of a field issue. - [ ] **Avoid repeating completed-card summaries on every queue render.** `QueueSessionPresentation.makeRow` still regroups each completed record's results: repeated O(N) work in the result count, not a demonstrated cause of issue #23. Consider a bounded cache of derived summaries, invalidated by result and verdict revisions; keep live mount/eject state and active progress current. **Acceptance:** amended results, changed verdicts, retry and removal cannot leave stale summaries or promote incomplete evidence to safe. Check many completed cards alongside an active transfer on Mac, iPad and iPhone; preserve the existing fail-closed verdict rules. - [ ] **Tests without permission prompts.** Investigate drive-access prompts that stall Mac tests; prefer owned temporary fixtures and scoped test configuration without weakening release entitlements. The documented DerivedData-outside-Desktop workaround addresses a separate loader stall. A separate close-window test-host crash was fixed in #46; that fix does not settle the permission-prompt audit. - [x] **Buffer benchmark first run.** Source creation and timed reads request `F_NOCACHE`. The [October 19–20 run](OVERNIGHT_VALIDATION_20261019.md) confirmed substantial read traffic in the selected OS storage driver during the first timed read. This does not prove a cold device cache or app-only attribution. Keep the current buffer and verification defaults until controlled end-to-end measurements justify a change. From 216f37aacd903d9bf47e66cf6e500dadee656a1b Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 01:55:28 -0400 Subject: [PATCH 50/56] Record stopped app benchmark setup and retry coverage limits --- docs/FAT32_BOUNDARY_VALIDATION_20261020.md | 2 ++ docs/OVERNIGHT_VALIDATION_20261019.md | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/FAT32_BOUNDARY_VALIDATION_20261020.md b/docs/FAT32_BOUNDARY_VALIDATION_20261020.md index 9448ba61..3ea1e6b9 100644 --- a/docs/FAT32_BOUNDARY_VALIDATION_20261020.md +++ b/docs/FAT32_BOUNDARY_VALIDATION_20261020.md @@ -15,6 +15,8 @@ The two cases use separate 7 GiB FAT32 images, mounted under a new marked tempor - **4,294,967,295 bytes:** Standard copy and destination verification, independent source/destination SHA-256 equality, ASC MHL publication, Saved Check for both recorded files, and an identical retry which preserves the published history and destination content. - **4,294,967,296 bytes:** a native filesystem `ftruncate` probe must reject the size with `EFBIG`; the actual transfer must also fail with `EFBIG`, refuse a complete MHL plan and a green verdict, retain an incomplete transfer marker, and preserve earlier media and history through a retry. If partial destination bytes remain, the retry must preserve them. Saved Check may still verify the earlier, smaller history; the partial file must remain outside that recorded inventory. +The successful retry repeats the copy/verification pipeline and checks that its existing history stayed unchanged; it does not invoke a second MHL publication. The initial successful handoff and the oversized retries' rejected MHL plans are tested separately. + The source files are sparse APFS fixtures with distinct bytes at the beginning, middle and final byte. Full SHA-256, size and modification-date checks establish that the source remains unchanged after the operations. The test deliberately adds the large file only while preparing the owned fixture, before capturing its source baseline. This is logical-size coverage, not evidence of camera-media throughput, real HDD or dock behavior, sudden power loss, or every FAT32 implementation. Cleanup normally detaches only the image proven to back the exact owned mount. Successful fixtures are removed after ownership and mount checks. Failed fixtures and command logs are retained, with normal detachment attempted. No force detachment is used. diff --git a/docs/OVERNIGHT_VALIDATION_20261019.md b/docs/OVERNIGHT_VALIDATION_20261019.md index 3462a927..04cedcd0 100644 --- a/docs/OVERNIGHT_VALIDATION_20261019.md +++ b/docs/OVERNIGHT_VALIDATION_20261019.md @@ -54,6 +54,10 @@ Copy/verification took approximately 7,682 seconds, MHL 3,264 seconds and Saved Like the SD matrix, this long-running binary predates the final harness guards. Its older receipt says `sourceUnchanged`, meaning the asserted inventory/size/content-hash comparison, not every filesystem attribute. The follow-up controller initially rejected that field because it expected the later `sourceSnapshotCompared` spelling; that controller failure occurred before app trials or HFS+ provisioning and does not invalidate the native passing transfer. Its original failed receipt is retained separately. +The corrected controller requires one native passing test with zero failures/skips, one exact known receipt schema and the expected byte/read-concurrency metrics. Seventeen control tests passed, including the real settled log and 46 negative variants. A subsequent launch used a Python runtime without descriptor-relative operations and stopped before creating app artifacts; it resumed with the installed compatible runtime. Ownership and trial-validation checks were retained. + +That application attempt did not reach its first transfer. A stack sample found the test host waiting in the fixture's initial root-folder open, before it created any source or destination fixture. The cause is unconfirmed; native UI automation was unavailable, so a possible OS access dialog could not be inspected. The controller was stopped, its children settled, and the original receipts and partial result bundle were retained. This is an unresolved setup failure, not a passing application benchmark or a demonstrated production-transfer hang. Any independent HFS+ follow-up must record that distinction explicitly. + The [FAT32 boundary follow-up](FAT32_BOUNDARY_VALIDATION_20261020.md) also passed two actual XCTest cases without skips or failures: the largest allowed file completed Standard → MHL → Saved Check and retry, while exactly 4 GiB returned `EFBIG` and preserved prior media/history and incomplete evidence. These used sparse APFS sources and fresh host disk images, not physical FAT32 media or materialized camera footage. Initial setup-only failures remain documented. ## Abrupt worker termination @@ -115,7 +119,7 @@ A preceding post-soak Release run failed two diagnostic-ordering assertions whil ## Pending finalization -- **Application performance:** isolated repeated Standard/Thorough application trials and their counter-aligned measurements need a settled receipt. Raw buffer throughput is not application performance evidence. +- **Application performance:** the repeated Standard/Thorough application attempt stopped during fixture setup, before any transfer. No comparative application-performance result is available. Raw buffer throughput is not application performance evidence. - **4K report-preview memory:** the [initial host-only probe](validation/report-preview-memory-2026-10-19.md) passed at 20 and 200 H.264 paths, peaking at 103 MiB. The [explicit-codec extension](validation/report-preview-codecs-2026-10-20.md) also passed actual 4K HEVC and ProRes 422 at 20 paths, plus ten H.264 reports in one process. Those three invocations each passed three methods without skips/failures; their peaks were 103, 143 and 102 MiB. All preview pixels, source/destination hashes and PDF placements were checked. Each codec still repeats one short generated pattern; there is no claim of long-GOP, large encoded-file or camera-footage coverage. The [footer fix](validation/report-pdf-footer-2026-10-19.md) retains normal 20/200-clip reports across 2/14 pages instead of 3/15; oversized identity text remains paginated rather than truncated. - **Intel:** [translated x86_64 engine tests](validation/ROSETTA_ENGINE_20261019.md) passed: XCTest 186 passed/37 skipped, Swift Testing 213 declarations passed/six skipped, zero final failures. Initial unsigned-loader bookmark setup failures remain documented. This is Rosetta coverage, not execution on a physical Intel Mac or older macOS. - **Final revision and cleanup:** update this document after the remaining jobs settle, final combined gates complete, and owned media fixtures, images, caches and simulators are accounted for. Retain compact receipts; preserve uncertain or failed evidence rather than deleting it prematurely. From 32d62c76c66b36a22ee36f647c731924566cd880 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 02:07:00 -0400 Subject: [PATCH 51/56] Clarify diagnostic limits and Mac drive-check scope --- CHANGELOG.md | 2 +- docs/DIAGNOSTICS.md | 2 +- docs/TODO.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4531257a..a7655251 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. ## [Unreleased] - Check no longer reports preserved Finder metadata as missing when it is listed in the saved checksums. -- Corrected physical-drive checks for selected subfolders, so two folders on one drive count as one backup. +- Mac: Corrected physical-drive checks for selected subfolders, so two folders on one drive count as one backup. - Cleaned up long names and page breaks in PDFs; metadata companions no longer inflate the clip-preview count. Job details now share the page footer instead of creating a nearly empty last page. - Long Master Report details and notes now continue onto another page instead of being cut off. - Reduced extra work when tracking transfers with lots of files. diff --git a/docs/DIAGNOSTICS.md b/docs/DIAGNOSTICS.md index 4f53361f..c1e3edb1 100644 --- a/docs/DIAGNOSTICS.md +++ b/docs/DIAGNOSTICS.md @@ -19,7 +19,7 @@ If the device refuses **every** consent write and deletion at once, a later laun MetricKit delivery for BitMatch's Developer ID/notarized Mac distribution has not yet been demonstrated. A signed Mac pilot is required before treating this as a dependable field diagnostic. This feature does not establish the causes of [issue #10](https://github.com/BitmatchApp/Bitmatch/issues/10) or [issue #23](https://github.com/BitmatchApp/Bitmatch/issues/23). -The transfer-event record keeps at most two 256 KiB files. Export soon after the problem, before later activity replaces the older events. Export also limits its input reads: an oversized, unreadable or unsupported file is listed in `rejectedInputs` by its fixed current/previous slot and reason, while valid events from the other file remain available. Malformed individual records are counted separately in `incompleteRecords`. Either case sets `recordingHadErrors`; an absent history file is normal. These flags never change the result of a transfer. +The transfer-event record keeps at most two 256 KiB files. Export soon after the problem, before later activity replaces the older events. Export also limits its input reads: an oversized, unreadable or unsupported file is listed in `rejectedInputs` by its fixed current/previous slot and reason, while valid events from the other file remain available. Malformed individual records are counted separately in `incompleteRecords`. Either case sets `recordingHadErrors`; an absent history file is normal. A false `recordingHadErrors` means no recording or input error was detected, not that every event was retained. These flags never change the result of a transfer. Screenshots, transfer reports and Check's differences export can contain names and paths. Review those separately before sharing them; they are not the same as the diagnostics export. diff --git a/docs/TODO.md b/docs/TODO.md index 0feb5d5a..17cfd2b1 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -1,6 +1,6 @@ # To do -The working list for BitMatch, reprioritized on October 16 and reviewed on October 19, 2026 against 0.2.9. These are roadmap candidates, not instructions to implement every feature or a promised release scope. The report and transfer-details batch shipped in 0.2.7; see [validation](TODO_REPORTS_VALIDATION.md). When something ships, record it in the [changelog](../CHANGELOG.md) and remove the finished task here. +The working list for BitMatch, reprioritized on October 16 and reviewed on October 20, 2026 against 0.2.9 and the unreleased validation branch. These are roadmap candidates, not instructions to implement every feature or a promised release scope. The report and transfer-details batch shipped in 0.2.7; see [validation](TODO_REPORTS_VALIDATION.md). When something ships, record it in the [changelog](../CHANGELOG.md) and remove the finished task here. **Source media → trustworthy copies → independent verification → accurate evidence → safe-to-erase decision.** @@ -76,7 +76,7 @@ Dependencies matter more than the row number: physical measurements precede a ve - [ ] **Transfer interruption on the reporter's setup ([#10](https://github.com/BitmatchApp/Bitmatch/issues/10)).** Later releases fixed reproduced exFAT handoff, cancellation-state and large-file verification defects, but the reporter's exact interruption has not been reproduced. Correlate any new 0.2.9 diagnostic export, outcome record and macOS termination evidence with the original drive, dock, filesystem and verification phase. Keep a disposable physical-drive reproduction separate from conclusions about the reporter's run. **Acceptance:** establish the cause with a reproduction or decisive diagnostic evidence; otherwise record what was ruled out and keep the issue open. -- [ ] **Reported application-memory exhaustion ([#23](https://github.com/BitmatchApp/Bitmatch/issues/23)).** The 0.2.9 fixes address measured memory growth in ASC MHL generation and saved-checksum Check, but the reporter's version and failing phase remain unknown. Opt-in probes merged in #53 cover 512 MiB APFS copies to one/two destinations and Paranoid folder comparison; they do not reproduce the reported full-app failure. Analyze the existing diagnostics if supplied; measure peak memory through copy, verification, MHL, Check and optional PDF preview work with large disposable files, including a physical destination. Keep the source untouched and avoid asking for another large transfer solely to gather data. **Acceptance:** distinguish a reproduced cause from a plausible fix, verify bounded memory on the affected path, and leave the report open until field evidence supports closing it. +- [ ] **Reported application-memory exhaustion ([#23](https://github.com/BitmatchApp/Bitmatch/issues/23)).** The 0.2.9 fixes address measured memory growth in ASC MHL generation and saved-checksum Check, but the reporter's version and failing phase remain unknown. Opt-in probes merged in #53 cover 512 MiB APFS copies to one/two destinations and Paranoid folder comparison. The [overnight validation](OVERNIGHT_VALIDATION_20261019.md) adds a 119.6 GB physical SSD engine transfer through MHL and Check, 20,000-file app-coordinator runs and synthetic 4K preview/PDF probes; engine and app-host measurements are qualified separately. None reproduces the reported full-app failure. Analyze the existing diagnostics if supplied; measure peak memory through copy, verification, MHL, Check and optional PDF preview work with large disposable files, including a physical destination. Keep the source untouched and avoid asking for another large transfer solely to gather data. **Acceptance:** distinguish a reproduced cause from a plausible fix, verify bounded memory on the affected path, and leave the report open until field evidence supports closing it. ## P1 — Reliability and recovery From 953c779f51fef0277f6f2462e750e73964f6ef01 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 02:36:55 -0400 Subject: [PATCH 52/56] Track archive-based updater requirements follow-up --- docs/TODO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/TODO.md b/docs/TODO.md index 17cfd2b1..f18587b3 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -171,7 +171,7 @@ Remote scope excludes screen sharing, arbitrary filesystem browsing, remote shel - [ ] **Shortcuts support.** App Intents for offloads, preserving the same validation and verdict rules. - [ ] **Cloud destinations, if users ask.** A selected backup destination, not a cloud dashboard. Define the provider's staging, exclusive publication, interruption recovery and independent readback semantics before choosing an API or claiming a verified result. Keep local source provenance, locally verified backup evidence and cloud-object evidence separate; upload acknowledgment or object metadata alone is not a readback. Do not assume that a cloud destination prevents a separate local source reread. - [ ] **Persistent SFTP connection.** Replace one `ssh` process per file if SFTP becomes a headline feature. -- [ ] **macOS 15.0 minimum.** An October 19 build-only investigation passed an unsigned Release build for Apple Silicon and Intel with a 15.0 override; compiler targets, Mach-O minimums and the app's Info.plist all recorded 15.0, with availability checking enabled. The engine already targets 15.0 and Sparkle 2.10.0 requires 12.0. The shipped minimum remains 15.5. Before changing it, run a real 15.0 walkthrough and coordinate the project, docs and `Scripts/publish_appcast.sh` minimum (currently hardcoded to 15.5; derive it from the archive). Do not go below 15. +- [ ] **macOS 15.0 minimum.** An October 19 build-only investigation passed an unsigned Release build for Apple Silicon and Intel with a 15.0 override; compiler targets, Mach-O minimums and the app's Info.plist all recorded 15.0, with availability checking enabled. The engine already targets 15.0 and Sparkle 2.10.0 requires 12.0. The shipped minimum remains 15.5. Before changing it, run a real 15.0 walkthrough and coordinate the project and docs. The separate [appcast follow-up (#62)](https://github.com/BitmatchApp/Bitmatch/pull/62) derives updater requirements from the built archive; ensure that is integrated before lowering the app minimum. Do not go below 15. - [ ] **Liquid Glass pass.** macOS/iOS 26 toolbar, composer, rows and completion, with a material fallback. Check it in person before merging. ## Validation, accessibility and release readiness From b34ea51aee80dc796617f42eafee1a2aa36d22b9 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 03:10:41 -0400 Subject: [PATCH 53/56] Record bounded History scaling checks --- docs/OVERNIGHT_VALIDATION_20261019.md | 3 +- docs/TODO.md | 1 + docs/validation/history-memory-2026-10-20.md | 47 ++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 docs/validation/history-memory-2026-10-20.md diff --git a/docs/OVERNIGHT_VALIDATION_20261019.md b/docs/OVERNIGHT_VALIDATION_20261019.md index 04cedcd0..e26df27b 100644 --- a/docs/OVERNIGHT_VALIDATION_20261019.md +++ b/docs/OVERNIGHT_VALIDATION_20261019.md @@ -107,7 +107,7 @@ The engine gates passed again at `79f07486`, including the soak and private diag Skipped opt-in tests are not counted as passes. Some were enabled in separate runs above; those runs do not turn the whole-suite skip count into zero. -A separate signed-app walkthrough completed a Standard transfer to two physical backups and produced a real PDF with clip previews. Settings import Cancel/Apply and export persistence were also exercised. The final rebuilt app's interactive recheck was interrupted by a closed native automation pipe; the app remained alive and an independent sample showed its main thread idle. That automation failure is neither a passed final walkthrough nor evidence of an app hang. Component captures and simulator builds do not replace physical iPhone/iPad file-provider and background-execution testing. +A separate signed-app walkthrough completed a Standard transfer to two physical backups and produced a real PDF with clip previews. Settings import Cancel/Apply and export persistence were also exercised. Native inspection later recovered briefly on the signed `2c3ac909` archive: Setup and General settings were accessible, and a native export contained exactly the portable-preferences schema. The connection dropped again after selecting the import fixture, before the compact review could be observed. The app remained alive and an independent sample showed its main thread idle. The final compact review, progress handoff and status-menu walkthrough remain unvalidated. That automation failure is neither a passed final walkthrough nor evidence of an app hang. Component captures and simulator builds do not replace physical iPhone/iPad file-provider and background-execution testing. The diagnostics-export hardening had its own **32 passing XCTest cases**, zero skips and a universal mobile build. An oversized-input regression was observed failing before the fix. The tests cover malformed, oversized, unreadable, symlink and FIFO inputs, preserving valid events from the other input and reporting incomplete evidence honestly. This does not identify the cause of issue #23. @@ -120,6 +120,7 @@ A preceding post-soak Release run failed two diagnostic-ordering assertions whil ## Pending finalization - **Application performance:** the repeated Standard/Thorough application attempt stopped during fixture setup, before any transfer. No comparative application-performance result is available. Raw buffer throughput is not application performance evidence. +- **Saved History:** the [host-only scaling probe](validation/history-memory-2026-10-20.md) preserved 20,000, 50,000 and 100,000 synthetic result rows through actual engine journal startup, mutation and reload. The largest worker peaked at a sampled 226 MiB. This does not measure the app's search index or main-actor responsiveness, reproduce a leak, or identify issue #23's cause. Whole-file persistence remains a documented scalability follow-up; no saved evidence was pruned. - **4K report-preview memory:** the [initial host-only probe](validation/report-preview-memory-2026-10-19.md) passed at 20 and 200 H.264 paths, peaking at 103 MiB. The [explicit-codec extension](validation/report-preview-codecs-2026-10-20.md) also passed actual 4K HEVC and ProRes 422 at 20 paths, plus ten H.264 reports in one process. Those three invocations each passed three methods without skips/failures; their peaks were 103, 143 and 102 MiB. All preview pixels, source/destination hashes and PDF placements were checked. Each codec still repeats one short generated pattern; there is no claim of long-GOP, large encoded-file or camera-footage coverage. The [footer fix](validation/report-pdf-footer-2026-10-19.md) retains normal 20/200-clip reports across 2/14 pages instead of 3/15; oversized identity text remains paginated rather than truncated. - **Intel:** [translated x86_64 engine tests](validation/ROSETTA_ENGINE_20261019.md) passed: XCTest 186 passed/37 skipped, Swift Testing 213 declarations passed/six skipped, zero final failures. Initial unsigned-loader bookmark setup failures remain documented. This is Rosetta coverage, not execution on a physical Intel Mac or older macOS. - **Final revision and cleanup:** update this document after the remaining jobs settle, final combined gates complete, and owned media fixtures, images, caches and simulators are accounted for. Retain compact receipts; preserve uncertain or failed evidence rather than deleting it prematurely. diff --git a/docs/TODO.md b/docs/TODO.md index f18587b3..3d637f22 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -188,6 +188,7 @@ These remain ongoing gates, not optional features. Use disposable data and retai ## Developer chores - [ ] **Preserve diagnostics when separate writers share a folder.** A private two-store reproduction lost one of 401 records without marking the export incomplete. Production uses one shared store per process; tests now use separate folders. Define ownership or safe merging across writers, then test concurrent recording, rotation and export without exposing names or paths. Test isolation does not fix shared-folder writes, and this reproduction does not establish the cause of a field issue. +- [ ] **Bound local History loading and persistence.** Journal startup and every mutation currently decode or encode the whole saved array; the app loads it on the main actor, and History eagerly indexes every saved result. The [bounded engine probe](validation/history-memory-2026-10-20.md) preserved up to 100,000 rows with a 226 MiB observed worker peak; it did not measure app search/views or reproduce a leak or issue #23. Measure many saved attempts in the actual app, then design bounded loading/persistence without silently pruning evidence. **Acceptance:** retained row IDs, outcomes, retries and exports survive paging/archive boundaries; malformed or interrupted records stay fail-closed; any derived cache invalidates with evidence revisions. Validate Mac, iPad and iPhone responsiveness and memory separately. - [ ] **Avoid repeating completed-card summaries on every queue render.** `QueueSessionPresentation.makeRow` still regroups each completed record's results: repeated O(N) work in the result count, not a demonstrated cause of issue #23. Consider a bounded cache of derived summaries, invalidated by result and verdict revisions; keep live mount/eject state and active progress current. **Acceptance:** amended results, changed verdicts, retry and removal cannot leave stale summaries or promote incomplete evidence to safe. Check many completed cards alongside an active transfer on Mac, iPad and iPhone; preserve the existing fail-closed verdict rules. - [ ] **Tests without permission prompts.** Investigate drive-access prompts that stall Mac tests; prefer owned temporary fixtures and scoped test configuration without weakening release entitlements. The documented DerivedData-outside-Desktop workaround addresses a separate loader stall. A separate close-window test-host crash was fixed in #46; that fix does not settle the permission-prompt audit. - [x] **Buffer benchmark first run.** Source creation and timed reads request `F_NOCACHE`. The [October 19–20 run](OVERNIGHT_VALIDATION_20261019.md) confirmed substantial read traffic in the selected OS storage driver during the first timed read. This does not prove a cold device cache or app-only attribution. Keep the current buffer and verification defaults until controlled end-to-end measurements justify a change. diff --git a/docs/validation/history-memory-2026-10-20.md b/docs/validation/history-memory-2026-10-20.md new file mode 100644 index 00000000..ea6b3333 --- /dev/null +++ b/docs/validation/history-memory-2026-10-20.md @@ -0,0 +1,47 @@ +# Local History scaling check — October 20, 2026 + +A bounded, host-only probe preserved synthetic histories containing up to 100,000 saved result rows through journal startup, a status update and reload. It did not reproduce runaway memory. The code audit still found whole-file work with no explicit history size limit; this is a scalability follow-up, not evidence of a leak or the cause of issue #23. + +## What ran + +An isolated Release CLI used a private copy of the engine sources from integration commit `953c779f51fef0277f6f2462e750e73964f6ef01`. No production code or retention policy changed. Fixture generation ran in a separate process from measurement. Each case had its own measurement worker; within that worker, initial load, mutation, reload and validation shared the same process. + +Each record contained 1,000 distinct synthetic paths with two destination rows per path. Records began as `.issues` or `.interrupted`, and every row had a failed outcome. Resources referenced three tiny owned host folders, with empty bookmark bytes and no volume/file identities. No media transfer, bookmark resolution, selected-drive access, app host or Xcode test ran. + +The measured operations called the actual `TransferJournal`: + +1. Initialize from the saved journal, including its whole-file recovery rewrite. +2. Cancel one synthetic `.issues` record through the supported journal mutation, preserving its rows. +3. Release the journal and initialize a new instance from the same file. + +Fixture construction, hashing and record validation had separate timing breadcrumbs. Canonical journal bytes had to remain unchanged across terminal-only startup and reload. After each of the three operations, every record and row ID and encoded field had to match deterministic expectations, except the one deliberately changed state, summary and end time. Every outcome remained unsafe. The probe completed 510,000 row-metadata validations across the three cases; all nine host-sentinel hash comparisons matched. All 43 copied engine source hashes remained unchanged. + +## Observed results + +These are single runs on a shared Mac while a separate physical-media validation was active, not controlled performance benchmarks. “Peak” is the external observer's sampled physical footprint of the entire measurement worker, including hashing, validation, Foundation and allocator overhead. + +| Saved records | Result rows | Input JSON bytes | Initial load + rewrite | Status update | Reload + rewrite | Observed worker peak | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 10 | 20,000 | 7,131,876 | 0.090 s | 0.045 s | 0.083 s | 50.08 MiB | +| 25 | 50,000 | 17,829,701 | 0.218 s | 0.117 s | 0.202 s | 115.42 MiB | +| 50 | 100,000 | 35,759,566 | 0.434 s | 0.223 s | 0.427 s | 226.34 MiB | + +All three cases passed. No memory guard or timeout fired. The parent sampled the exact owned worker's `proc_pid_rusage` physical footprint every requested 20 ms, with a 1 GiB threshold and a 120-second timeout per process. Inputs were capped at 64 MiB. The observer could interrupt only its own verified child with SIGINT; it did not use SIGKILL. Sampling can miss short peaks, and neither sampling nor SIGINT provides a hard ceiling on transient memory or synchronous filesystem calls. + +After journal release and a 0.2-second wait, the workers measured approximately 38, 98 and 196 MiB. Those figures include earlier validation and Foundation/allocator caches; they are not steady-state leak measurements. Swift arrays use copy-on-write, so these results do not establish a count of independent deep copies. + +The first measurement attempt was refused before journal access by an overly strict harness path comparison: Foundation standardized an existing temporary path differently from the expected raw path. The guard was corrected to compare the exact private paths while retaining ownership, device and no-symlink checks. The failed receipt was preserved, and the already successful 20,000-row fixture was reused only after its SHA-256 matched. That harness false negative was not a product failure. + +## Separate static audit + +The following findings come from code inspection, not the CLI memory measurements: + +- `TransferJournal.init` reads and decodes the entire history file, then encodes and atomically writes every record, even if no running record needed recovery. Every subsequent mutation also encodes the complete array under the journal lock. There is no explicit record, result-row or history-file byte limit in this path. +- `LocalTransferJournal` invokes startup synchronously on the main actor. The CLI did not measure its effect on app responsiveness. +- History builds a clip-name/path/destination search index over all saved result rows when the view is initialized and when records are published. The CLI did not instantiate or measure this index or the app's views. +- The first-100-result limit in expanded History and `recent(limit:)` are presentation limits. Clearing finished queue cards removes their visible session membership while preserving their saved History evidence. +- Completed queue presentation still groups each selected record's rows to derive summaries. A separate TODO covers avoiding that repeated work. Report preferences and results contain metadata and settings, not embedded PDF or thumbnail bytes. + +The next step is to measure larger saved-history use in the actual app, including search and completed cards beside active work, then design bounded loading and persistence. Any paging, summary cache, archive or explicit retention control must preserve durable evidence, retries and fail-closed verdicts across Mac, iPad and iPhone. This check does not justify silently pruning old results. + +Private receipts retain input sizes and hashes, per-stage times, memory samples, canonical-data checks, the harness source and binary hash, and the initial harness failure. Synthetic CLI coverage does not replace real bookmarks, app layouts, mobile-device testing or investigation of the reporter's workload. From 423aab92e429c23ff8d526aec7eb720503b001b6 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 03:34:00 -0400 Subject: [PATCH 54/56] Preserve legacy row identity in History optimization plans --- docs/TODO.md | 2 +- docs/validation/history-memory-2026-10-20.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/TODO.md b/docs/TODO.md index 3d637f22..37ad6797 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -188,7 +188,7 @@ These remain ongoing gates, not optional features. Use disposable data and retai ## Developer chores - [ ] **Preserve diagnostics when separate writers share a folder.** A private two-store reproduction lost one of 401 records without marking the export incomplete. Production uses one shared store per process; tests now use separate folders. Define ownership or safe merging across writers, then test concurrent recording, rotation and export without exposing names or paths. Test isolation does not fix shared-folder writes, and this reproduction does not establish the cause of a field issue. -- [ ] **Bound local History loading and persistence.** Journal startup and every mutation currently decode or encode the whole saved array; the app loads it on the main actor, and History eagerly indexes every saved result. The [bounded engine probe](validation/history-memory-2026-10-20.md) preserved up to 100,000 rows with a 226 MiB observed worker peak; it did not measure app search/views or reproduce a leak or issue #23. Measure many saved attempts in the actual app, then design bounded loading/persistence without silently pruning evidence. **Acceptance:** retained row IDs, outcomes, retries and exports survive paging/archive boundaries; malformed or interrupted records stay fail-closed; any derived cache invalidates with evidence revisions. Validate Mac, iPad and iPhone responsiveness and memory separately. +- [ ] **Bound local History loading and persistence.** Journal startup and every mutation currently decode or encode the whole saved array; the app loads it on the main actor, and History eagerly indexes every saved result. The [bounded engine probe](validation/history-memory-2026-10-20.md) preserved up to 100,000 rows with a 226 MiB observed worker peak; it did not measure app search/views or reproduce a leak or issue #23. Measure many saved attempts in the actual app, then design bounded loading/persistence without silently pruning evidence. Do not simply skip the startup rewrite: older rows without IDs receive UUIDs during decoding, and the write makes those IDs stable. Detect and preserve required migrations before optimizing that path. **Acceptance:** retained row IDs, outcomes, retries and exports survive paging/archive boundaries; malformed or interrupted records stay fail-closed; any derived cache invalidates with evidence revisions. Validate Mac, iPad and iPhone responsiveness and memory separately. - [ ] **Avoid repeating completed-card summaries on every queue render.** `QueueSessionPresentation.makeRow` still regroups each completed record's results: repeated O(N) work in the result count, not a demonstrated cause of issue #23. Consider a bounded cache of derived summaries, invalidated by result and verdict revisions; keep live mount/eject state and active progress current. **Acceptance:** amended results, changed verdicts, retry and removal cannot leave stale summaries or promote incomplete evidence to safe. Check many completed cards alongside an active transfer on Mac, iPad and iPhone; preserve the existing fail-closed verdict rules. - [ ] **Tests without permission prompts.** Investigate drive-access prompts that stall Mac tests; prefer owned temporary fixtures and scoped test configuration without weakening release entitlements. The documented DerivedData-outside-Desktop workaround addresses a separate loader stall. A separate close-window test-host crash was fixed in #46; that fix does not settle the permission-prompt audit. - [x] **Buffer benchmark first run.** Source creation and timed reads request `F_NOCACHE`. The [October 19–20 run](OVERNIGHT_VALIDATION_20261019.md) confirmed substantial read traffic in the selected OS storage driver during the first timed read. This does not prove a cold device cache or app-only attribution. Keep the current buffer and verification defaults until controlled end-to-end measurements justify a change. diff --git a/docs/validation/history-memory-2026-10-20.md b/docs/validation/history-memory-2026-10-20.md index ea6b3333..edad40a3 100644 --- a/docs/validation/history-memory-2026-10-20.md +++ b/docs/validation/history-memory-2026-10-20.md @@ -42,6 +42,6 @@ The following findings come from code inspection, not the CLI memory measurement - The first-100-result limit in expanded History and `recent(limit:)` are presentation limits. Clearing finished queue cards removes their visible session membership while preserving their saved History evidence. - Completed queue presentation still groups each selected record's rows to derive summaries. A separate TODO covers avoiding that repeated work. Report preferences and results contain metadata and settings, not embedded PDF or thumbnail bytes. -The next step is to measure larger saved-history use in the actual app, including search and completed cards beside active work, then design bounded loading and persistence. Any paging, summary cache, archive or explicit retention control must preserve durable evidence, retries and fail-closed verdicts across Mac, iPad and iPhone. This check does not justify silently pruning old results. +The next step is to measure larger saved-history use in the actual app, including search and completed cards beside active work, then design bounded loading and persistence. A subsequent audit stopped a proposed simple startup-write skip before implementation: `ResultRow` assigns a UUID when an older row has no saved ID, and startup persistence currently makes that ID durable. Any optimization must detect and persist that migration rather than generate different IDs on each launch. Paging, summary caches, archives and explicit retention controls must preserve durable evidence, retries and fail-closed verdicts across Mac, iPad and iPhone. This check does not justify silently pruning old results. Private receipts retain input sizes and hashes, per-stage times, memory samples, canonical-data checks, the harness source and binary hash, and the initial harness failure. Synthetic CLI coverage does not replace real bookmarks, app layouts, mobile-device testing or investigation of the reporter's workload. From e8284631590567ff1757c2137298cccfa60004d7 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 05:16:52 -0400 Subject: [PATCH 55/56] Handle vanished metadata companions during test cleanup --- Scripts/test-physical-drives.py | 14 +- Scripts/test_physical_drive_runner.py | 188 +++++++++++++++++++++++++- 2 files changed, 199 insertions(+), 3 deletions(-) diff --git a/Scripts/test-physical-drives.py b/Scripts/test-physical-drives.py index 121eaf48..bec5c0d4 100755 --- a/Scripts/test-physical-drives.py +++ b/Scripts/test-physical-drives.py @@ -137,11 +137,21 @@ def remove_tree_at(parent_fd, name, identity, device): with os.scandir(fd) as entries: ordered = sorted(entries, key=lambda item: item.name == '.bitmatch-validation-owned.json') for entry in ordered: - item = entry.stat(follow_symlinks=False) + try: + item = entry.stat(follow_symlinks=False) + except FileNotFoundError: + # macOS can remove a companion when its principal is unlinked. + if entry.name.startswith('._') and len(entry.name) > 2: + continue + raise if stat.S_ISDIR(item.st_mode): remove_tree_at(fd, entry.name, (item.st_dev, item.st_ino), device) else: - os.unlink(entry.name, dir_fd=fd) + try: + os.unlink(entry.name, dir_fd=fd) + except FileNotFoundError: + if not (entry.name.startswith('._') and len(entry.name) > 2): + raise named = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) if (named.st_dev, named.st_ino) != identity: raise UnsafeFixture('Fixture name was replaced; retaining replacement.') diff --git a/Scripts/test_physical_drive_runner.py b/Scripts/test_physical_drive_runner.py index 1cf80b55..fe0fe129 100644 --- a/Scripts/test_physical_drive_runner.py +++ b/Scripts/test_physical_drive_runner.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 -"""Mock-only regressions: no drive queries, subprocesses, or media writes.""" +"""Mocks and tiny owned host fixtures; no drive queries or physical-media writes.""" +import contextlib import importlib.util import io +import os from pathlib import Path import subprocess import sys @@ -85,5 +87,189 @@ def test_interruption_and_timeout_are_propagated_after_graceful_child_exit(self) self.assertIsNone(run.child) +class PhysicalFixtureCleanupTests(unittest.TestCase): + marker = '.bitmatch-validation-owned.json' + + @contextlib.contextmanager + def fixture(self): + with tempfile.TemporaryDirectory(prefix='bitmatch-cleanup-test-', dir='/private/tmp') as temporary: + parent = Path(temporary) + root = parent / 'owned' + root.mkdir() + (root / self.marker).write_bytes(b'owned test marker') + parent_fd = os.open(parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + yield parent, root, parent_fd + finally: + os.close(parent_fd) + + def remove(self, root, parent_fd): + item = root.lstat() + runner.remove_tree_at(parent_fd, root.name, (item.st_dev, item.st_ino), item.st_dev) + + def cached_entries(self, root, names): + with os.scandir(root) as entries: + by_name = {entry.name: entry for entry in entries} + context = MagicMock() + context.__enter__.return_value = iter(by_name[name] for name in names) + context.__exit__.return_value = False + return context + + def test_cached_companion_autoremoved_with_principal_is_skipped(self): + with self.fixture() as (_, root, parent_fd): + (root / 'empty.bin').write_bytes(b'') + (root / '._empty.bin').write_bytes(b'generated companion') + entries = self.cached_entries(root, [self.marker, 'empty.bin', '._empty.bin']) + unlink = os.unlink + deleted = [] + + def unlink_with_metadata(name, *, dir_fd): + deleted.append(name) + unlink(name, dir_fd=dir_fd) + if name == 'empty.bin': + # Model macOS removing the companion before its cached stat. + unlink('._empty.bin', dir_fd=dir_fd) + + with patch.object(runner.os, 'scandir', return_value=entries), \ + patch.object(runner.os, 'unlink', side_effect=unlink_with_metadata): + self.remove(root, parent_fd) + self.assertFalse(root.exists()) + self.assertEqual(deleted, ['empty.bin', self.marker]) + + def test_companion_disappearing_between_stat_and_unlink_is_skipped(self): + with self.fixture() as (_, root, parent_fd): + (root / '._clip.bin').write_bytes(b'generated companion') + entries = self.cached_entries(root, [self.marker, '._clip.bin']) + unlink = os.unlink + + def unlink_after_metadata_removal(name, *, dir_fd): + if name == '._clip.bin': + unlink(name, dir_fd=dir_fd) + unlink(name, dir_fd=dir_fd) + + with patch.object(runner.os, 'scandir', return_value=entries), \ + patch.object(runner.os, 'unlink', side_effect=unlink_after_metadata_removal): + self.remove(root, parent_fd) + self.assertFalse(root.exists()) + + def test_root_marker_stays_until_every_nested_child_has_been_removed(self): + with self.fixture() as (_, root, parent_fd): + child = root / 'nested' + child.mkdir() + (child / 'payload').write_bytes(b'tiny host fixture') + (child / self.marker).write_bytes(b'ordinary nested filename') + unlink = os.unlink + root_inode = root.stat().st_ino + saw_root_marker = False + + def checked_unlink(name, *, dir_fd): + nonlocal saw_root_marker + if name == self.marker and os.fstat(dir_fd).st_ino == root_inode: + self.assertEqual([p.name for p in root.iterdir()], [self.marker]) + saw_root_marker = True + unlink(name, dir_fd=dir_fd) + + with patch.object(runner.os, 'unlink', side_effect=checked_unlink): + self.remove(root, parent_fd) + self.assertTrue(saw_root_marker) + self.assertFalse(root.exists()) + + def test_unknown_cached_file_absence_stops_before_marker_removal(self): + with self.fixture() as (_, root, parent_fd): + missing = root / 'unknown.bin' + missing.write_bytes(b'not metadata') + entries = self.cached_entries(root, [self.marker, missing.name]) + missing.unlink() + with patch.object(runner.os, 'scandir', return_value=entries): + with self.assertRaises(FileNotFoundError): + self.remove(root, parent_fd) + self.assertEqual((root / self.marker).read_bytes(), b'owned test marker') + + def test_unknown_file_disappearing_at_unlink_is_not_ignored(self): + with self.fixture() as (_, root, parent_fd): + (root / 'unknown.bin').write_bytes(b'not metadata') + entries = self.cached_entries(root, [self.marker, 'unknown.bin']) + unlink = os.unlink + + def unlink_after_unknown_removal(name, *, dir_fd): + if name == 'unknown.bin': + unlink(name, dir_fd=dir_fd) + unlink(name, dir_fd=dir_fd) + + with patch.object(runner.os, 'scandir', return_value=entries), \ + patch.object(runner.os, 'unlink', side_effect=unlink_after_unknown_removal): + with self.assertRaises(FileNotFoundError): + self.remove(root, parent_fd) + self.assertEqual((root / self.marker).read_bytes(), b'owned test marker') + + def test_bare_dot_underscore_is_not_a_companion_exception(self): + with self.fixture() as (_, root, parent_fd): + unknown = root / '._' + unknown.write_bytes(b'not a companion filename') + entries = self.cached_entries(root, [self.marker, unknown.name]) + unknown.unlink() + with patch.object(runner.os, 'scandir', return_value=entries): + with self.assertRaises(FileNotFoundError): + self.remove(root, parent_fd) + self.assertTrue((root / self.marker).is_file()) + + def test_wrong_directory_identity_or_device_preserves_sentinel(self): + for change in ('inode', 'device'): + with self.subTest(change=change), self.fixture() as (_, root, parent_fd): + sentinel = root / 'sentinel' + sentinel.write_bytes(b'unchanged') + info = root.stat() + identity = (info.st_dev, info.st_ino + (change == 'inode')) + device = info.st_dev + (change == 'device') + with self.assertRaises(runner.UnsafeFixture): + runner.remove_tree_at(parent_fd, root.name, identity, device) + self.assertEqual(sentinel.read_bytes(), b'unchanged') + self.assertTrue((root / self.marker).is_file()) + + def test_root_symlink_is_refused_without_touching_its_target(self): + with self.fixture() as (parent, root, parent_fd): + (root / 'sentinel').write_bytes(b'unchanged') + link = parent / 'link' + link.symlink_to(root, target_is_directory=True) + info = root.stat() + with self.assertRaises(OSError): + runner.remove_tree_at(parent_fd, link.name, (info.st_dev, info.st_ino), info.st_dev) + self.assertEqual((root / 'sentinel').read_bytes(), b'unchanged') + self.assertTrue(link.is_symlink()) + + def test_child_symlink_is_unlinked_without_following_it(self): + with self.fixture() as (parent, root, parent_fd): + outside = parent / 'outside' + outside.mkdir() + sentinel = outside / 'sentinel' + sentinel.write_bytes(b'unchanged') + (root / 'link').symlink_to(outside, target_is_directory=True) + self.remove(root, parent_fd) + self.assertFalse(root.exists()) + self.assertEqual(sentinel.read_bytes(), b'unchanged') + + def test_replaced_named_root_retains_the_replacement(self): + with self.fixture() as (parent, root, parent_fd): + (root / 'payload').write_bytes(b'owned payload') + info = root.stat() + original_stat = os.stat + replaced = False + + def replace_before_named_check(path, *args, **kwargs): + nonlocal replaced + if path == root.name and kwargs.get('dir_fd') == parent_fd and not replaced: + root.rename(parent / 'retained-original') + root.mkdir() + (root / 'sentinel').write_bytes(b'replacement stays') + replaced = True + return original_stat(path, *args, **kwargs) + + with patch.object(runner.os, 'stat', side_effect=replace_before_named_check): + with self.assertRaises(runner.UnsafeFixture): + runner.remove_tree_at(parent_fd, root.name, (info.st_dev, info.st_ino), info.st_dev) + self.assertTrue(replaced) + self.assertEqual((root / 'sentinel').read_bytes(), b'replacement stays') + + if __name__ == '__main__': unittest.main() From 6aef644fd86c22d3047aeec475d71a6b8055f837 Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Tue, 20 Oct 2026 05:32:28 -0400 Subject: [PATCH 56/56] Record completed hardware validation and cleanup --- docs/HARDWARE_COMPATIBILITY.md | 3 +- docs/OVERNIGHT_VALIDATION_20261019.md | 53 ++++++++++++++++++++++++--- docs/TODO.md | 2 + 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/docs/HARDWARE_COMPATIBILITY.md b/docs/HARDWARE_COMPATIBILITY.md index e6cc0744..7df2564e 100644 --- a/docs/HARDWARE_COMPATIBILITY.md +++ b/docs/HARDWARE_COMPATIBILITY.md @@ -1,6 +1,6 @@ # Hardware validation status -The [October 19–20 validation](OVERNIGHT_VALIDATION_20261019.md) records an exFAT SD card copied to two physical exFAT SSDs, plus mounted-image coverage of APFS, HFS+, exFAT and FAT32, including case-sensitive variants. Results apply only to their recorded revision, devices and scenarios. Disk images do not validate physical enclosures, hubs, readers or cables, and neither kind of passing transfer proves power-loss behavior. This is an evidence register, not a certification or a list of unsupported hardware. +The [October 19–20 validation](OVERNIGHT_VALIDATION_20261019.md) records an exFAT SD card copied to two physical exFAT SSDs, a large exFAT SSD-to-SSD transfer, and an exFAT SSD source copied into a journaled HFS+ image backed by the other SSD. Mounted-image coverage also includes APFS, HFS+, exFAT and FAT32, with case-sensitive variants. Results apply only to their recorded revision, devices and scenarios. Disk images do not validate physical enclosures, hubs, readers or cables, and neither kind of passing transfer proves power-loss behavior. This is an evidence register, not a certification or a list of unsupported hardware. | Evidence type | Available workflow | What it establishes | | --- | --- | --- | @@ -41,5 +41,6 @@ What the code on `main` does for these cases, with where it lives. These are cod | --- | --- | --- | --- | --- | --- | --- | | [Local SD / two-SSD matrix](OVERNIGHT_VALIDATION_20261019.md#physical-sd-to-two-ssds) | 2026-10-19–20 | Engine `5396f6c5` plus corrected test harness; macOS 27.0.1 | exFAT microSD in an adapter → two separate exFAT SSDs | Built-in SD reader; SSD A reports USB 10 Gb/s, SSD B USB 480 Mb/s | Four modes × pipelining off/on, plus Standard reuse; 134 selected files / 2.15 GB; ASC MHL and Saved Check for verified modes | **Pass:** nine scenarios, source inventory and content hashes unchanged; Quick/exclusion safety limits retained. Concurrent large workload makes timings unsuitable for mode comparisons. Not a power-loss or cable-pull test. | | [Large SSD transfer](OVERNIGHT_VALIDATION_20261019.md#large-physical-ssd-transfer) | 2026-10-20 | Same earlier engine/test-harness qualification; macOS 27.0.1 | exFAT SSD A → separate exFAT SSD B | Reported USB 10 Gb/s → USB 480 Mb/s | Eight synthetic MOV containers / 119.6 GB; Standard, two requested workers; MHL and Saved Check | **Pass:** one XCTest, zero failures/skips; two large readers observed, 32 MiB sampled engine peak, source inventory/size/content hashes preserved. Earlier overlap prevents a controlled performance comparison. No HDD/dock, GUI-memory or power-loss claim. | +| [Large HFS+ image on an SSD](OVERNIGHT_VALIDATION_20261019.md#large-hfs-image-backed-by-a-physical-ssd) | 2026-10-20 | Engine `32d62c76`, harness SHA-256 `eca0a69f…`; macOS 27.0.1 | Physical exFAT SSD A → journaled HFS+ disk image backed by physical SSD B | Reported USB 10 Gb/s → USB 480 Mb/s | Eight synthetic MOV containers / 119.6 GB, including 44.5/45.3 GB files; Standard, MHL and independent Saved Check | **Pass:** one native XCTest in 10,186.538 s, zero failures/skips; two large readers observed; 11/32 MiB initial/peak engine footprint; source inventory/size/content hashes matched. HFS+ reused permitted readback evidence for MHL; logical coverage is not a physical reread. Not a physical HFS+ HDD, dock, GUI-memory, app-performance or power-loss result. | Use **Pass**, **Fail**, **Inconclusive**, or **Not tested**. A pass requires the scenario's expected behavior and independent hashes for all outputs reported as successful. Document failures and incomplete runs as well as passes. Use the [report template](HARDWARE_REPORT_TEMPLATE.md) and link the evidence when adding a row. diff --git a/docs/OVERNIGHT_VALIDATION_20261019.md b/docs/OVERNIGHT_VALIDATION_20261019.md index e26df27b..35937df7 100644 --- a/docs/OVERNIGHT_VALIDATION_20261019.md +++ b/docs/OVERNIGHT_VALIDATION_20261019.md @@ -1,6 +1,6 @@ # Overnight validation — October 19–20, 2026 -**Draft checkpoint.** This records local validation and its limits, not a release approval or proof that a field issue is fixed. The work started from `d6096442`; individual gate revisions and earlier long-running binaries are qualified below. Final integration and cleanup results still need to be recorded. +This records completed local runs and their limits, not a release approval or proof that a field issue is fixed. The work started from `d6096442`; individual gate revisions, earlier long-running binaries and cleanup are qualified below. [Issue #10](https://github.com/BitmatchApp/Bitmatch/issues/10) and [issue #23](https://github.com/BitmatchApp/Bitmatch/issues/23) remain open. These runs do not establish the reporters' exact failure mechanism. @@ -60,6 +60,14 @@ That application attempt did not reach its first transfer. A stack sample found The [FAT32 boundary follow-up](FAT32_BOUNDARY_VALIDATION_20261020.md) also passed two actual XCTest cases without skips or failures: the largest allowed file completed Standard → MHL → Saved Check and retry, while exactly 4 GiB returned `EFBIG` and preserved prior media/history and incomplete evidence. These used sparse APFS sources and fresh host disk images, not physical FAT32 media or materialized camera footage. Initial setup-only failures remain documented. +## Large HFS+ image backed by a physical SSD + +An independent follow-up passed **one native XCTest, zero failures and zero skips**, in 10,186.538 seconds. It used the same physical exFAT SSD source and a journaled HFS+ disk image backed by the other physical SSD, on its reported USB 480 Mb/s connection. The binary was built from `32d62c76`, with harness SHA-256 `eca0a69f92d85534f7674157b305829c22d4d557373193baea8c14ac8fd60c60`. This is the revision actually run, not a claim that the later combined code was rerun over all 119.6 GB. + +Eight generated MOV containers contained **119,600,000,000 selected media bytes**, including 44.5 GB and 45.3 GB files. Standard copy/verification took 6,505.159 seconds, with two large destination readers observed. ASC MHL finished in 0.041855 seconds and Saved Check in 3,340.573 seconds. The run recorded 19,920 transfer progress updates; the final source inventory, sizes and content hashes matched their baseline. Sampled engine physical footprint began at 11 MiB and peaked at **32 MiB**. These are engine-process observations, not GUI memory, a leak proof or controlled performance comparisons. + +HFS+ permits reuse of the completed destination-readback evidence for MHL. Its 119.6 GB MHL metric is **logical media coverage**, not another physical reread; the short MHL time does not imply a 119.6 GB disk scan. The exFAT reread policy is unchanged. The independent Saved Check still ran. This adds the HFS+ filesystem path over physical SSD storage, not a physical HFS+ HDD, the reporter's dock, cold-cache proof or a reproduction of the unresolved app-benchmark setup failure. That application benchmark remains unvalidated. + ## Abrupt worker termination The [process-death matrix](validation/PROCESS_DEATH_RECOVERY_20261019.md) passed 20 scenarios on APFS, journaled HFS+, exFAT and FAT32 images. Each directly spawned worker reached a durable, kernel-stopped boundary and then exited by SIGKILL. Its running journal reopened as interrupted; source hashes and existing bytes were preserved. Incomplete MHL could not pass Check. The positive boundary also matters: a completely published history remained intact and passed a fresh Check, while the unsettled journal stayed interrupted. This is process-death coverage, not a power-loss or OOM-cause reproduction. @@ -90,7 +98,7 @@ Before that run, the benchmark harness was corrected so each FileHandle read and ## App, engine and simulator gates -The engine gates passed again at `79f07486`, including the soak and private diagnostic-scope additions. Fresh integrated Mac, iPad and iPhone suites passed at `2c3ac909`; the only commit during that app run changed the TODO document. Logical test counts and parameterized case counts are kept separate. +The earlier engine gates passed at `79f07486`, including the soak and private diagnostic-scope additions. Earlier integrated Mac, iPad and iPhone suites passed at `2c3ac909`; the only commit during that app run changed the TODO document. The table below records those earlier checks; the final combined source checks follow afterward. Logical test counts and parameterized case counts are kept separate. | Gate | Completed result | | --- | --- | @@ -107,7 +115,7 @@ The engine gates passed again at `79f07486`, including the soak and private diag Skipped opt-in tests are not counted as passes. Some were enabled in separate runs above; those runs do not turn the whole-suite skip count into zero. -A separate signed-app walkthrough completed a Standard transfer to two physical backups and produced a real PDF with clip previews. Settings import Cancel/Apply and export persistence were also exercised. Native inspection later recovered briefly on the signed `2c3ac909` archive: Setup and General settings were accessible, and a native export contained exactly the portable-preferences schema. The connection dropped again after selecting the import fixture, before the compact review could be observed. The app remained alive and an independent sample showed its main thread idle. The final compact review, progress handoff and status-menu walkthrough remain unvalidated. That automation failure is neither a passed final walkthrough nor evidence of an app hang. Component captures and simulator builds do not replace physical iPhone/iPad file-provider and background-execution testing. +A separate signed-app walkthrough completed a Standard transfer to two physical backups and produced a real PDF with clip previews. Settings import Cancel/Apply and export persistence were also exercised. Native inspection later recovered briefly on the signed `2c3ac909` archive: Setup and General settings were accessible, and a native export contained exactly the portable-preferences schema. The connection dropped again after selecting the import fixture, before the compact review could be observed. The app remained alive and an independent sample showed its main thread idle. The final native compact-review, progress-handoff and status-menu walkthrough remain unvalidated; the hosted review-layout check below is separate. That automation failure is neither a passed final walkthrough nor evidence of an app hang. Component captures and simulator builds do not replace physical iPhone/iPad file-provider and background-execution testing. The diagnostics-export hardening had its own **32 passing XCTest cases**, zero skips and a universal mobile build. An oversized-input regression was observed failing before the fix. The tests cover malformed, oversized, unreadable, symlink and FIFO inputs, preserving valid events from the other input and reporting incomplete evidence honestly. This does not identify the cause of issue #23. @@ -117,12 +125,47 @@ The later diagnostic-scope changes also passed Thread Sanitizer: 15 XCTest metho A preceding post-soak Release run failed two diagnostic-ordering assertions while reading the shared ring. A separate private-directory reproduction confirmed silent record loss between independent store instances; it does not establish which process affected the original test. The regression fixtures now use private diagnostic stores, and the fresh full Debug/Release engine gates above pass. This fixes test isolation, not independent-process writes to one diagnostics file; the failed logs remain retained. -## Pending finalization +### Final combined source checks + +The [filename-safety changes in PR #63](https://github.com/BitmatchApp/Bitmatch/pull/63) and [reviewed-exclusion queue policy in PR #64](https://github.com/BitmatchApp/Bitmatch/pull/64) were tested together at source `9581e48e`. [The queue validation record](https://github.com/BitmatchApp/Bitmatch/blob/6b84651ba615f202427e5331441f95f9a223a00e/docs/validation/queue-expected-exclusions-2026-10-20.md) gives the detailed cases and limits; `6b84651` adds documentation without changing that tested source. All 450 tracked Swift/build-configuration hashes stayed stable through the seven core gates. + +| Final gate | Passed | Skipped | Failed | Count basis | +| --- | ---: | ---: | ---: | --- | +| Engine Debug, XCTest | 205 | 43 | 0 | 248 scheduled methods | +| Engine Debug, Swift Testing | 217 | 6 | 0 | 223 reported tests, 40 suites | +| Engine Release, XCTest | 205 | 43 | 0 | 248 scheduled methods | +| Engine Release, Swift Testing | 217 | 6 | 0 | 223 reported tests, 40 suites | +| Full Mac app suite | 1,301 | 23 | 0 | 1,324 total | +| iPad simulator suite | 75 | 10 | 0 | 85 total | +| iPhone simulator suite | 75 | 10 | 0 | 85 total | + +The 10 new engine queue cases, 19 new app cases and existing Quick queue control passed explicitly. Quick retains its existing queue policy; it cannot earn the new verified-exclusions qualification. Exclusions remain disclosed and cannot earn a whole-card safe-to-erase verdict or automatic eject permission. Both raw and structured runtime-warning checks were clear for the accepted runs; existing compiler/toolchain warnings remain separate. Repeated configurations and earlier baseline results are not extra unique tests, and skipped opt-in cases are not passes. + +Three optional mobile controllers failed auxiliary process-enumeration or settlement checks after process-list queries timed out; their failed receipts remain failed. The iPad suite had completed in the first attempt, and its actual passing result was accepted separately. The iPhone suite passed in a separate bounded direct run on the same source: 75 passed, ten skipped, zero failures and zero expected failures, with stable source hashes and completed clone shutdown. That pass does not relabel the earlier controller outcomes. + +Unsigned iOS-device arm64 and universal Mac arm64/x86_64 Release builds passed. A separate local universal diagnostic archive passed deep, strict signature verification and executable/dSYM UUID matching for both architectures, with existing signing settings and all 450 source/configuration hashes unchanged. It was not exported, installed, notarized or published. A universal archive does not establish native Intel runtime behavior. + +One opt-in production `NSHostingView`/`NSWindow` review-layout test also passed, with zero skips or failures and four inspected PNGs. Nine-change top/middle/bottom captures and a short two-change capture checked compact height, title padding, reachable content and pinned Cancel/Apply buttons. The wrapper initially expected `TEST SUCCEEDED` instead of the actual `TEST EXECUTE SUCCEEDED`; the existing native test result was revalidated without rerunning or changing the test. Its failed wrapper receipt remains retained. This does not validate the native file picker, Cancel/Apply callbacks or applied-settings persistence on the final build. + +## Remaining validation - **Application performance:** the repeated Standard/Thorough application attempt stopped during fixture setup, before any transfer. No comparative application-performance result is available. Raw buffer throughput is not application performance evidence. - **Saved History:** the [host-only scaling probe](validation/history-memory-2026-10-20.md) preserved 20,000, 50,000 and 100,000 synthetic result rows through actual engine journal startup, mutation and reload. The largest worker peaked at a sampled 226 MiB. This does not measure the app's search index or main-actor responsiveness, reproduce a leak, or identify issue #23's cause. Whole-file persistence remains a documented scalability follow-up; no saved evidence was pruned. - **4K report-preview memory:** the [initial host-only probe](validation/report-preview-memory-2026-10-19.md) passed at 20 and 200 H.264 paths, peaking at 103 MiB. The [explicit-codec extension](validation/report-preview-codecs-2026-10-20.md) also passed actual 4K HEVC and ProRes 422 at 20 paths, plus ten H.264 reports in one process. Those three invocations each passed three methods without skips/failures; their peaks were 103, 143 and 102 MiB. All preview pixels, source/destination hashes and PDF placements were checked. Each codec still repeats one short generated pattern; there is no claim of long-GOP, large encoded-file or camera-footage coverage. The [footer fix](validation/report-pdf-footer-2026-10-19.md) retains normal 20/200-clip reports across 2/14 pages instead of 3/15; oversized identity text remains paginated rather than truncated. - **Intel:** [translated x86_64 engine tests](validation/ROSETTA_ENGINE_20261019.md) passed: XCTest 186 passed/37 skipped, Swift Testing 213 declarations passed/six skipped, zero final failures. Initial unsigned-loader bookmark setup failures remain documented. This is Rosetta coverage, not execution on a physical Intel Mac or older macOS. -- **Final revision and cleanup:** update this document after the remaining jobs settle, final combined gates complete, and owned media fixtures, images, caches and simulators are accounted for. Retain compact receipts; preserve uncertain or failed evidence rather than deleting it prematurely. +- **Review branches:** the tested work is in draft [PR #61](https://github.com/BitmatchApp/Bitmatch/pull/61), the independent updater-metadata correction in [#62](https://github.com/BitmatchApp/Bitmatch/pull/62), filename safety in [#63](https://github.com/BitmatchApp/Bitmatch/pull/63), and reviewed-exclusion queue continuation in [#64](https://github.com/BitmatchApp/Bitmatch/pull/64). The stacked order is #61 → #63 → #64; #62 is based directly on main. No merge or release was performed. No run here reproduces a real HDD through the reporter's dock, a USB controller fault, power loss, a forced unplug during active I/O, physical-media failure, or every iOS file-provider/background condition. The successful cases strengthen the tested paths; they do not close issues #10 or #23 without decisive field evidence. + + +## Fixture cleanup + +The two large test images were normally detached, and the three original nonce-marked fixture folders on the authorized SD/SSDs were removed after preserving their compact evidence. An independent post-check found the exact test folders absent and all three original physical volume UUIDs still mounted. No physical drive was formatted or ejected. The last allocated-block sample across the three fixture folders was 497,412,079,616 bytes; mounted image contents were not counted again. This is pre-cleanup accounting, not a measurement of newly freed capacity. + +Cleanup initially stopped on an already-removed AppleDouble companion. The partial receipt was retained; a separate inspection confirmed all original ownership markers and folder identities before a newly authorized continuation. The public test harness now handles that specific missing-companion case. All 17 host regressions passed; the two positive disappearance cases failed against the old helper. Ordinary missing files still stop cleanup, and marker ordering, device/identity checks and symlink protections are unchanged. This is a test-harness correction, not another change to the app transfer path. + +No explicit deletion targeted outside-root companions. They were absent after their principal test folders were removed; their persistence or byte invariance is not claimed. Original top-level filename inventories were not recorded, so there is no whole-volume filename-invariance claim. + +Newly owned build caches were removed only after binaries, symbols and logs were preserved and rechecked. The newest five caches accounted for 5,422,403,584 allocated entry bytes; 494,103,676 regular bytes of test products/logs were retained separately, along with external results and the signed diagnostic archive. These are allocated-entry totals, not unique APFS extents or measured newly freed capacity. Older archives outside this cleanup scope were retained. Temporary review worktrees were removed after their branches were pushed; all 11 pre-existing checkouts, including the primary checkout, were preserved. + +Three earlier owned simulator clones were deleted. The final six are retained and were independently confirmed shut down: an unrelated project build started between the prepared ownership proof and deletion, so the cleanup guard stopped before any of those six was removed. Other devices and their running work were not stopped. This remaining simulator cleanup is separate from the completed physical-drive fixture cleanup. diff --git a/docs/TODO.md b/docs/TODO.md index 37ad6797..e1b93fb4 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -6,6 +6,8 @@ The working list for BitMatch, reprioritized on October 16 and reviewed on Octob Follow evidence and actual user reports. The AppleDouble request [#16](https://github.com/BitmatchApp/Bitmatch/issues/16) is closed; the [Guide](GUIDE.md) describes the current setting, and [the investigation](ISSUE16_APPLEDOUBLE.md) retains its earlier validation. Preserve source protection, no-overwrite guarantees, complete verification and honest verdicts. Core workflows apply to Mac, iPad and iPhone; keep platform limits explicit. See the [thesis](THESIS.md) and [architecture](../ARCHITECTURE.md). +Unreleased review work from October 19–20 is recorded in the [overnight validation](OVERNIGHT_VALIDATION_20261019.md): saved-evidence and report fixes, portable preferences, broader filesystem/physical tests, a reproduced filename crash, and queue continuation after reviewed metadata exclusions. These changes are in draft PRs, with the native release walkthrough and active user reports still open. + ## Ranked work queue This is the order to **investigate or start a bounded slice**, not a promise to ship in this order. It weighs everyday usefulness and safety against implementation difficulty, the blast radius of a mistake, and whether the result can ship and be tested independently. These are judgments, not measured scores. **Quick** means a contained change with local regression coverage; **scoped** means a defined feature or audit; **long** needs design, physical tests, or multiple stages; **demand** needs stronger user evidence. Blast radius means the potential effect on transfers, evidence or platform behavior, not the number of files edited. The detailed acceptance criteria remain in the sections below.