From 14829b816be2bd44c7247a74a3e4605d15960d8f Mon Sep 17 00:00:00 2001 From: Michael Cerisano Date: Sat, 17 Oct 2026 14:04:07 -0400 Subject: [PATCH] Make ASC MHL FAT publication fail closed --- .../BitMatchEngine/ASCMHLGenerator.swift | 355 +++++++++++++++--- .../File/AppleDoubleSelection.swift | 115 +++--- .../BitMatchEngine/SavedChecksumCheck.swift | 44 ++- .../Sources/BitMatchEngine/SharedLogger.swift | 2 +- .../ASCMHLGeneratorTests.swift | 328 ++++++++++++++++ .../Issue10ExFATHandoffTests.swift | 28 ++ .../SavedChecksumCheckTests.swift | 87 +++++ docs/ISSUE10-INVESTIGATION.md | 11 +- docs/TODO.md | 2 +- docs/validation/ascmhl/README.md | 4 +- 10 files changed, 852 insertions(+), 124 deletions(-) diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/ASCMHLGenerator.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/ASCMHLGenerator.swift index ce491f3e..ddf38fd1 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/ASCMHLGenerator.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/ASCMHLGenerator.swift @@ -7,6 +7,26 @@ import Darwin /// the copy-verification algorithm; an independent MD5 digest provides ASC compatibility. /// No directory/root hashes are claimed: only the supplied files are inventoried. public enum ASCMHLGenerator: Sendable { + /// FAT histories retain this status file. Check requires its completion + /// record and verifies it against both ASC files before trusting them. + static let publicationStatusName = "BITMATCH_HISTORY_STATUS.txt" + static let fatManifestPrefix = "0001_BitMatch_FAT_" + static let publicationOpening = Data("BitMatch ASC MHL publication v1\nINCOMPLETE until a valid COMPLETE record follows. Keep the source.\n".utf8) + + static func publicationCompletion(name: String, manifest: Data, chain: Data) -> Data { + let manifestHash = hex(SHA256.hash(data: manifest)) + let chainHash = hex(SHA256.hash(data: chain)) + return Data("COMPLETE \(name) \(manifest.count) \(manifestHash) \(chain.count) \(chainHash)\n".utf8) + } + + private static func isGeneratedFATManifestName(_ name: String) -> Bool { + let match = name.range( + of: #"^0001_BitMatch_FAT_[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{6}Z\.mhl$"#, + options: .regularExpression + ) + return match == name.startIndex.. Void)? var fileSystemType: (@Sendable () -> String?)? var didClaimHistory: (@Sendable () throws -> Void)? + var beforePublishingClaim: (@Sendable () throws -> Void)? + var didWritePublicHistoryFile: (@Sendable (String) throws -> Void)? + var beforePublishingCompletion: (@Sendable () throws -> Void)? + var afterPublishingCompletion: (@Sendable () throws -> Void)? + var beforeExclusiveRename: (@Sendable () throws -> Void)? } public enum GenerationError: LocalizedError, Sendable { @@ -86,7 +111,8 @@ public enum ASCMHLGenerator: Sendable { /// Call only after every expected file in this destination completed verification. /// Reuses digests from a matching pinned readback identity, otherwise re-reads - /// the destination with F_NOCACHE. Publication remains atomic and no-replace. + /// the destination with F_NOCACHE. Exclusive rename is atomic where available; + /// FAT histories carry an incomplete marker until their direct writes are durable. public static func generateInitialHistory( destinationURL: URL, files: [VerifiedFile], startTime: Date, sourceURL: URL? = nil, toolVersion: String, progress: ProgressHandler? = nil, diagnosticRun: UUID? = nil, @@ -267,7 +293,8 @@ public enum ASCMHLGenerator: Sendable { timestamp.locale = Locale(identifier: "en_US_POSIX") timestamp.timeZone = TimeZone(secondsFromGMT: 0) timestamp.dateFormat = "yyyy-MM-dd_HHmmss'Z'" - let name = "0001_BitMatch_\(timestamp.string(from: Date())).mhl" + let fatFamily = ["exfat", "msdos"].contains(destinationFileSystemType?.lowercased() ?? "") + let name = "\(fatFamily ? fatManifestPrefix : "0001_BitMatch_")\(timestamp.string(from: Date())).mhl" let chain = """ @@ -287,67 +314,101 @@ public enum ASCMHLGenerator: Sendable { // All writes and publication are relative to the pinned destination descriptor. // Neither replacing the destination URL nor a staging symlink can redirect writes. try rejectSourceOverlap(rootFD: rootFD, sourceFD: sourceFD, initialSourcePath: initialSourcePath) - let stagingName = ".bitmatch-ascmhl-\(UUID().uuidString)" - guard mkdirat(rootFD, stagingName, 0o700) == 0 else { throw posixError() } - let stageFD = openat(rootFD, stagingName, O_RDONLY | O_DIRECTORY | O_NOFOLLOW) - guard stageFD >= 0 else { - unlinkat(rootFD, stagingName, AT_REMOVEDIR) - throw posixError() - } - var published = false - defer { - if !published { - unlinkat(stageFD, name, 0) - unlinkat(stageFD, "ascmhl_chain.xml", 0) - unlinkat(rootFD, stagingName, AT_REMOVEDIR) - } - close(stageFD) - } - try write(Data(xml.utf8), name: name, directoryFD: stageFD) - try write(Data(chain.utf8), name: "ascmhl_chain.xml", directoryFD: stageFD) - try Task.checkCancellation() - guard lstat(root.path, ¤tIdentity) == 0, - currentIdentity.st_dev == rootIdentity.st_dev, - currentIdentity.st_ino == rootIdentity.st_ino, - (currentIdentity.st_mode & S_IFMT) == S_IFDIR else { - throw GenerationError.changedFile(root.path) - } - try rejectSourceOverlap(rootFD: rootFD, sourceFD: sourceFD, initialSourcePath: initialSourcePath) - if renameatx_np(rootFD, stagingName, rootFD, "ascmhl", UInt32(RENAME_EXCL)) != 0 { - let publicationErrno = errno - SharedLogger.transferEvent(.mhlExclusiveRename, run: diagnosticRun, code: Int(publicationErrno)) - // exFAT rejects exclusive directory rename. Claim an EMPTY directory - // with mkdirat (exclusive even on exFAT), then replace only that claim. - // POSIX directory rename cannot replace a nonempty history. No files - // are copied into the public directory: manifest + chain appear together. - guard publicationErrno == ENOTSUP || publicationErrno == ENOSYS || publicationErrno == EINVAL else { - throw NSError(domain: NSPOSIXErrorDomain, code: Int(publicationErrno)) - } + if fatFamily { + // FAT cannot do an exclusive directory rename. Claim the public + // name once, then write only through its pinned descriptor. No + // staging name is created, renamed over, or deleted on this path. guard mkdirat(rootFD, "ascmhl", 0o700) == 0 else { throw posixError() } - var claim = stat() - guard fstatat(rootFD, "ascmhl", &claim, AT_SYMLINK_NOFOLLOW) == 0 else { throw posixError() } - // On interruption leave the empty reservation as an incomplete history. - // A retry refuses it; cleanup must never delete somebody else's history. + let historyFD = openat(rootFD, "ascmhl", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC) + guard historyFD >= 0 else { throw posixError() } + defer { close(historyFD) } + try synchronizeDirectory(rootFD) + // An interrupted empty reservation also blocks retries; never + // delete it automatically because another writer may now own it. try readHooks?.didClaimHistory?() try Task.checkCancellation() try rejectSourceOverlap(rootFD: rootFD, sourceFD: sourceFD, initialSourcePath: initialSourcePath) - var currentClaim = stat() - guard fstatat(rootFD, "ascmhl", ¤tClaim, AT_SYMLINK_NOFOLLOW) == 0, - currentClaim.st_dev == claim.st_dev, currentClaim.st_ino == claim.st_ino, - (currentClaim.st_mode & S_IFMT) == S_IFDIR else { throw GenerationError.existingHistory } - guard renameat(rootFD, stagingName, rootFD, "ascmhl") == 0 else { + try requireCurrentHistory(rootFD: rootFD, historyFD: historyFD) + try requireHistoryEntries([], directoryFD: historyFD) + try readHooks?.beforePublishingClaim?() + try requireCurrentHistory(rootFD: rootFD, historyFD: historyFD) + + let statusFD = openat(historyFD, publicationStatusName, + O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0o600) + guard statusFD >= 0 else { throw posixError() } + let statusHandle = FileHandle(fileDescriptor: statusFD, closeOnDealloc: true) + defer { try? statusHandle.close() } + try statusHandle.write(contentsOf: publicationOpening) + try PinnedDestinationDirectory.flushToMedium(statusFD) + // The incomplete status and directory name are durable before + // either public ASC file is created. No public name is removed. + try synchronizeDirectory(historyFD) + try synchronizeDirectory(rootFD) + try requireCurrentHistory(rootFD: rootFD, historyFD: historyFD) + + let manifestData = Data(xml.utf8) + let chainData = Data(chain.utf8) + try write(manifestData, name: name, directoryFD: historyFD, durable: true) + try readHooks?.didWritePublicHistoryFile?(name) + try Task.checkCancellation() + try requireCurrentHistory(rootFD: rootFD, historyFD: historyFD) + try write(chainData, name: "ascmhl_chain.xml", directoryFD: historyFD, durable: true) + try readHooks?.didWritePublicHistoryFile?("ascmhl_chain.xml") + try Task.checkCancellation() + try synchronizeDirectory(historyFD) + try requireCurrentHistory(rootFD: rootFD, historyFD: historyFD) + try requireHistoryEntries([publicationStatusName, name, "ascmhl_chain.xml"], directoryFD: historyFD) + try requireWrittenFile(publicationOpening, name: publicationStatusName, directoryFD: historyFD) + try requireWrittenFile(manifestData, name: name, directoryFD: historyFD) + try requireWrittenFile(chainData, name: "ascmhl_chain.xml", directoryFD: historyFD) + try Task.checkCancellation() + try rejectSourceOverlap(rootFD: rootFD, sourceFD: sourceFD, initialSourcePath: initialSourcePath) + try requireCurrentRoot(root, expected: rootIdentity) + try requireCurrentHistory(rootFD: rootFD, historyFD: historyFD) + try readHooks?.beforePublishingCompletion?() + try requireCurrentStatus(directoryFD: historyFD, statusFD: statusFD) + let completion = publicationCompletion(name: name, manifest: manifestData, chain: chainData) + try statusHandle.write(contentsOf: completion) + try PinnedDestinationDirectory.flushToMedium(statusFD) + try synchronizeDirectory(historyFD) + try readHooks?.afterPublishingCompletion?() + try requireWrittenFile(publicationOpening + completion, name: publicationStatusName, directoryFD: historyFD) + try requireCurrentStatus(directoryFD: historyFD, statusFD: statusFD) + try requireHistoryEntries([publicationStatusName, name, "ascmhl_chain.xml"], directoryFD: historyFD) + try requireWrittenFile(manifestData, name: name, directoryFD: historyFD) + try requireWrittenFile(chainData, name: "ascmhl_chain.xml", directoryFD: historyFD) + try requireCurrentRoot(root, expected: rootIdentity) + try requireCurrentHistory(rootFD: rootFD, historyFD: historyFD) + SharedLogger.transferEvent(.mhlClaimDirectWrite, run: diagnosticRun) + } else { + let stagingName = ".bitmatch-ascmhl-\(UUID().uuidString)" + guard mkdirat(rootFD, stagingName, 0o700) == 0 else { throw posixError() } + let stageFD = openat(rootFD, stagingName, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC) + guard stageFD >= 0 else { throw posixError() } + defer { close(stageFD) } + // A failed private stage is left alone: unlinking mutable names + // could remove another writer's files or substituted directory. + try write(Data(xml.utf8), name: name, directoryFD: stageFD) + try write(Data(chain.utf8), name: "ascmhl_chain.xml", directoryFD: stageFD) + try Task.checkCancellation() + try readHooks?.beforeExclusiveRename?() + try requireCurrentRoot(root, expected: rootIdentity) + try rejectSourceOverlap(rootFD: rootFD, sourceFD: sourceFD, initialSourcePath: initialSourcePath) + try requireCurrentNamedDirectory(rootFD: rootFD, name: stagingName, pinnedFD: stageFD) + guard renameatx_np(rootFD, stagingName, rootFD, "ascmhl", UInt32(RENAME_EXCL)) == 0 else { let code = errno - SharedLogger.transferEvent(.mhlClaimRename, run: diagnosticRun, code: Int(code)) + SharedLogger.transferEvent(.mhlExclusiveRename, run: diagnosticRun, code: Int(code)) throw NSError(domain: NSPOSIXErrorDomain, code: Int(code)) } - SharedLogger.transferEvent(.mhlClaimRename, run: diagnosticRun) - } else { + try requireCurrentHistory(rootFD: rootFD, historyFD: stageFD) + try requireHistoryEntries([name, "ascmhl_chain.xml"], directoryFD: stageFD) + try requireWrittenFile(Data(xml.utf8), name: name, directoryFD: stageFD) + try requireWrittenFile(Data(chain.utf8), name: "ascmhl_chain.xml", directoryFD: stageFD) + try requireCurrentRoot(root, expected: rootIdentity) SharedLogger.transferEvent(.mhlExclusiveRename, run: diagnosticRun) } - published = true + try requireCurrentRoot(root, expected: rootIdentity) emit(true) - // Do not remove successfully published contents in the deferred staging cleanup. - // Closing the descriptor remains necessary; names are now owned by the history. return root.appendingPathComponent("ascmhl").appendingPathComponent(name) } @@ -398,6 +459,186 @@ public enum ASCMHLGenerator: Sendable { } } + private static func requireCurrentHistory(rootFD: Int32, historyFD: Int32) throws { + try requireCurrentNamedDirectory(rootFD: rootFD, name: "ascmhl", pinnedFD: historyFD) + } + + private static func requireCurrentNamedDirectory(rootFD: Int32, name: String, pinnedFD: Int32) throws { + var pinned = stat() + var named = stat() + guard fstat(pinnedFD, &pinned) == 0, + fstatat(rootFD, name, &named, AT_SYMLINK_NOFOLLOW) == 0, + pinned.st_dev == named.st_dev, pinned.st_ino == named.st_ino, + (pinned.st_mode & S_IFMT) == S_IFDIR, + (named.st_mode & S_IFMT) == S_IFDIR else { + throw GenerationError.existingHistory + } + } + + private static func requireCurrentRoot(_ root: URL, expected: stat) throws { + var named = stat() + guard lstat(root.path, &named) == 0, + named.st_dev == expected.st_dev, named.st_ino == expected.st_ino, + (named.st_mode & S_IFMT) == S_IFDIR else { + throw GenerationError.changedFile(root.path) + } + } + + private static func requireCurrentStatus(directoryFD: Int32, statusFD: Int32) throws { + var pinned = stat() + var named = stat() + guard fstat(statusFD, &pinned) == 0, + fstatat(directoryFD, publicationStatusName, &named, AT_SYMLINK_NOFOLLOW) == 0, + pinned.st_dev == named.st_dev, pinned.st_ino == named.st_ino, + (pinned.st_mode & S_IFMT) == S_IFREG, + (named.st_mode & S_IFMT) == S_IFREG else { throw GenerationError.existingHistory } + } + + /// Check's reader uses the same exact status format as the writer. A + /// missing, truncated, replaced or mismatching completion stays unsafe. + static func publicationIsComplete(at statusURL: URL) throws -> Bool { + let directoryFD = open(statusURL.deletingLastPathComponent().path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC) + guard directoryFD >= 0 else { return false } + defer { close(directoryFD) } + do { + guard let status = try boundedFile(publicationStatusName, directoryFD: directoryFD, maxBytes: 1024), + status.starts(with: publicationOpening), + let line = String(data: Data(status.dropFirst(publicationOpening.count)), encoding: .utf8), + line.hasSuffix("\n") else { return false } + let fields = line.dropLast().split(separator: " ", omittingEmptySubsequences: false) + guard fields.count == 6, fields[0] == "COMPLETE", + isGeneratedFATManifestName(String(fields[1])), + let manifestSize = Int(fields[2]), manifestSize >= 0, + let chainSize = Int(fields[4]), chainSize >= 0, + fields[3].count == 64, fields[5].count == 64, + fields[3].allSatisfy({ $0.isASCII && $0.isHexDigit }), + fields[5].allSatisfy({ $0.isASCII && $0.isHexDigit }) else { return false } + let manifestName = String(fields[1]) + guard let manifest = try boundedFile(manifestName, directoryFD: directoryFD, maxBytes: EvidenceReader.maxReportBytes), + let chain = try boundedFile("ascmhl_chain.xml", directoryFD: directoryFD, maxBytes: EvidenceReader.maxReportBytes), + manifest.count == manifestSize, chain.count == chainSize else { return false } + return hex(SHA256.hash(data: manifest)) == fields[3].lowercased() + && hex(SHA256.hash(data: chain)) == fields[5].lowercased() + } catch is CancellationError { + throw CancellationError() + } catch { + return false + } + } + + private static func boundedFile(_ name: String, directoryFD: Int32, maxBytes: Int) throws -> Data? { + let fd = openat(directoryFD, name, O_RDONLY | O_NOFOLLOW | O_NONBLOCK | O_CLOEXEC) + guard fd >= 0 else { return nil } + let handle = FileHandle(fileDescriptor: fd, closeOnDealloc: true) + defer { try? handle.close() } + var before = stat() + guard fstat(fd, &before) == 0, (before.st_mode & S_IFMT) == S_IFREG, + before.st_size >= 0, before.st_size <= Int64(maxBytes) else { return nil } + var data = Data() + while data.count < Int(before.st_size) { + try Task.checkCancellation() + let count = min(64 * 1024, Int(before.st_size) - data.count) + let chunk = try autoreleasepool { try handle.read(upToCount: count) ?? Data() } + guard !chunk.isEmpty else { return nil } + data.append(chunk) + } + guard (try handle.read(upToCount: 1) ?? Data()).isEmpty else { return nil } + var after = stat() + var named = stat() + guard fstat(fd, &after) == 0, + fstatat(directoryFD, name, &named, AT_SYMLINK_NOFOLLOW) == 0, + before.st_dev == after.st_dev, before.st_ino == after.st_ino, + before.st_size == after.st_size, + before.st_mtimespec.tv_sec == after.st_mtimespec.tv_sec, + before.st_mtimespec.tv_nsec == after.st_mtimespec.tv_nsec, + before.st_ctimespec.tv_sec == after.st_ctimespec.tv_sec, + before.st_ctimespec.tv_nsec == after.st_ctimespec.tv_nsec, + after.st_dev == named.st_dev, after.st_ino == named.st_ino, + (named.st_mode & S_IFMT) == S_IFREG else { return nil } + return data + } + + private static func requireHistoryEntries(_ expected: Set, directoryFD: Int32) throws { + // A fresh open gives readdir its own offset; dup would share the + // pinned descriptor's offset and later checks could see no entries. + let scanFD = openat(directoryFD, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC) + guard scanFD >= 0 else { throw posixError() } + guard let directory = fdopendir(scanFD) else { + let code = errno + close(scanFD) + throw NSError(domain: NSPOSIXErrorDomain, code: Int(code)) + } + defer { closedir(directory) } + var actual = Set() + errno = 0 + while let entry = readdir(directory) { + let capacity = MemoryLayout.size(ofValue: entry.pointee.d_name) + let name = withUnsafePointer(to: &entry.pointee.d_name) { pointer in + pointer.withMemoryRebound(to: CChar.self, capacity: capacity) { + String(cString: $0) + } + } + if name != "." && name != ".." { actual.insert(name) } + errno = 0 + } + guard errno == 0 else { throw posixError() } + // macOS creates AppleDouble companions on exFAT. A matching name + // alone is insufficient: classify its bounded contents through a + // descriptor opened relative to our pinned public history. + guard expected.isSubset(of: actual) else { throw GenerationError.existingHistory } + for companion in actual.subtracting(expected) { + guard companion.hasPrefix("._"), expected.contains(String(companion.dropFirst(2))) else { + throw GenerationError.existingHistory + } + let fd = openat(directoryFD, companion, O_RDONLY | O_NOFOLLOW | O_CLOEXEC) + guard fd >= 0 else { throw GenerationError.existingHistory } + defer { close(fd) } + var info = stat() + guard fstat(fd, &info) == 0, (info.st_mode & S_IFMT) == S_IFREG, + info.st_size >= 26 else { throw GenerationError.existingHistory } + let valid: Bool + do { + valid = try AppleDoubleSelection.isMetadataOnlyCompanion(fd: fd, size: Int64(info.st_size)) + } catch is CancellationError { + throw CancellationError() + } catch { + throw GenerationError.existingHistory + } + guard valid else { throw GenerationError.existingHistory } + } + } + + private static func synchronizeDirectory(_ fd: Int32) throws { + guard fsync(fd) == 0 else { throw posixError() } + } + + private static func requireWrittenFile(_ expected: Data, name: String, directoryFD: Int32) throws { + let fd = openat(directoryFD, name, O_RDONLY | O_NOFOLLOW | O_NONBLOCK | O_CLOEXEC) + guard fd >= 0 else { throw GenerationError.existingHistory } + let handle = FileHandle(fileDescriptor: fd, closeOnDealloc: true) + defer { try? handle.close() } + var opened = stat() + guard fstat(fd, &opened) == 0, + (opened.st_mode & S_IFMT) == S_IFREG, + opened.st_size == Int64(expected.count) else { throw GenerationError.existingHistory } + var offset = 0 + while offset < expected.count { + try Task.checkCancellation() + let count = min(64 * 1024, expected.count - offset) + let chunk = try autoreleasepool { try handle.read(upToCount: count) ?? Data() } + guard !chunk.isEmpty, + chunk.elementsEqual(expected[offset..<(offset + chunk.count)]) else { + throw GenerationError.existingHistory + } + offset += chunk.count + } + var named = stat() + guard fstatat(directoryFD, name, &named, AT_SYMLINK_NOFOLLOW) == 0, + named.st_dev == opened.st_dev, named.st_ino == opened.st_ino, + named.st_size == opened.st_size, + (named.st_mode & S_IFMT) == S_IFREG else { throw GenerationError.existingHistory } + } + private static func openFile(parts: [String], rootFD: Int32) throws -> Int32 { var directoryFD = dup(rootFD) guard directoryFD >= 0 else { throw posixError() } @@ -418,13 +659,17 @@ public enum ASCMHLGenerator: Sendable { return fd } - private static func write(_ data: Data, name: String, directoryFD: Int32) throws { + private static func write(_ data: Data, name: String, directoryFD: Int32, durable: Bool = false) throws { let fd = openat(directoryFD, name, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, 0o600) guard fd >= 0 else { throw posixError() } let handle = FileHandle(fileDescriptor: fd, closeOnDealloc: true) defer { try? handle.close() } try handle.write(contentsOf: data) - try handle.synchronize() + if durable { + try PinnedDestinationDirectory.flushToMedium(fd) + } else { + try handle.synchronize() + } } private static func rejectExistingHistory(_ root: URL) throws { diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/AppleDoubleSelection.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/AppleDoubleSelection.swift index ee0b63f6..498e6a1a 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/AppleDoubleSelection.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/AppleDoubleSelection.swift @@ -26,61 +26,70 @@ public enum AppleDoubleSelection { let fd = open(entry.url.path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC) guard fd >= 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } defer { close(fd) } - var before = stat() - guard fstat(fd, &before) == 0, (before.st_mode & S_IFMT) == S_IFREG, - Int64(before.st_size) == entry.size else { - throw FileOperationError.unsafeOperation("Source metadata changed while reviewing exclusions. Review the source again.") - } - func read(_ count: Int, at offset: Int64) throws -> [UInt8] { - var buffer = [UInt8](repeating: 0, count: count) - var done = 0 - while done < count { - try Task.checkCancellation() - let received = buffer.withUnsafeMutableBytes { bytes in - pread(fd, bytes.baseAddress!.advanced(by: done), count - done, offset + Int64(done)) - } - if received < 0 && errno == EINTR { continue } - guard received > 0 else { - throw FileOperationError.unsafeOperation("Unable to read source metadata while reviewing exclusions.") - } - done += received + return try isMetadataOnlyCompanion(fd: fd, size: entry.size) + }.sorted { $0.relativePath < $1.relativePath } + } + + /// The same bounded parser is used for source selection and the exact + /// filesystem-created companions beside a directly published FAT history. + /// The caller owns the descriptor, which can be opened relative to a + /// pinned directory without following the companion's pathname. + static func isMetadataOnlyCompanion(fd: Int32, size: Int64) throws -> Bool { + guard size >= 26 else { return false } + var before = stat() + guard fstat(fd, &before) == 0, (before.st_mode & S_IFMT) == S_IFREG, + Int64(before.st_size) == size else { + throw FileOperationError.unsafeOperation("Source metadata changed while reviewing exclusions. Review the source again.") + } + func read(_ count: Int, at offset: Int64) throws -> [UInt8] { + var buffer = [UInt8](repeating: 0, count: count) + var done = 0 + while done < count { + try Task.checkCancellation() + let received = buffer.withUnsafeMutableBytes { bytes in + pread(fd, bytes.baseAddress!.advanced(by: done), count - done, offset + Int64(done)) } - return buffer - } - let header = try read(26, at: 0) - func uint32(_ bytes: [UInt8], _ offset: Int) -> UInt32 { - bytes[offset.. 0, count <= 1024, 26 + count * 12 <= entry.size else { return false } - let records = try read(count * 12, at: 26) - var ids = Set() - var ranges: [(UInt64, UInt64)] = [] - for index in 0..= minimumLength, - ids.insert(id).inserted, - offset >= UInt64(26 + count * 12), offset + length <= UInt64(entry.size) else { return false } - ranges.append((offset, offset + length)) - } - let ordered = ranges.filter { $0.1 > $0.0 }.sorted { $0.0 < $1.0 } - for index in ordered.indices.dropFirst() { - guard ordered[index - 1].1 <= ordered[index].0 else { return false } - } - var after = stat() - guard fstat(fd, &after) == 0, before.st_size == after.st_size, - before.st_mtimespec.tv_sec == after.st_mtimespec.tv_sec, before.st_mtimespec.tv_nsec == after.st_mtimespec.tv_nsec, - before.st_ctimespec.tv_sec == after.st_ctimespec.tv_sec, before.st_ctimespec.tv_nsec == after.st_ctimespec.tv_nsec else { - throw FileOperationError.unsafeOperation("Source metadata changed while reviewing exclusions. Review the source again.") + if received < 0 && errno == EINTR { continue } + guard received > 0 else { + throw FileOperationError.unsafeOperation("Unable to read source metadata while reviewing exclusions.") + } + done += received } - return true - }.sorted { $0.relativePath < $1.relativePath } + return buffer + } + let header = try read(26, at: 0) + func uint32(_ bytes: [UInt8], _ offset: Int) -> UInt32 { + bytes[offset.. 0, count <= 1024, 26 + count * 12 <= size else { return false } + let records = try read(count * 12, at: 26) + var ids = Set() + var ranges: [(UInt64, UInt64)] = [] + for index in 0..= minimumLength, + ids.insert(id).inserted, + offset >= UInt64(26 + count * 12), offset + length <= UInt64(size) else { return false } + ranges.append((offset, offset + length)) + } + let ordered = ranges.filter { $0.1 > $0.0 }.sorted { $0.0 < $1.0 } + for index in ordered.indices.dropFirst() { + guard ordered[index - 1].1 <= ordered[index].0 else { return false } + } + var after = stat() + guard fstat(fd, &after) == 0, before.st_size == after.st_size, + before.st_mtimespec.tv_sec == after.st_mtimespec.tv_sec, before.st_mtimespec.tv_nsec == after.st_mtimespec.tv_nsec, + before.st_ctimespec.tv_sec == after.st_ctimespec.tv_sec, before.st_ctimespec.tv_nsec == after.st_ctimespec.tv_nsec else { + throw FileOperationError.unsafeOperation("Source metadata changed while reviewing exclusions. Review the source again.") + } + return true } public static func review(source: URL) throws -> [String] { diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/SavedChecksumCheck.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/SavedChecksumCheck.swift index cf0df0f6..a36837cd 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/SavedChecksumCheck.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/SavedChecksumCheck.swift @@ -52,11 +52,13 @@ public struct SavedChecksumCheck: Sendable { public let root: URL public let expectedFiles: [String: ExpectedFile] public let records: [Record] + public let publicationStatuses: [URL] - public init(root: URL, expectedFiles: [String: ExpectedFile], records: [Record]) { + public init(root: URL, expectedFiles: [String: ExpectedFile], records: [Record], publicationStatuses: [URL] = []) { self.root = root self.expectedFiles = expectedFiles self.records = records + self.publicationStatuses = publicationStatuses } public var newestRecordDate: Date { records.map(\.date).max() ?? .distantPast } @@ -121,6 +123,7 @@ public struct SavedChecksumCheck: Sendable { let files = try await fileAccess.getFileList(from: root) try Task.checkCancellation() + let statuses = try Self.validPublicationStatuses(in: files, under: canonicalRoot) let reports = files.filter { Self.isBitMatchReport($0, under: canonicalRoot) } var reportEntries: [ExpectedFile] = [] for reportURL in reports { @@ -128,7 +131,7 @@ public struct SavedChecksumCheck: Sendable { reportEntries.append(contentsOf: try Self.readReport(reportURL, root: canonicalRoot)) } if !reportEntries.isEmpty { - return Self.makeDiscovery(root: canonicalRoot, entries: reportEntries) + return Self.makeDiscovery(root: canonicalRoot, entries: reportEntries, publicationStatuses: statuses) } let histories = files.filter { Self.isMHLHistory($0, under: canonicalRoot) } @@ -141,7 +144,7 @@ public struct SavedChecksumCheck: Sendable { mhlEntries.append(contentsOf: try Self.readMHL(historyURL, root: canonicalRoot)) } guard !mhlEntries.isEmpty else { throw CheckError.notFound } - return Self.makeDiscovery(root: canonicalRoot, entries: mhlEntries) + return Self.makeDiscovery(root: canonicalRoot, entries: mhlEntries, publicationStatuses: statuses) } /// Re-reads every recorded file, reports ordered progress, and identifies @@ -155,6 +158,11 @@ public struct SavedChecksumCheck: Sendable { defer { if scoped { fileAccess.stopAccessing(url: root) } } let files = try await fileAccess.getFileList(from: root) + try Task.checkCancellation() + let statuses = try Self.validPublicationStatuses(in: files, under: root) + guard statuses == discovery.publicationStatuses else { + throw CheckError.unreadableEvidence + } let resolver = RelativePathResolver(base: root) var actual: [String: URL] = [:] actual.reserveCapacity(files.count) @@ -195,6 +203,11 @@ public struct SavedChecksumCheck: Sendable { } try Task.checkCancellation() + let finalFiles = try await fileAccess.getFileList(from: root) + try Task.checkCancellation() + guard try Self.validPublicationStatuses(in: finalFiles, under: root) == discovery.publicationStatuses else { + throw CheckError.unreadableEvidence + } let newPaths = Set(actual.keys).subtracting(expected.keys).sorted() let used = Set(expected.values.map(\.record.url)) return Result( @@ -379,6 +392,27 @@ public struct SavedChecksumCheck: Sendable { return relative.split(separator: "/").contains { $0.caseInsensitiveCompare("ascmhl") == .orderedSame } } + private static func isPublicationStatus(_ url: URL, under root: URL) -> Bool { + guard url.lastPathComponent.caseInsensitiveCompare(ASCMHLGenerator.publicationStatusName) == .orderedSame, + let relative = try? RelativePathResolver(base: root).resolve(url) else { return false } + let parts = relative.split(separator: "/") + return parts.count >= 2 && parts[parts.count - 2].caseInsensitiveCompare("ascmhl") == .orderedSame + } + + private static func validPublicationStatuses(in files: [URL], under root: URL) throws -> [URL] { + let statuses = files.filter { isPublicationStatus($0, under: root) }.sorted { $0.path < $1.path } + let statusDirectories = Set(statuses.map { $0.deletingLastPathComponent().path }) + let fatDirectories = Set(files.filter { url in + url.lastPathComponent.lowercased().hasPrefix(ASCMHLGenerator.fatManifestPrefix.lowercased()) + && isMHLHistory(url, under: root) + }.map { $0.deletingLastPathComponent().path }) + guard fatDirectories.isSubset(of: statusDirectories) else { throw CheckError.unreadableEvidence } + for status in statuses { + guard try ASCMHLGenerator.publicationIsComplete(at: status) else { throw CheckError.unreadableEvidence } + } + return statuses + } + private static func isMHLMetadataCompanion(_ url: URL) throws -> Bool { guard url.lastPathComponent.hasPrefix("._") else { return false } guard let size = try url.resourceValues(forKeys: [.fileSizeKey]).fileSize else { @@ -517,7 +551,7 @@ public struct SavedChecksumCheck: Sendable { } } - private static func makeDiscovery(root: URL, entries: [ExpectedFile]) -> Discovery { + private static func makeDiscovery(root: URL, entries: [ExpectedFile], publicationStatuses: [URL]) -> Discovery { let sorted = entries.sorted { if $0.record.date != $1.record.date { return $0.record.date < $1.record.date } return $0.record.url.path < $1.record.url.path @@ -532,7 +566,7 @@ public struct SavedChecksumCheck: Sendable { if $0.date != $1.date { return $0.date > $1.date } return $0.url.path < $1.url.path } - return Discovery(root: root, expectedFiles: expected, records: records) + return Discovery(root: root, expectedFiles: expected, records: records, publicationStatuses: publicationStatuses) } private static func progress(processed: Int, total: Int, path: String?) -> OperationProgress { diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/SharedLogger.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/SharedLogger.swift index ac84f932..624496bb 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/SharedLogger.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/SharedLogger.swift @@ -40,7 +40,7 @@ public enum SharedLogger: Sendable { extension SharedLogger { public enum TransferEvent: String, Codable, Sendable { case admitted, copyDrained, verificationDrained, pipelineDrained, authoritativeAccepted, authoritativeRefused - case mhlStarted, mhlFinished, mhlExclusiveRename, mhlClaimRename + case mhlStarted, mhlFinished, mhlExclusiveRename, mhlClaimRename, mhlClaimDirectWrite case reportStarted, reportFinished, explicitCancel, parentCancelled case phase, cancelOrigin, error, pipelineResult, config, destination, progress, appStarted, appEnded case terminalError, journalCancelled, journalInterrupted, journalFinished, phaseChanged diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ASCMHLGeneratorTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ASCMHLGeneratorTests.swift index 63b50175..8ec711ce 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ASCMHLGeneratorTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ASCMHLGeneratorTests.swift @@ -99,6 +99,327 @@ final class ASCMHLGeneratorTests: XCTestCase { XCTAssertEqual(try Data(contentsOf: chain), originalChain) } + func testFallbackDoesNotReplaceAnEmptyHistorySwappedAfterClaimValidation() throws { + let (root, file) = try fixture() + let history = root.appendingPathComponent("ascmhl", isDirectory: true) + let displacedClaim = root.appendingPathComponent("displaced-claim", isDirectory: true) + XCTAssertThrowsError(try ASCMHLGenerator.generateInitialHistory( + destinationURL: root, files: [file], startTime: Date(), toolVersion: "test", + readHooks: .init( + fileSystemType: { "exfat" }, + beforePublishingClaim: { + try FileManager.default.moveItem(at: history, to: displacedClaim) + try FileManager.default.createDirectory(at: history, withIntermediateDirectories: false) + } + ) + )) + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: history.path), [], + "The substituted empty directory must not be replaced by a finished history") + XCTAssertTrue(FileManager.default.fileExists(atPath: displacedClaim.path)) + XCTAssertFalse(FileManager.default.fileExists(atPath: history.appendingPathComponent("ascmhl_chain.xml").path)) + } + + func testDirectFATFallbackPublishesACompleteCheckableHistory() async throws { + let (root, file) = try fixture() + let manifest = try ASCMHLGenerator.generateInitialHistory( + destinationURL: root, files: [file], startTime: Date(), toolVersion: "test", + readHooks: .init(fileSystemType: { "exfat" }) + ) + let history = manifest.deletingLastPathComponent() + XCTAssertTrue(FileManager.default.fileExists(atPath: manifest.path)) + XCTAssertTrue(FileManager.default.fileExists(atPath: history.appendingPathComponent("ascmhl_chain.xml").path)) + let status = history.appendingPathComponent(ASCMHLGenerator.publicationStatusName) + XCTAssertTrue(FileManager.default.fileExists(atPath: status.path)) + XCTAssertTrue(try ASCMHLGenerator.publicationIsComplete(at: status)) + XCTAssertFalse(try FileManager.default.contentsOfDirectory(atPath: root.path).contains { $0.hasPrefix(".bitmatch-ascmhl-") }) + let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + let discovery = try await checker.discover(under: root) + let result = try await checker.check(discovery) { _ in } + XCTAssertTrue(result.isIntact) + XCTAssertEqual(result.recordedFileCount, 1) + } + + func testInterruptedDirectFATWritesStayMarkedAndUncheckable() async throws { + for stopAfterChain in [false, true] { + let (root, file) = try fixture() + let media = try Data(contentsOf: root.appendingPathComponent("clip.txt")) + let history = root.appendingPathComponent("ascmhl", isDirectory: true) + do { + _ = try ASCMHLGenerator.generateInitialHistory( + destinationURL: root, files: [file], startTime: Date(), toolVersion: "test", + readHooks: .init( + fileSystemType: { "exfat" }, + didWritePublicHistoryFile: { name in + if (name == "ascmhl_chain.xml") == stopAfterChain { throw CancellationError() } + } + ) + ) + XCTFail("An interrupted public history must not be reported as published") + } catch is CancellationError { + // The warning stays even when both evidence files have been written. + } + let contents = try FileManager.default.contentsOfDirectory(atPath: history.path) + XCTAssertTrue(contents.contains(ASCMHLGenerator.publicationStatusName)) + XCTAssertFalse(try ASCMHLGenerator.publicationIsComplete(at: + history.appendingPathComponent(ASCMHLGenerator.publicationStatusName))) + XCTAssertEqual(contents.contains("ascmhl_chain.xml"), stopAfterChain) + XCTAssertEqual(try Data(contentsOf: root.appendingPathComponent("clip.txt")), media) + let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + do { + _ = try await checker.discover(under: root) + XCTFail("Check must refuse a marked history") + } catch let error as SavedChecksumCheck.CheckError { + XCTAssertEqual(error, .unreadableEvidence) + } + XCTAssertThrowsError(try ASCMHLGenerator.generateInitialHistory( + destinationURL: root, files: [file], startTime: Date(), toolVersion: "test" + )) + } + } + + func testMovedDestinationRootLeavesIncompleteHistoryAndCannotReportSuccess() throws { + let (root, file) = try fixture() + let movedRoot = root.deletingLastPathComponent().appendingPathComponent("moved-mhl-root-\(UUID())") + addTeardownBlock { try? FileManager.default.removeItem(at: movedRoot) } + XCTAssertThrowsError(try ASCMHLGenerator.generateInitialHistory( + destinationURL: root, files: [file], startTime: Date(), toolVersion: "test", + readHooks: .init( + fileSystemType: { "exfat" }, + didWritePublicHistoryFile: { name in + guard name == "ascmhl_chain.xml" else { return } + try FileManager.default.moveItem(at: root, to: movedRoot) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: false) + } + ) + )) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("ascmhl").path)) + XCTAssertTrue(FileManager.default.fileExists(atPath: + movedRoot.appendingPathComponent("ascmhl").appendingPathComponent(ASCMHLGenerator.publicationStatusName).path)) + } + + func testSubstitutedPrivateStageIsNeverRemovedByCleanup() throws { + let (root, file) = try fixture() + let movedStage = root.appendingPathComponent("moved-stage", isDirectory: true) + XCTAssertThrowsError(try ASCMHLGenerator.generateInitialHistory( + destinationURL: root, files: [file], startTime: Date(), toolVersion: "test", + readHooks: .init( + fileSystemType: { "apfs" }, + beforeExclusiveRename: { + let stageName = try XCTUnwrap(FileManager.default.contentsOfDirectory(atPath: root.path) + .first { $0.hasPrefix(".bitmatch-ascmhl-") }) + let stage = root.appendingPathComponent(stageName, isDirectory: true) + try FileManager.default.moveItem(at: stage, to: movedStage) + try FileManager.default.createDirectory(at: stage, withIntermediateDirectories: false) + throw CancellationError() + } + ) + )) { XCTAssertTrue($0 is CancellationError) } + let substituteName = try XCTUnwrap(FileManager.default.contentsOfDirectory(atPath: root.path) + .first { $0.hasPrefix(".bitmatch-ascmhl-") }) + XCTAssertEqual(try FileManager.default.contentsOfDirectory( + atPath: root.appendingPathComponent(substituteName).path), []) + XCTAssertTrue(FileManager.default.fileExists(atPath: movedStage.path)) + } + + func testUnexpectedPublicHistoryEntryKeepsIncompleteMarker() throws { + let (root, file) = try fixture() + let history = root.appendingPathComponent("ascmhl", isDirectory: true) + let other = history.appendingPathComponent("other.mhl") + XCTAssertThrowsError(try ASCMHLGenerator.generateInitialHistory( + destinationURL: root, files: [file], startTime: Date(), toolVersion: "test", + readHooks: .init( + fileSystemType: { "exfat" }, + didWritePublicHistoryFile: { name in + if name == "ascmhl_chain.xml" { try Data("preserve".utf8).write(to: other) } + } + ) + )) + XCTAssertEqual(try String(contentsOf: other, encoding: .utf8), "preserve") + XCTAssertTrue(FileManager.default.fileExists(atPath: + history.appendingPathComponent(ASCMHLGenerator.publicationStatusName).path)) + } + + func testInvalidAppleDoubleLookalikeCannotAuthorizeFATCompletion() throws { + let (root, file) = try fixture() + let history = root.appendingPathComponent("ascmhl", isDirectory: true) + XCTAssertThrowsError(try ASCMHLGenerator.generateInitialHistory( + destinationURL: root, files: [file], startTime: Date(), toolVersion: "test", + readHooks: .init( + fileSystemType: { "exfat" }, + didWritePublicHistoryFile: { name in + guard name == "ascmhl_chain.xml" else { return } + let manifest = try XCTUnwrap(FileManager.default.contentsOfDirectory(atPath: history.path) + .first { $0.hasSuffix(".mhl") }) + try Data("not AppleDouble".utf8).write(to: history.appendingPathComponent("._" + manifest)) + } + ) + )) + let status = history.appendingPathComponent(ASCMHLGenerator.publicationStatusName) + XCTAssertFalse(try ASCMHLGenerator.publicationIsComplete(at: status)) + let manifest = try XCTUnwrap(FileManager.default.contentsOfDirectory(atPath: history.path) + .first { $0.hasSuffix(".mhl") }) + XCTAssertEqual(try Data(contentsOf: history.appendingPathComponent("._" + manifest)), + Data("not AppleDouble".utf8)) + } + + func testStatusSubstitutionCannotBeDeletedOrCompleted() throws { + let (root, file) = try fixture() + let history = root.appendingPathComponent("ascmhl", isDirectory: true) + let status = history.appendingPathComponent(ASCMHLGenerator.publicationStatusName) + let movedStatus = root.appendingPathComponent("moved-status.txt") + XCTAssertThrowsError(try ASCMHLGenerator.generateInitialHistory( + destinationURL: root, files: [file], startTime: Date(), toolVersion: "test", + readHooks: .init( + fileSystemType: { "exfat" }, + beforePublishingCompletion: { + try FileManager.default.moveItem(at: status, to: movedStatus) + try Data("other writer's status".utf8).write(to: status) + } + ) + )) + XCTAssertEqual(try Data(contentsOf: status), Data("other writer's status".utf8)) + XCTAssertTrue(FileManager.default.fileExists(atPath: movedStatus.path)) + } + + func testChangedChainAtCompletionNeverReportsPublication() async throws { + for afterCompletion in [false, true] { + let (root, file) = try fixture() + let history = root.appendingPathComponent("ascmhl", isDirectory: true) + let chain = history.appendingPathComponent("ascmhl_chain.xml") + let progress = MHLPublicationProbe() + let changeChain: @Sendable () throws -> Void = { + try Data("another writer changed the chain".utf8).write(to: chain) + } + XCTAssertThrowsError(try ASCMHLGenerator.generateInitialHistory( + destinationURL: root, files: [file], startTime: Date(), toolVersion: "test", + readHooks: .init( + fileSystemType: { "exfat" }, + beforePublishingCompletion: afterCompletion ? nil : changeChain, + afterPublishingCompletion: afterCompletion ? changeChain : nil + ), + progress: { progress.record($0.published) } + )) + XCTAssertFalse(progress.published, "No complete progress event may escape after a chain change") + let status = history.appendingPathComponent(ASCMHLGenerator.publicationStatusName) + XCTAssertFalse(try ASCMHLGenerator.publicationIsComplete(at: status)) + let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + do { + _ = try await checker.discover(under: root) + XCTFail("Check must reject a changed chain") + } catch let error as SavedChecksumCheck.CheckError { + XCTAssertEqual(error, .unreadableEvidence) + } + } + } + + func testSubstitutedPrivateStageCannotReportPublication() throws { + let (root, file) = try fixture() + let movedStage = root.appendingPathComponent("moved-stage", isDirectory: true) + let progress = MHLPublicationProbe() + XCTAssertThrowsError(try ASCMHLGenerator.generateInitialHistory( + destinationURL: root, files: [file], startTime: Date(), toolVersion: "test", + readHooks: .init( + fileSystemType: { "apfs" }, + beforeExclusiveRename: { + let stageName = try XCTUnwrap(FileManager.default.contentsOfDirectory(atPath: root.path) + .first { $0.hasPrefix(".bitmatch-ascmhl-") }) + let stage = root.appendingPathComponent(stageName, isDirectory: true) + try FileManager.default.moveItem(at: stage, to: movedStage) + try FileManager.default.createDirectory(at: stage, withIntermediateDirectories: false) + try Data("foreign".utf8).write(to: stage.appendingPathComponent("keep.txt")) + } + ), + progress: { progress.record($0.published) } + )) + XCTAssertFalse(progress.published) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("ascmhl").path)) + XCTAssertTrue(FileManager.default.fileExists(atPath: movedStage.appendingPathComponent("ascmhl_chain.xml").path)) + let substituteName = try XCTUnwrap(FileManager.default.contentsOfDirectory(atPath: root.path) + .first { $0.hasPrefix(".bitmatch-ascmhl-") }) + XCTAssertEqual(try Data(contentsOf: root.appendingPathComponent(substituteName).appendingPathComponent("keep.txt")), + Data("foreign".utf8)) + } + + func testRemovingStatusAfterPartialFATManifestStillCannotPassCheck() async throws { + let (root, file) = try fixture() + let history = root.appendingPathComponent("ascmhl", isDirectory: true) + XCTAssertThrowsError(try ASCMHLGenerator.generateInitialHistory( + destinationURL: root, files: [file], startTime: Date(), toolVersion: "test", + readHooks: .init( + fileSystemType: { "exfat" }, + didWritePublicHistoryFile: { name in + if name.hasSuffix(".mhl") { throw CancellationError() } + } + ) + )) + try FileManager.default.removeItem(at: history.appendingPathComponent(ASCMHLGenerator.publicationStatusName)) + let manifest = try XCTUnwrap(FileManager.default.contentsOfDirectory(atPath: history.path) + .first { $0.hasSuffix(".mhl") }) + XCTAssertEqual(rename(history.appendingPathComponent(manifest).path, + history.appendingPathComponent(manifest.uppercased()).path), 0) + let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + do { + _ = try await checker.discover(under: root) + XCTFail("A FAT manifest without its publication status is never legacy evidence") + } catch let error as SavedChecksumCheck.CheckError { + XCTAssertEqual(error, .unreadableEvidence) + } + } + + func testCompletedFATStatusBindsExactManifestAndChainBytes() async throws { + for target in ["manifest", "chain"] { + let (root, file) = try fixture() + let manifest = try ASCMHLGenerator.generateInitialHistory( + destinationURL: root, files: [file], startTime: Date(), toolVersion: "test", + readHooks: .init(fileSystemType: { "exfat" }) + ) + let history = manifest.deletingLastPathComponent() + let status = history.appendingPathComponent(ASCMHLGenerator.publicationStatusName) + let changed = target == "manifest" ? manifest : history.appendingPathComponent("ascmhl_chain.xml") + var bytes = try Data(contentsOf: changed) + bytes.append(0x20) + try bytes.write(to: changed) + XCTAssertFalse(try ASCMHLGenerator.publicationIsComplete(at: status)) + let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + do { + _ = try await checker.discover(under: root) + XCTFail("A \(target) change must invalidate the publication record") + } catch let error as SavedChecksumCheck.CheckError { + XCTAssertEqual(error, .unreadableEvidence) + } + } + } + + func testExternalChainExtensionMakesBitMatchCheckFailClosed() async throws { + let (root, file) = try fixture() + let manifest = try ASCMHLGenerator.generateInitialHistory( + destinationURL: root, files: [file], startTime: Date(), toolVersion: "test", + readHooks: .init(fileSystemType: { "exfat" }) + ) + let history = manifest.deletingLastPathComponent() + let chainURL = history.appendingPathComponent("ascmhl_chain.xml") + let secondName = "0002_External.mhl" + let manifestBytes = try Data(contentsOf: manifest) + try manifestBytes.write(to: history.appendingPathComponent(secondName)) + let originalChain = try String(contentsOf: chainURL, encoding: .utf8) + let extensionEntry = "\(secondName)\(ASCMHLGenerator.c4(manifestBytes))" + let extendedChain = originalChain.replacingOccurrences( + of: "", with: " \(extensionEntry)\n" + ) + XCTAssertNotEqual(extendedChain, originalChain) + try extendedChain.write(to: chainURL, atomically: false, encoding: .utf8) + let status = history.appendingPathComponent(ASCMHLGenerator.publicationStatusName) + XCTAssertFalse(try ASCMHLGenerator.publicationIsComplete(at: status), + "The status binds generation one; a later chain needs its own validation") + let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + do { + _ = try await checker.discover(under: root) + XCTFail("BitMatch Check must not claim an unvalidated external chain is intact") + } catch let error as SavedChecksumCheck.CheckError { + XCTAssertEqual(error, .unreadableEvidence) + } + } + func testNestedHistoryAndUnsafePathsAreRejected() throws { let (root, file) = try fixture() for path in ["../clip.txt", "/clip.txt", "a/../clip.txt", "ascmhl/clip.txt", "a\\clip.txt"] { @@ -417,3 +738,10 @@ private final class MHLReadProbe: @unchecked Sendable { func didOpen() { lock.withLock { openCount += 1 } } func didRead(_ count: Int) { lock.withLock { byteCount += count } } } + +private final class MHLPublicationProbe: @unchecked Sendable { + private let lock = NSLock() + private var value = false + var published: Bool { lock.withLock { value } } + func record(_ published: Bool) { lock.withLock { value = value || published } } +} diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Issue10ExFATHandoffTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Issue10ExFATHandoffTests.swift index de4e8cf8..182dc0d3 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Issue10ExFATHandoffTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Issue10ExFATHandoffTests.swift @@ -120,6 +120,10 @@ final class Issue10ExFATHandoffTests: XCTestCase { XCTAssertTrue(FileManager.default.fileExists(atPath: manifest.path)) XCTAssertTrue(FileManager.default.fileExists(atPath: manifest.deletingLastPathComponent().appendingPathComponent("ascmhl_chain.xml").path)) + let status = manifest.deletingLastPathComponent().appendingPathComponent(ASCMHLGenerator.publicationStatusName) + XCTAssertTrue(try ASCMHLGenerator.publicationIsComplete(at: status)) + XCTAssertFalse(try FileManager.default.contentsOfDirectory(atPath: destination.path) + .contains { $0.hasPrefix(".bitmatch-ascmhl-") }, "Successful exFAT publication must not orphan its private staging directory") } catch { let ns = error as NSError print("ISSUE10 direct_mhl_failure domain=\(ns.domain) code=\(ns.code) swiftCancellation=\(error is CancellationError) taskCancelled=\(Task.isCancelled)") @@ -298,6 +302,30 @@ final class Issue10ExFATHandoffTests: XCTestCase { startTime: Date(), toolVersion: "test")) } + func testInterruptedPublicHistoryWithBothFilesIsNotCheckableOnRealExFAT() async throws { + let files = try inventory() + XCTAssertThrowsError(try ASCMHLGenerator.generateInitialHistory( + destinationURL: mount, files: files, startTime: Date(), toolVersion: "test", + readHooks: .init(didWritePublicHistoryFile: { name in + if name == "ascmhl_chain.xml" { throw CancellationError() } + }) + )) { XCTAssertTrue($0 is CancellationError) } + let history = mount.appendingPathComponent("ascmhl", isDirectory: true) + let names = try FileManager.default.contentsOfDirectory(atPath: history.path) + XCTAssertTrue(names.contains(ASCMHLGenerator.publicationStatusName)) + XCTAssertFalse(try ASCMHLGenerator.publicationIsComplete(at: + history.appendingPathComponent(ASCMHLGenerator.publicationStatusName))) + XCTAssertTrue(names.contains("ascmhl_chain.xml")) + XCTAssertTrue(names.contains { $0.hasSuffix(".mhl") && !$0.hasPrefix("._") }) + let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + do { + _ = try await checker.discover(under: mount) + XCTFail("Check must refuse a marked public exFAT history") + } catch let error as SavedChecksumCheck.CheckError { + XCTAssertEqual(error, .unreadableEvidence) + } + } + func testHistoryAppearingInsideClaimCannotBeReplaced() throws { let files = try inventory() let history = mount.appendingPathComponent("ascmhl") diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SavedChecksumCheckTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SavedChecksumCheckTests.swift index bc0072a1..3a4420c6 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SavedChecksumCheckTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SavedChecksumCheckTests.swift @@ -149,6 +149,93 @@ struct SavedChecksumCheckTests { #expect(result.records.allSatisfy { $0.kind == .bitMatchReport }) } + @Test func markedIncompleteHistoryBlocksEvenAnAdjacentReport() async throws { + let root = try makeRoot() + defer { try? FileManager.default.removeItem(at: root) } + let file = try writeFile("clip.mov", text: "original", under: root) + _ = try writeReport(root: root, target: file, checksum: sha256("original"), date: date(2026, 2, 1)) + let history = root.appendingPathComponent("ascmhl", isDirectory: true) + try FileManager.default.createDirectory(at: history, withIntermediateDirectories: false) + try Data("interrupted".utf8).write(to: history.appendingPathComponent(ASCMHLGenerator.publicationStatusName)) + await #expect(throws: SavedChecksumCheck.CheckError.unreadableEvidence) { + _ = try await makeChecker().discover(under: root) + } + } + + @Test func malformedFATCompletionNameCannotEscapeHistoryOrUseAdjacentReport() async throws { + let root = try makeRoot() + defer { try? FileManager.default.removeItem(at: root) } + let file = try writeFile("clip.mov", text: "original", under: root) + _ = try writeReport(root: root, target: file, checksum: sha256("original"), date: date(2026, 2, 1)) + let history = root.appendingPathComponent("ascmhl", isDirectory: true) + try FileManager.default.createDirectory(at: history, withIntermediateDirectories: false) + let manifest = Data("outside history".utf8) + let chain = Data("chain".utf8) + try chain.write(to: history.appendingPathComponent("ascmhl_chain.xml")) + let prefixDirectory = history.appendingPathComponent("0001_BitMatch_FAT_2026", isDirectory: true) + try FileManager.default.createDirectory(at: prefixDirectory, withIntermediateDirectories: false) + try manifest.write(to: root.appendingPathComponent("outside.mhl")) + let traversal = "0001_BitMatch_FAT_2026/../../outside.mhl" + let trailingNewline = "0001_BitMatch_FAT_2026-10-17_120000Z.mhl\n" + try manifest.write(to: history.appendingPathComponent(trailingNewline)) + let status = history.appendingPathComponent(ASCMHLGenerator.publicationStatusName) + for invalidName in [traversal, trailingNewline] { + let bytes = ASCMHLGenerator.publicationOpening + + ASCMHLGenerator.publicationCompletion(name: invalidName, manifest: manifest, chain: chain) + try bytes.write(to: status) + #expect(try !ASCMHLGenerator.publicationIsComplete(at: status)) + await #expect(throws: SavedChecksumCheck.CheckError.unreadableEvidence) { + _ = try await makeChecker().discover(under: root) + } + } + } + + @Test func markerAppearingAfterDiscoveryStopsTheSavedCheck() async throws { + let root = try makeRoot() + defer { try? FileManager.default.removeItem(at: root) } + _ = try writeFile("clip.mov", text: "original", under: root) + try writeMHL(root: root, path: "clip.mov", sha256: sha256("original")) + let checker = makeChecker() + let discovery = try await checker.discover(under: root) + try Data("interrupted".utf8).write( + to: root.appendingPathComponent("ascmhl").appendingPathComponent(ASCMHLGenerator.publicationStatusName) + ) + await #expect(throws: SavedChecksumCheck.CheckError.unreadableEvidence) { + _ = try await checker.check(discovery) { _ in } + } + } + + @Test func caseChangedFATManifestWithoutStatusBlocksAdjacentReport() async throws { + let root = try makeRoot() + defer { try? FileManager.default.removeItem(at: root) } + let file = try writeFile("clip.mov", text: "original", under: root) + _ = try writeReport(root: root, target: file, checksum: sha256("original"), date: date(2026, 2, 1)) + let history = root.appendingPathComponent("ascmhl", isDirectory: true) + try FileManager.default.createDirectory(at: history, withIntermediateDirectories: false) + try Data("partial history".utf8).write(to: history.appendingPathComponent("0001_BITMATCH_FAT_2026-10-17.MHL")) + await #expect(throws: SavedChecksumCheck.CheckError.unreadableEvidence) { + _ = try await makeChecker().discover(under: root) + } + } + + @Test func statusAppearingAfterCheckBeginsStopsIntactVerdict() async throws { + let root = try makeRoot() + defer { try? FileManager.default.removeItem(at: root) } + _ = try writeFile("clip.mov", text: "original", under: root) + try writeMHL(root: root, path: "clip.mov", sha256: sha256("original")) + let checker = makeChecker() + let discovery = try await checker.discover(under: root) + let status = root.appendingPathComponent("ascmhl").appendingPathComponent(ASCMHLGenerator.publicationStatusName) + await #expect(throws: SavedChecksumCheck.CheckError.unreadableEvidence) { + _ = try await checker.check(discovery) { progress in + if progress.filesProcessed == 0 { + _ = FileManager.default.createFile(atPath: status.path, contents: Data("interrupted".utf8)) + } + } + } + #expect(FileManager.default.fileExists(atPath: status.path)) + } + @Test func oneChangedByteIsAMismatch() async throws { let root = try makeRoot() defer { try? FileManager.default.removeItem(at: root) } diff --git a/docs/ISSUE10-INVESTIGATION.md b/docs/ISSUE10-INVESTIGATION.md index 69ec4c2e..ae55aa34 100644 --- a/docs/ISSUE10-INVESTIGATION.md +++ b/docs/ISSUE10-INVESTIGATION.md @@ -22,16 +22,11 @@ The supplied tests were reviewed and extended: they now assert the mounted files ## Publication contract -The fast path remains `renameatx_np(RENAME_EXCL)`. Only unsupported-operation errors take the fallback: +The later race fix keeps `renameatx_np(RENAME_EXCL)` for filesystems that support it. FAT-family volumes take a separate direct path because exFAT returned `ENOTSUP` (errno 45) for exclusive directory rename, and replacing a claimed empty directory with plain `renameat` could replace another writer's substituted directory. -1. Write and synchronize the complete manifest and chain in a private staging directory relative to the pinned destination descriptor. -2. Exclusively reserve `ascmhl` with `mkdirat`. Any existing file, symlink or directory at that name refuses the reservation. -3. Check cancellation, source separation and the reservation's directory identity. -4. Rename the staging directory over that owned empty reservation. A directory rename cannot replace a nonempty history. No public history is populated file by file. +On FAT, BitMatch now reserves `ascmhl` once with `mkdirat`, opens that exact directory without following links, and durably writes a neutral status file **before** the public manifest or chain. It writes those two ASC files through the pinned descriptor with exclusive creation, rereads and checks their bytes and identities, then appends a completion record through its pinned status descriptor. That record binds the exact manifest and chain sizes and SHA-256 values. BitMatch never renames over or removes the public claim or status name. Check refuses the incomplete status, mismatched completion, and a FAT-tagged partial manifest whose status was removed, even if another BitMatch report exists nearby. -An interruption before step 4 can leave an empty `ascmhl` reservation; it has no manifest or chain and is never reported as complete. A later attempt refuses it rather than guessing that it is safe to delete. Review an interrupted reservation before retrying. A process death can also leave a hidden staging directory. The patch does not automatically recover or delete these artifacts. - -Real-exFAT tests prove successful handoff, refusal of an existing history, preservation of a file added inside the reserved directory before publication, and absence of a complete history after interruption. The source remains untouched. Destination readback and checksum/identity validation remain mandatory. +An interruption may leave an empty reservation or partial files. A later attempt refuses that history rather than deleting or replacing it. Unsupported non-FAT filesystems fail closed if exclusive rename is unavailable; a failed private stage is left untouched. Real-exFAT tests cover successful Standard copy/readback → MHL, interruption after both files, preserved existing histories, and no orphan stage after a successful FAT publication. The pinned ASC reference CLI accepts the completed extra-status history, verifies the media, and can append a generation. BitMatch Check currently refuses an externally extended chain because its own completion record binds the initial chain bytes. FAT32, physical drives and power-loss behavior remain untested. None of this establishes the cause of the reporter's interruption; the source remains untouched and destination readback stays mandatory. ## Progress diff --git a/docs/TODO.md b/docs/TODO.md index dca67dea..8e8324ef 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -88,7 +88,7 @@ The priority buckets are not a strict implementation sequence: investigate P0 re - [x] **Close the exFAT/FAT media-file publication race in code.** The no-hard-link FAT-family path now durably creates a visible incomplete-status warning before opening each media file at its final name with exclusive create. It never renames or removes that final name. An interrupted copy leaves its partial bytes and warning in place; existing files and another writer's bytes remain untouched. Exact marker validation, independent readback and named-file identity checks guard status advancement and the safe verdict. Deterministic replacement, cleanup, interruption and recovery regressions pass on the forced-direct path; mounted exFAT marker/root sync and full Standard-to-ASC-MHL handoff pass. See the [design and limits](EXFAT_PUBLICATION_DESIGN.md). **Remaining validation:** FAT32 images could not be mounted on this Mac, and no physical drive or sudden-power-loss result is recorded. Confirm FAT32 sync and transfer behavior on another host, then test representative physical drives without treating OS flush success as a power-loss guarantee. -- [ ] **Close the ASC MHL exFAT empty-directory claim race.** This is separate from media-file publication. The current directory fallback preserves nonempty valid histories, but a concurrent writer could substitute an empty directory at the claim boundary. Add a deterministic replacement regression, then design a no-replacement directory handoff or fail closed while preserving the other writer's directory. Verify on mounted exFAT and retain independent MHL validation before accepting the new history. +- [x] **Close the ASC MHL exFAT empty-directory claim race in code.** FAT-family publication now claims `ascmhl` once and writes through its pinned descriptor; it never renames over the claim or deletes a mutable public name. A durable neutral status file remains incomplete until a completion record binds the exact manifest and chain bytes. BitMatch Check refuses partial or altered FAT histories, even if the status file is removed or a valid report sits beside them. Deterministic replacement, interruption, status-substitution and AppleDouble-lookalike regressions pass; mounted exFAT handoff passes, and the official ASC reference verifier accepts the completed history and can append a generation. **Remaining validation:** FAT32 and physical-drive/power-loss behavior are untested. BitMatch Check deliberately refuses an externally extended chain because its completion record binds the original chain; read-only chain-lineage validation belongs in the existing upstream MHL item. This code fix does not establish the cause of issue #10. - [ ] **Make failed readback output unmistakable on disk.** BitMatch publishes a copied file at its normal destination name before readback; the existing corruption regression confirms that the bad file remains there even though the app verdict correctly fails closed. Investigate marking or moving only an attempt-owned, identity-checked file that fails verification. A read error does not prove the bytes are corrupt, and even bad bytes may be worth preserving. **Acceptance:** injected mismatch, read error, external replacement, interrupted marking and marking failure never overwrite, delete or relabel another writer's file; the real on-disk path and uncertainty appear in History/reports and recovery; source media stays untouched. diff --git a/docs/validation/ascmhl/README.md b/docs/validation/ascmhl/README.md index 9673faec..4d6fb43f 100644 --- a/docs/validation/ascmhl/README.md +++ b/docs/validation/ascmhl/README.md @@ -10,7 +10,9 @@ Existing, nested, and ancestor ASC histories are refused and preserved, includin When the caller supplies its source URL, source and destination descriptors are pinned and checked for equal, ancestor, or descendant relationships before reads, staging writes, and publication. Descriptor-resolved paths catch ancestor symlinks that lead into the source. -Publication uses a pinned destination directory descriptor, no-follow file opens, exclusive staging writes and no-replace directory publication. On exFAT, an exclusively claimed empty directory permits publication without replacing an existing nonempty history. Missing, changed, duplicate, unsafe-path and symbolic-link entries are rejected before publication. Cancellation is checked during reads and before publication. When fresh readback is needed, generating the record adds a full destination read, so this work can materially extend completion time on slow storage. Paranoid transfers no longer automatically emit the old proprietary MHL companions; optional handoff records are generated centrally through this ASC implementation. +Publication uses a pinned destination directory descriptor and no-follow file opens. Filesystems supporting exclusive directory rename publish a synchronized private stage with `RENAME_EXCL`; failures leave that private stage untouched. FAT-family volumes instead claim `ascmhl` exclusively and write through its pinned descriptor. They retain `BITMATCH_HISTORY_STATUS.txt`: BitMatch Check requires its completed record to match the exact initial manifest and chain bytes, and refuses interrupted or altered histories. The FAT manifest filename is distinct so a missing status cannot turn a partial new history into apparently valid legacy evidence. Missing, changed, duplicate, unsafe-path and symbolic-link entries are rejected before publication. Cancellation is checked during reads and publication. When fresh readback is needed, generating the record adds a full destination read, so this work can materially extend completion time on slow storage. Paranoid transfers no longer automatically emit the old proprietary MHL companions; optional handoff records are generated centrally through this ASC implementation. + +The pinned official ASC CLI accepts a completed exFAT-generated history with the extra BitMatch status file, verifies its bytes, and can append a second generation. BitMatch Check currently binds that status to the **initial** chain bytes, so it intentionally refuses a chain extended by another tool until read-only chain-lineage validation is implemented. Use the official ASC validator for externally extended histories. FAT32, physical-drive power loss and other vendors' readers have not been validated here. ## Independent validation