diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DestinationWriter.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DestinationWriter.swift index 25835d05..f83b729b 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DestinationWriter.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/DestinationWriter.swift @@ -1,5 +1,6 @@ -// DestinationWriter.swift - Writes into a pinned backup folder: atomic, -// never-replace publish (exFAT included), reuse checks, and verified reads. +// DestinationWriter.swift - Writes into a pinned backup folder: atomic +// no-replace links where supported, durable direct final files on FAT-family +// drives, reuse checks, and verified reads. import Foundation import Synchronization import CryptoKit @@ -12,27 +13,50 @@ import Darwin public final class PinnedDestinationDirectory: @unchecked Sendable { public let logicalRootURL: URL private let directoryFD: Int32 - - private init(logicalRootURL: URL, directoryFD: Int32) { + private let durableDirectDirectories: Bool + private let beforeDirectorySync: (@Sendable () throws -> Void)? + private let directorySyncFailed = Mutex(false) + private let statusMarker = Mutex(nil) + + private init( + logicalRootURL: URL, + directoryFD: Int32, + durableDirectDirectories: Bool, + beforeDirectorySync: (@Sendable () throws -> Void)? + ) { self.logicalRootURL = logicalRootURL self.directoryFD = directoryFD + self.durableDirectDirectories = durableDirectDirectories + self.beforeDirectorySync = beforeDirectorySync } deinit { _ = Darwin.close(directoryFD) } - public static func open(destination: URL, rootComponents: [String]) throws -> PinnedDestinationDirectory { + public static func open( + destination: URL, + rootComponents: [String], + forceDirectFinalPublication: Bool = false, + beforeDirectorySync: (@Sendable () throws -> Void)? = nil + ) throws -> PinnedDestinationDirectory { let destinationFD = try openDirectory(at: destination, description: "selected destination") var currentFD = destinationFD var logicalRoot = destination do { + let direct = try forceDirectFinalPublication || isFATFamily(directoryFD: destinationFD) for component in rootComponents { try validate(component: component) - let childFD = try openOrCreateDirectory(named: component, relativeTo: currentFD) + let childFD = try openOrCreateDirectory( + named: component, relativeTo: currentFD, + syncCreatedParent: direct, beforeDirectorySync: beforeDirectorySync + ) _ = Darwin.close(currentFD) currentFD = childFD logicalRoot.appendPathComponent(component, isDirectory: true) } - return PinnedDestinationDirectory(logicalRootURL: logicalRoot, directoryFD: currentFD) + return PinnedDestinationDirectory( + logicalRootURL: logicalRoot, directoryFD: currentFD, + durableDirectDirectories: direct, beforeDirectorySync: beforeDirectorySync + ) } catch { _ = Darwin.close(currentFD) throw error @@ -40,12 +64,22 @@ public final class PinnedDestinationDirectory: @unchecked Sendable { } public func openOrCreateDirectory(at relativeComponents: [String]) throws -> Int32 { + guard !directorySyncFailed.withLock({ $0 }) else { + throw FileOperationError.unsafeOperation( + "A new destination folder could not be saved on this drive; keep the source" + ) + } var currentFD = Darwin.dup(directoryFD) guard currentFD >= 0 else { throw Self.posixError("Unable to duplicate pinned destination directory") } do { for component in relativeComponents { try Self.validate(component: component) - let childFD = try Self.openOrCreateDirectory(named: component, relativeTo: currentFD) + let childFD = try Self.openOrCreateDirectory( + named: component, relativeTo: currentFD, + syncCreatedParent: durableDirectDirectories, + beforeDirectorySync: beforeDirectorySync, + onSyncFailure: { self.directorySyncFailed.withLock { $0 = true } } + ) _ = Darwin.close(currentFD) currentFD = childFD } @@ -60,6 +94,45 @@ public final class PinnedDestinationDirectory: @unchecked Sendable { logicalRootURL.appendingPathComponent(relativePath) } + /// FAT-family volumes have no hard links. A marker is saved before the + /// first direct final name can be created. Unknown filesystems still try + /// the atomic link path and fail closed if the capability is unavailable. + func preparePublication( + attemptID: UUID, + sourcePaths: [String], + markerHooks: TransferStatusMarker.CreationHooks? = nil + ) throws { + guard durableDirectDirectories else { return } + let marker = try TransferStatusMarker.create(rootFD: directoryFD, attemptID: attemptID, + sourcePaths: sourcePaths, hooks: markerHooks) + statusMarker.withLock { $0 = marker } + } + + var usesDirectFinalPublication: Bool { statusMarker.withLock { $0 != nil } } + + /// A removed or changed visible warning blocks the next direct final + /// create or write. The final completion gate validates it again. + func requireIncompletePublicationStatus() throws { + guard let marker = statusMarker.withLock({ $0 }), + try marker.currentStatus() == .incomplete else { + throw FileOperationError.unsafeOperation( + "The visible transfer status changed; keep the source and inspect this backup" + ) + } + } + + func finishPublicationStatus(_ status: TransferStatusMarker.Status) throws { + guard let marker = statusMarker.withLock({ $0 }) else { return } + try marker.append(status) + } + + func checkPublicationStatus(_ status: TransferStatusMarker.Status) throws { + guard let marker = statusMarker.withLock({ $0 }) else { return } + guard try marker.currentStatus() == status else { + throw FileOperationError.unsafeOperation("Transfer status is incomplete; keep the source") + } + } + /// Confirms that the folder path shown to the user still names this pinned /// directory. Verification may still succeed through the descriptor after /// a rename, but that orphaned location is not a safe completed backup. @@ -171,6 +244,45 @@ public final class PinnedDestinationDirectory: @unchecked Sendable { return fd } + /// Called only after the visible status marker and its root directory are + /// durable. An error leaves any created final file in place as incomplete + /// evidence; it never pathname-unlinks that file. + static func createDirectFinalFile(named name: String, relativeTo parentFD: Int32) throws -> Int32 { + let fd = name.withCString { + openat(parentFD, $0, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0o600) + } + guard fd >= 0 else { + if errno == EEXIST { + throw posixError("Destination file already exists; choose a new folder or recheck the existing copy") + } + throw posixError("Unable to create destination file on this drive") + } + guard fcntl(fd, F_NOCACHE, 1) != -1 else { + let error = posixError("This drive would not let BitMatch bypass the system cache") + _ = Darwin.close(fd) + throw error + } + return fd + } + + static func namedFileMatchesOpenFile(_ fd: Int32, named name: String, relativeTo parentFD: Int32) throws { + var opened = stat() + guard fstat(fd, &opened) == 0 else { + throw posixError("Unable to inspect open destination file") + } + try namedFileMatchesIdentity(opened, named: name, relativeTo: parentFD) + } + + static func namedFileMatchesIdentity(_ opened: stat, named name: String, relativeTo parentFD: Int32) throws { + var named = stat() + guard name.withCString({ fstatat(parentFD, $0, &named, AT_SYMLINK_NOFOLLOW) }) == 0, + (named.st_mode & S_IFMT) == S_IFREG, + opened.st_dev == named.st_dev, opened.st_ino == named.st_ino, + opened.st_size == named.st_size else { + throw DestinationWriter.existingDestinationConflictError("Destination file name changed during copy; keep the source and inspect this backup") + } + } + /// Pushes the file's data and metadata to the medium, not only to the /// drive's volatile cache (`F_FULLFSYNC`; plain `fsync` does not on /// macOS). A file system that does not implement the full flush at all @@ -184,8 +296,8 @@ public final class PinnedDestinationDirectory: @unchecked Sendable { guard fsync(fd) == 0 else { throw posixError("Unable to flush the copy to the drive") } } - /// Makes a just-published name durable: the directory entry itself is - /// flushed, so a power cut cannot lose the file's name after "verified". + /// Requests durability for a just-created name by flushing its parent + /// directory. Power-loss behavior still depends on the OS and drive. public static func synchronizeDirectory(_ parentFD: Int32) throws { guard fsync(parentFD) == 0 else { throw posixError("Unable to save the destination folder on the drive") } } @@ -209,54 +321,14 @@ public final class PinnedDestinationDirectory: @unchecked Sendable { removeItem(named: temporaryName, relativeTo: parentFD) return } - // exFAT and FAT have no hard links (and no RENAME_EXCL): claim the - // name instead. Every other failure, including EEXIST, fails closed. - guard linkError == ENOTSUP || linkError == EOPNOTSUPP else { - errno = linkError - throw posixError("Destination file appeared during copy; refusing to overwrite it") - } - try publishByClaimingName(temporaryName: temporaryName, name: name, relativeTo: parentFD) - } - - /// No-replace publication for filesystems without hard links. An - /// exclusive create claims the final name (failing if anything is - /// there), then the verified temporary file is renamed over that empty - /// claim once it is confirmed to still be ours. Only a file deleted and - /// recreated at this exact name between that check and the rename could - /// be replaced. The identity is read after fsync because macOS's exFAT - /// driver reports a temporary inode for a file until it is committed. - public static func publishByClaimingName(temporaryName: String, name: String, relativeTo parentFD: Int32) throws { - let flags = O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC - let claimFD = name.withCString { openat(parentFD, $0, flags, 0o600) } - guard claimFD >= 0 else { throw posixError("Destination file appeared during copy; refusing to overwrite it") } - defer { _ = Darwin.close(claimFD) } - - var claimed = stat() - guard fsync(claimFD) == 0, fstat(claimFD, &claimed) == 0 else { - throw posixError("Unable to claim destination file name") + errno = linkError + if linkError == ENOTSUP || linkError == EOPNOTSUPP { + throw posixError("This destination filesystem cannot safely publish files; choose a different backup folder") } - var current = stat() - let lookup = name.withCString { fstatat(parentFD, $0, ¤t, AT_SYMLINK_NOFOLLOW) } - guard lookup == 0, current.st_dev == claimed.st_dev, current.st_ino == claimed.st_ino, - (current.st_mode & S_IFMT) == S_IFREG, current.st_size == 0 else { - throw DestinationWriter.existingDestinationConflictError("Destination file changed during copy; refusing to overwrite it") - } - - let renamed = temporaryName.withCString { temporaryNamePointer in - name.withCString { namePointer in - renameat(parentFD, temporaryNamePointer, parentFD, namePointer) - } - } - guard renamed == 0 else { - let renameError = posixError("Unable to publish destination file") - // Remove the empty claim only while the name is still ours. - var after = stat() - if name.withCString({ fstatat(parentFD, $0, &after, AT_SYMLINK_NOFOLLOW) }) == 0, - after.st_dev == claimed.st_dev, after.st_ino == claimed.st_ino, after.st_size == 0 { - removeItem(named: name, relativeTo: parentFD) - } - throw renameError + if linkError == EEXIST { + throw posixError("Destination file appeared during copy; refusing to overwrite it") } + throw posixError("Could not safely publish on this destination filesystem; choose a different backup folder") } /// Descends from `/` one descriptor at a time so `O_NOFOLLOW` protects @@ -315,13 +387,31 @@ public final class PinnedDestinationDirectory: @unchecked Sendable { return status == 0 && (info.st_mode & S_IFMT) == S_IFLNK } - private static func openOrCreateDirectory(named name: String, relativeTo parentFD: Int32) throws -> Int32 { + private static func openOrCreateDirectory( + named name: String, + relativeTo parentFD: Int32, + syncCreatedParent: Bool, + beforeDirectorySync: (@Sendable () throws -> Void)?, + onSyncFailure: (() -> Void)? = nil + ) throws -> Int32 { let flags = O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC let openExisting = { name.withCString { openat(parentFD, $0, flags) } } var fd = openExisting() if fd < 0 && errno == ENOENT { let created = name.withCString { mkdirat(parentFD, $0, 0o755) } if created != 0 && errno != EEXIST { throw posixError("Unable to create destination directory") } + // ENOENT followed by EEXIST means another writer just created + // the name. Flush that observed parent entry as well before a + // direct final file may be opened beneath it. + if syncCreatedParent { + do { + try beforeDirectorySync?() + try synchronizeDirectory(parentFD) + } catch { + onSyncFailure?() + throw error + } + } fd = openExisting() } guard fd >= 0 else { @@ -339,6 +429,17 @@ public final class PinnedDestinationDirectory: @unchecked Sendable { return fd } + private static func isFATFamily(directoryFD: Int32) throws -> Bool { + var info = statfs() + guard fstatfs(directoryFD, &info) == 0 else { + throw posixError("Unable to identify destination filesystem") + } + let filesystem = withUnsafeBytes(of: info.f_fstypename) { bytes in + String(decoding: bytes.prefix(while: { $0 != 0 }), as: UTF8.self).lowercased() + } + return ["exfat", "msdos", "msdosfs", "fat", "fat32"].contains(filesystem) + } + private static func validate(component: String) throws { guard !component.isEmpty, component != ".", @@ -489,7 +590,12 @@ public final class DestinationWriter { public var beforeFlush: (@Sendable (Int, URL) throws -> Void)? public var beforeClose: (@Sendable (Int, URL) throws -> Void)? public var beforePublish: (@Sendable (Int, URL) throws -> Void)? - public var useClaimedNamePublish: (@Sendable (Int) -> Bool)? + /// Forces the FAT-style direct-final path on a test filesystem. + public var forceDirectFinalPublication: (@Sendable (Int) -> Bool)? + public var beforeDirectFinalCreate: (@Sendable (Int, URL) throws -> Void)? + public var beforeDirectorySync: (@Sendable (Int) throws -> Void)? + public var beforeMarkerFileFlush: (@Sendable (Int) throws -> Void)? + public var beforeMarkerRootSync: (@Sendable (Int) throws -> Void)? public var afterPublish: (@Sendable (Int, URL) throws -> Void)? public var beforeDestinationRead: (@Sendable (Int, String) async throws -> Void)? public var destinationDidOpenForRead: (@Sendable (Int, String) -> Void)? @@ -509,7 +615,11 @@ public final class DestinationWriter { beforeFlush: (@Sendable (Int, URL) throws -> Void)? = nil, beforeClose: (@Sendable (Int, URL) throws -> Void)? = nil, beforePublish: (@Sendable (Int, URL) throws -> Void)? = nil, - useClaimedNamePublish: (@Sendable (Int) -> Bool)? = nil, + forceDirectFinalPublication: (@Sendable (Int) -> Bool)? = nil, + beforeDirectFinalCreate: (@Sendable (Int, URL) throws -> Void)? = nil, + beforeDirectorySync: (@Sendable (Int) throws -> Void)? = nil, + beforeMarkerFileFlush: (@Sendable (Int) throws -> Void)? = nil, + beforeMarkerRootSync: (@Sendable (Int) throws -> Void)? = nil, afterPublish: (@Sendable (Int, URL) throws -> Void)? = nil, beforeDestinationRead: (@Sendable (Int, String) async throws -> Void)? = nil, destinationDidOpenForRead: (@Sendable (Int, String) -> Void)? = nil, @@ -526,7 +636,11 @@ public final class DestinationWriter { self.beforeFlush = beforeFlush self.beforeClose = beforeClose self.beforePublish = beforePublish - self.useClaimedNamePublish = useClaimedNamePublish + self.forceDirectFinalPublication = forceDirectFinalPublication + self.beforeDirectFinalCreate = beforeDirectFinalCreate + self.beforeDirectorySync = beforeDirectorySync + self.beforeMarkerFileFlush = beforeMarkerFileFlush + self.beforeMarkerRootSync = beforeMarkerRootSync self.afterPublish = afterPublish self.beforeDestinationRead = beforeDestinationRead self.destinationDidOpenForRead = destinationDidOpenForRead @@ -589,6 +703,7 @@ public final class DestinationWriter { workers: Int, checksumService: any ChecksumService, preEnumeratedFiles: [URL], + attemptID: UUID = UUID(), pauseCheck: (@Sendable () async throws -> Void)? = nil, hooks: FanOutHooks? = nil, onIOBytes: IOByteCallback? = nil, @@ -596,12 +711,37 @@ public final class DestinationWriter { onProgress: @escaping @Sendable (Int, String, Int64, Bool) async -> Void, onError: @escaping @Sendable (Int, String, Error) async -> Void ) async throws { + let sourceResolver = RelativePathResolver(base: src) + let sourcePaths = try preEnumeratedFiles.map { try sourceResolver.resolve($0) } + // Root directories are also source names. A marker must not take one + // of those names on a case-insensitive backup filesystem. + let sourceRootNames = try FileManager.default.contentsOfDirectory(atPath: src.path) + var prepared: [FanOutDestination] = [] for destination in destinations { + try Task.checkCancellation() + do { + try destination.root.preparePublication( + attemptID: attemptID, + sourcePaths: sourcePaths + sourceRootNames, + markerHooks: .init( + beforeFileFlush: { try hooks?.beforeMarkerFileFlush?(destination.index) }, + beforeRootSync: { try hooks?.beforeMarkerRootSync?(destination.index) } + ) + ) + prepared.append(destination) + } catch is CancellationError { + throw CancellationError() + } catch { + for path in sourcePaths { await onError(destination.index, path, error) } + } + } + for destination in prepared { + try Task.checkCancellation() try await createDirectoryTreeSafely(from: src, in: destination.root) { path, error in await onError(destination.index, path, error) } } - let sourceResolver = RelativePathResolver(base: src) + let activeDestinations = prepared // FAT/exFAT companion publication can retire or change an AppleDouble // entry. Finish media publication before copying its metadata files; @@ -624,7 +764,7 @@ public final class DestinationWriter { do { relativePath = try sourceResolver.resolve(fileURL) } catch { - for destination in destinations { + for destination in activeDestinations { await onError(destination.index, fileURL.path, error) } continue @@ -635,7 +775,7 @@ public final class DestinationWriter { code: NSFileWriteNoPermissionError, userInfo: [NSLocalizedDescriptionKey: "Path contains traversal component"] ) - for destination in destinations { + for destination in activeDestinations { await onError(destination.index, relativePath, error) } continue @@ -648,7 +788,7 @@ public final class DestinationWriter { code: NSFileWriteNoPermissionError, userInfo: [NSLocalizedDescriptionKey: "Source file resolves outside source directory"] ) - for destination in destinations { + for destination in activeDestinations { await onError(destination.index, relativePath, error) } continue @@ -659,7 +799,7 @@ public final class DestinationWriter { from: fileURL, relativePath: relativePath, components: components, - destinations: destinations, + destinations: activeDestinations, verificationMode: verificationMode, checksumService: checksumService, pauseCheck: pauseCheck, @@ -672,7 +812,7 @@ public final class DestinationWriter { } catch is CancellationError { throw CancellationError() } catch { - for destination in destinations { + for destination in activeDestinations { await onError(destination.index, relativePath, error) } } @@ -691,8 +831,10 @@ public final class DestinationWriter { private(set) var parentFD: Int32 let filename: String let temporaryName: String + let directFinal: Bool private(set) var temporaryFD: Int32 private(set) var published = false + private var flushedIdentity: stat? init( destination: FanOutDestination, @@ -703,13 +845,16 @@ public final class DestinationWriter { self.destination = destination self.parentFD = parentFD self.filename = filename - temporaryName = ".bitmatch.tmp." + UUID().uuidString - temporaryFD = try PinnedDestinationDirectory.createTemporaryFile( - named: temporaryName, - relativeTo: parentFD - ) - // Reaching this point means createTemporaryFile's mandatory - // F_NOCACHE call succeeded for this still-open descriptor. + directFinal = destination.root.usesDirectFinalPublication + temporaryName = directFinal ? filename : ".bitmatch.tmp." + UUID().uuidString + if directFinal { + try hooks?.beforeDirectFinalCreate?(destination.index, destination.root.logicalRootURL) + try destination.root.requireIncompletePublicationStatus() + } + temporaryFD = try directFinal + ? PinnedDestinationDirectory.createDirectFinalFile(named: filename, relativeTo: parentFD) + : PinnedDestinationDirectory.createTemporaryFile(named: temporaryName, relativeTo: parentFD) + // Both creation paths require F_NOCACHE before the first write. hooks?.temporaryFileDidDisableCache?(destination.index, temporaryFD) } @@ -718,6 +863,14 @@ public final class DestinationWriter { func closeTemporaryFile() throws { guard temporaryFD >= 0 else { return } let fd = temporaryFD + if directFinal { + var info = stat() + guard fstat(fd, &info) == 0 else { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno), + userInfo: [NSLocalizedDescriptionKey: "Unable to inspect direct destination file"]) + } + flushedIdentity = info + } // POSIX leaves the descriptor state unspecified when close fails, // including after EINTR. Retire it before close and fail the file; // the caller must never publish after this method throws. @@ -733,12 +886,19 @@ public final class DestinationWriter { func markPublished() { published = true } + func checkDirectFinalName() throws { + guard directFinal, let flushedIdentity else { return } + try PinnedDestinationDirectory.namedFileMatchesIdentity( + flushedIdentity, named: filename, relativeTo: parentFD + ) + } + func cleanup() { if temporaryFD >= 0 { _ = Darwin.close(temporaryFD) temporaryFD = -1 } - if !published, parentFD >= 0 { + if !published, !directFinal, parentFD >= 0 { PinnedDestinationDirectory.removeItem(named: temporaryName, relativeTo: parentFD) } if parentFD >= 0 { @@ -890,6 +1050,13 @@ public final class DestinationWriter { file.destination.root.logicalRootURL, currentChunkIndex ) + // A marker read on every 4 MiB chunk would + // add thousands of small disk reads to one + // long clip. Check at entry and every 256 MiB; + // final coverage validates it again. + if file.directFinal && currentChunkIndex.isMultiple(of: 64) { + try file.destination.root.requireIncompletePublicationStatus() + } try writeAll(chunkData, to: file.temporaryFD) { count, seconds, ended in onIOBytes?(file.destination.index, .writing, count, seconds, ended) } @@ -993,16 +1160,15 @@ public final class DestinationWriter { try hooks?.beforeFlush?(file.destination.index, file.destination.root.logicalRootURL) try PinnedDestinationDirectory.flushToMedium(file.temporaryFD) try hooks?.beforeClose?(file.destination.index, file.destination.root.logicalRootURL) + if file.directFinal { + try PinnedDestinationDirectory.namedFileMatchesOpenFile( + file.temporaryFD, named: file.filename, relativeTo: file.parentFD + ) + } try file.closeTemporaryFile() try hooks?.beforePublish?(file.destination.index, file.destination.root.logicalRootURL) do { - if hooks?.useClaimedNamePublish?(file.destination.index) == true { - try PinnedDestinationDirectory.publishByClaimingName( - temporaryName: file.temporaryName, - name: file.filename, - relativeTo: file.parentFD - ) - } else { + if !file.directFinal { try PinnedDestinationDirectory.publishTemporaryFile( named: file.temporaryName, as: file.filename, @@ -1029,7 +1195,9 @@ public final class DestinationWriter { } file.markPublished() try PinnedDestinationDirectory.synchronizeDirectory(file.parentFD) + try file.checkDirectFinalName() try hooks?.afterPublish?(file.destination.index, file.destination.root.logicalRootURL) + try file.checkDirectFinalName() await onProgress(file.destination.index, relativePath, sourceSize, false) } catch is CancellationError { file.cleanup() @@ -1090,7 +1258,7 @@ public final class DestinationWriter { NSError( domain: "DestinationWriter", code: -4, - userInfo: [NSLocalizedDescriptionKey: "Source file changed during copy; destination was not modified"] + userInfo: [NSLocalizedDescriptionKey: "Source file changed during copy; keep the source and inspect the backup for a partial file"] ) } #endif diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/TransferStatusMarker.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/TransferStatusMarker.swift new file mode 100644 index 00000000..cb27487d --- /dev/null +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/TransferStatusMarker.swift @@ -0,0 +1,274 @@ +import Foundation +import CryptoKit +#if canImport(Darwin) +import Darwin + +/// A visible, durable warning for FAT-family output, which must be written at +/// its final name. This file is never evidence that media was verified. +final class TransferStatusMarker: @unchecked Sendable { + enum Status: String { + case incomplete + case filesVerified + case copyCompleteUnverified + } + + private static let prefix = "BITMATCH_TRANSFER_STATUS_" + private static let suffix = ".txt" + private static let header = "BITMATCH TRANSFER STATUS V1\nWARNING: This transfer may be incomplete. Keep the source until BitMatch independently verifies every backup.\nThis file is not a safe-to-erase result. Check BitMatch History before clearing the source.\n" + private static let maximumLength = 2_048 + + struct CreationHooks { + var beforeFileFlush: (() throws -> Void)? = nil + var beforeRootSync: (() throws -> Void)? = nil + } + + let name: String + private let attemptID: UUID + private let markerID: UUID + private let rootFD: Int32 + private let fileFD: Int32 + + private init(name: String, attemptID: UUID, markerID: UUID, rootFD: Int32, fileFD: Int32) { + self.name = name + self.attemptID = attemptID + self.markerID = markerID + self.rootFD = rootFD + self.fileFD = fileFD + } + + deinit { + _ = Darwin.close(fileFD) + _ = Darwin.close(rootFD) + } + + static func create( + rootFD: Int32, + attemptID: UUID, + sourcePaths: [String], + firstCandidateID: UUID? = nil, + hooks: CreationHooks? = nil + ) throws -> TransferStatusMarker { + // The source manifest remains authoritative, including an ordinary + // source file whose name happens to look exactly like our marker. + // Callers include explicit root entries, including empty folders. + // Also derive them from nested paths so this API cannot choose a + // marker name occupied by a source folder if only files were passed. + let sourceRootNames = sourcePaths.compactMap { + $0.split(separator: "/", maxSplits: 1).first.map(String.init) + } + for candidate in 0..<64 { + let markerID = candidate == 0 ? (firstCandidateID ?? UUID()) : UUID() + let name = prefix + markerID.uuidString + suffix + if sourceRootNames.contains(where: { $0.precomposedStringWithCanonicalMapping.caseInsensitiveCompare(name) == .orderedSame }) { + continue + } + let fd = name.withCString { + openat(rootFD, $0, O_RDWR | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0o600) + } + if fd < 0 { + if errno == EEXIST { continue } + throw failure("Unable to create the visible transfer status file") + } + let retainedRoot = Darwin.dup(rootFD) + guard retainedRoot >= 0 else { + let error = failure("Unable to retain the transfer status folder") + _ = Darwin.close(fd) + throw error + } + let marker = TransferStatusMarker(name: name, attemptID: attemptID, + markerID: markerID, rootFD: retainedRoot, fileFD: fd) + do { + try marker.writeInitialStatus(hooks: hooks) + return marker + } catch { + // An uncertain marker is retained as a warning. No media name + // is opened after this failure. + throw error + } + } + throw FileOperationError.unsafeOperation("Could not choose a transfer status name that avoids source files and existing destination items") + } + + func append(_ status: Status) throws { + guard status != .incomplete else { + throw FileOperationError.unsafeOperation("Invalid transfer status transition") + } + guard try currentStatus() == .incomplete else { + throw FileOperationError.unsafeOperation("Transfer status file changed; keep the source and inspect this backup") + } + let record = Self.record(sequence: 1, status: status, attemptID: attemptID, markerID: markerID) + guard lseek(fileFD, 0, SEEK_END) >= 0 else { throw Self.failure("Unable to seek transfer status file") } + try Self.writeAll(Data(record.utf8), to: fileFD) + try Self.flush(fileFD, description: "Unable to save transfer status on the drive") + guard try currentStatus() == status else { + throw FileOperationError.unsafeOperation("Transfer status file changed; keep the source and inspect this backup") + } + } + + func currentStatus() throws -> Status { + var opened = stat() + var named = stat() + guard fstat(fileFD, &opened) == 0, + name.withCString({ fstatat(rootFD, $0, &named, AT_SYMLINK_NOFOLLOW) }) == 0, + (named.st_mode & S_IFMT) == S_IFREG, + opened.st_dev == named.st_dev, + opened.st_ino == named.st_ino else { + throw FileOperationError.unsafeOperation("Transfer status file is missing or was replaced; keep the source") + } + guard opened.st_size >= 0, opened.st_size <= Int64(Self.maximumLength) else { + throw FileOperationError.unsafeOperation("Transfer status file is damaged; keep the source") + } + let reader = name.withCString { openat(rootFD, $0, O_RDONLY | O_NOFOLLOW | O_CLOEXEC) } + guard reader >= 0 else { throw Self.failure("Unable to read transfer status file") } + defer { _ = Darwin.close(reader) } + var readerInfo = stat() + guard fstat(reader, &readerInfo) == 0, + (readerInfo.st_mode & S_IFMT) == S_IFREG, + readerInfo.st_dev == opened.st_dev, + readerInfo.st_ino == opened.st_ino, + readerInfo.st_size == opened.st_size else { + throw FileOperationError.unsafeOperation("Transfer status file was replaced while reading; keep the source") + } + let bytes = try Self.readAll(reader, maximum: Self.maximumLength) + guard fstat(reader, &readerInfo) == 0, + readerInfo.st_dev == opened.st_dev, + readerInfo.st_ino == opened.st_ino, + readerInfo.st_size == opened.st_size, + name.withCString({ fstatat(rootFD, $0, &named, AT_SYMLINK_NOFOLLOW) }) == 0, + (named.st_mode & S_IFMT) == S_IFREG, + named.st_dev == opened.st_dev, + named.st_ino == opened.st_ino, + named.st_size == opened.st_size else { + throw FileOperationError.unsafeOperation("Transfer status file changed while reading; keep the source") + } + guard let status = Self.parse(bytes, expectedAttempt: attemptID, expectedMarker: markerID) else { + throw FileOperationError.unsafeOperation("Transfer status file is damaged; keep the source") + } + return status + } + + /// Only a root-level, exact-name, fully validated file can be treated as + /// BitMatch metadata by Compare or Check. A source-side twin remains data. + static func validatedStatus(at url: URL, relativePath: String) -> Status? { + guard !relativePath.contains("/"), let markerID = markerID(in: relativePath) else { return nil } + let fd = url.path.withCString { Darwin.open($0, O_RDONLY | O_NOFOLLOW | O_CLOEXEC) } + guard fd >= 0 else { return nil } + defer { _ = Darwin.close(fd) } + var info = stat() + guard fstat(fd, &info) == 0, (info.st_mode & S_IFMT) == S_IFREG, + info.st_size >= 0, info.st_size <= Int64(maximumLength), + let data = try? readAll(fd, maximum: maximumLength), + data.count == Int(info.st_size) else { return nil } + var after = stat() + var named = stat() + guard fstat(fd, &after) == 0, + lstat(url.path, &named) == 0, + (named.st_mode & S_IFMT) == S_IFREG, + after.st_dev == info.st_dev, after.st_ino == info.st_ino, + after.st_size == info.st_size, + named.st_dev == info.st_dev, named.st_ino == info.st_ino, + named.st_size == info.st_size else { return nil } + return parse(data, expectedAttempt: nil, expectedMarker: markerID) + } + + static func hasExactName(_ relativePath: String) -> Bool { + !relativePath.contains("/") && markerID(in: relativePath) != nil + } + + private func writeInitialStatus(hooks: CreationHooks?) throws { + let body = Self.header + + "Attempt: \(attemptID.uuidString)\n" + + "Marker: \(markerID.uuidString)\n" + + Self.record(sequence: 0, status: .incomplete, attemptID: attemptID, markerID: markerID) + try Self.writeAll(Data(body.utf8), to: fileFD) + try hooks?.beforeFileFlush?() + try Self.flush(fileFD, description: "Unable to save the transfer status file on this drive") + try hooks?.beforeRootSync?() + guard fsync(rootFD) == 0 else { + throw Self.failure("This drive cannot durably save its transfer status name; no media file was created") + } + guard try currentStatus() == .incomplete else { + throw FileOperationError.unsafeOperation("Transfer status could not be confirmed on the drive; no media file was created") + } + } + + private static func record(sequence: Int, status: Status, attemptID: UUID, markerID: UUID) -> String { + let payload = "\(sequence)|\(status.rawValue)|\(attemptID.uuidString)|\(markerID.uuidString)" + let hash = SHA256.hash(data: Data(payload.utf8)).map { String(format: "%02x", $0) }.joined() + return "Record: \(sequence) \(status.rawValue) \(hash)\n" + } + + private static func parse(_ data: Data, expectedAttempt: UUID?, expectedMarker: UUID) -> Status? { + guard data.count <= maximumLength, let text = String(data: data, encoding: .utf8), + text.hasPrefix(header), text.hasSuffix("\n") else { return nil } + let remainder = String(text.dropFirst(header.count)) + let lines = remainder.split(separator: "\n", omittingEmptySubsequences: false).map(String.init) + guard (lines.count == 4 || lines.count == 5), lines.last == "", + lines[0].hasPrefix("Attempt: "), lines[1].hasPrefix("Marker: "), + let attempt = UUID(uuidString: String(lines[0].dropFirst(9))), + let marker = UUID(uuidString: String(lines[1].dropFirst(8))), + attempt.uuidString == String(lines[0].dropFirst(9)), + marker.uuidString == String(lines[1].dropFirst(8)), + marker == expectedMarker, + expectedAttempt == nil || attempt == expectedAttempt, + lines[2] == String(record(sequence: 0, status: .incomplete, attemptID: attempt, markerID: marker).dropLast()) else { + return nil + } + if lines.count == 4 { return .incomplete } + for status in [Status.filesVerified, .copyCompleteUnverified] { + if lines[3] == String(record(sequence: 1, status: status, attemptID: attempt, markerID: marker).dropLast()) { + return status + } + } + return nil + } + + private static func markerID(in name: String) -> UUID? { + guard name.hasPrefix(prefix), name.hasSuffix(suffix) else { return nil } + let id = String(name.dropFirst(prefix.count).dropLast(suffix.count)) + guard let uuid = UUID(uuidString: id), uuid.uuidString == id else { return nil } + return uuid + } + + private static func writeAll(_ data: Data, to fd: Int32) throws { + try data.withUnsafeBytes { bytes in + guard let base = bytes.baseAddress else { return } + var offset = 0 + while offset < bytes.count { + let count = Darwin.write(fd, base.advanced(by: offset), bytes.count - offset) + if count > 0 { offset += count } + else if count < 0 && errno == EINTR { continue } + else { throw failure("Unable to write transfer status file") } + } + } + } + + private static func readAll(_ fd: Int32, maximum: Int) throws -> Data { + var data = Data() + var buffer = [UInt8](repeating: 0, count: maximum + 1) + while true { + let count = buffer.withUnsafeMutableBytes { bytes in + Darwin.read(fd, bytes.baseAddress, bytes.count) + } + if count > 0 { + data.append(contentsOf: buffer.prefix(count)) + if data.count > maximum { return Data() } + } else if count == 0 { return data } + else if errno != EINTR { throw failure("Unable to read transfer status file") } + } + } + + private static func flush(_ fd: Int32, description: String) throws { + if fcntl(fd, F_FULLFSYNC) == 0 { return } + guard errno == ENOTSUP || errno == EOPNOTSUPP, fsync(fd) == 0 else { + throw failure(description) + } + } + + private static func failure(_ message: String) -> NSError { + let code = errno + return NSError(domain: NSPOSIXErrorDomain, code: Int(code), + userInfo: [NSLocalizedDescriptionKey: "\(message): \(String(cString: strerror(code)))"]) + } +} +#endif diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/FolderComparer.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/FolderComparer.swift index 2c796b56..9eba523e 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/FolderComparer.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/FolderComparer.swift @@ -71,8 +71,14 @@ public struct FolderComparer: Sendable { // held a `._` sidecar the backup does not have, and doubt fails // closed. Files present on both sides are never content-compared. let onlyInSource = sourceSet.subtracting(destSet) - let onlyInDest = destSet.subtracting(sourceSet).filter { - !Self.isOffloadManifest($0) && !Self.isFinderMetadata($0) + let onlyInDest = destSet.subtracting(sourceSet).filter { path in + guard !Self.isOffloadManifest(path), !Self.isFinderMetadata(path) else { return false } + #if canImport(Darwin) + if let item = destMap[path], TransferStatusMarker.validatedStatus(at: item.url, relativePath: path) != nil { + return false + } + #endif + return true } let common = sourceSet.intersection(destSet).filter { !Self.isFinderMetadata($0) } diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/SavedChecksumCheck.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/SavedChecksumCheck.swift index 45cc461a..cf0df0f6 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/SavedChecksumCheck.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/SavedChecksumCheck.swift @@ -163,6 +163,10 @@ public struct SavedChecksumCheck: Sendable { try await PauseGate.waitIfCurrentIsPaused() let path = try resolver.resolve(file) if Self.isEvidenceOrMetadata(path) { continue } + #if canImport(Darwin) + if discovery.expectedFiles[path] == nil, + TransferStatusMarker.validatedStatus(at: file, relativePath: path) != nil { continue } + #endif actual[path] = file.standardizedFileURL.resolvingSymlinksKeepingCase() } diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferPipeline.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferPipeline.swift index eecc82fc..a34bc61c 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferPipeline.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferPipeline.swift @@ -135,6 +135,20 @@ public actor RunLedger { return !rows[index].success } + func hasSuccessfulCoverage( + destination: Int, + sourcePaths: [String], + verificationMode: VerificationMode + ) -> Bool { + sourcePaths.allSatisfy { path in + guard let index = rowIndex[FileResultKey(sourceRelativePath: path, destinationIndex: destination)] else { + return false + } + let row = rows[index] + return row.success && (verificationMode == .quick || row.verificationResult?.isValid == true) + } + } + private func store(_ row: FileOperationResult, relativePath: String, destination: Int) { let key = FileResultKey(sourceRelativePath: relativePath, destinationIndex: destination) if let index = rowIndex[key] { @@ -732,9 +746,12 @@ public final class TransferPipeline: FileOperationsService, Sendable { // creates every recipe component descriptor-relative with O_NOFOLLOW. try Task.checkCancellation() try destinationSetupHook?(destinationURL) + let forceDirect = fanOutHooks?.forceDirectFinalPublication?(destIndex) == true pinnedDestinations[destIndex] = try PinnedDestinationDirectory.open( destination: destinationURL, - rootComponents: rootComponents + rootComponents: rootComponents, + forceDirectFinalPublication: forceDirect, + beforeDirectorySync: { try self.fanOutHooks?.beforeDirectorySync?(destIndex) } ) } catch let error as FileOperationError { // A safety-policy rejection (e.g. a symlink substituted into @@ -812,6 +829,7 @@ public final class TransferPipeline: FileOperationsService, Sendable { workers: copyWorkers, checksumService: self.checksumService, preEnumeratedFiles: sourceFileURLs, + attemptID: operation.id, pauseCheck: { try await pauseGate.wait() }, hooks: fanOutHooks, onIOBytes: onIOBytes, @@ -1136,6 +1154,43 @@ public final class TransferPipeline: FileOperationsService, Sendable { } } + // A FAT-family destination may expose a partial final-name file while + // it is being written. Only the current attempt's validated marker may + // advance from incomplete, and only after the ordinary independent + // readback and exact selected-file coverage have passed. + for (destIndex, destinationURL) in operation.destinationURLs.enumerated() { + guard let pinned = pinnedDestinations[destIndex], + pinned.usesDirectFinalPublication, + await ledger.hasSuccessfulCoverage( + destination: destIndex, + sourcePaths: sourceManifest.map(\.relativePath), + verificationMode: operation.verificationMode + ) else { continue } + let status: TransferStatusMarker.Status = operation.verificationMode == .quick + ? .copyCompleteUnverified : .filesVerified + do { + try pinned.finishPublicationStatus(status) + try pinned.checkPublicationStatus(status) + } catch { + let failure = completionGateError( + "\(destinationURL.lastPathComponent)'s transfer status could not be saved or confirmed: \(error.localizedDescription). Keep the source." + ) + for entry in sourceManifest { + let row = FileOperationResult( + sourceURL: entry.url, + destinationURL: pinned.logicalRootURL.appendingPathComponent(entry.relativePath), + success: false, + error: failure, + fileSize: entry.size, + verificationResult: nil, + processingTime: 0 + ) + await ledger.record(row, relativePath: entry.relativePath, destination: destIndex) + await onFileResult?(row) + } + } + } + // The executor writes optional ASC MHL handoff records after authoritative verification. let finalResults = await ledger.results() diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/AppleDoubleFilesystemMatrixTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/AppleDoubleFilesystemMatrixTests.swift index 25b66a88..73894140 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/AppleDoubleFilesystemMatrixTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/AppleDoubleFilesystemMatrixTests.swift @@ -88,7 +88,10 @@ final class AppleDoubleFilesystemMatrixTests: XCTestCase { XCTAssertEqual(operation.results.count, before.count * 2) if traceEnabled { let observations = trace.withLock { $0 } - XCTAssertTrue(observations.contains("before publication: companion exists=false")) + // Direct-final publication exposes the media name at open. + // The filesystem may create its companion before or after + // this hook; only its final presence is the test contract. + XCTAssertTrue(observations.contains { $0.hasPrefix("before publication: companion exists=") }) XCTAssertTrue(observations.contains("after publication: companion exists=true")) print("Issue16 fresh exFAT trace: " + observations.joined(separator: "; ")) } diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ClaimedNamePublicationRaceTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ClaimedNamePublicationRaceTests.swift new file mode 100644 index 00000000..dcddd761 --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ClaimedNamePublicationRaceTests.swift @@ -0,0 +1,129 @@ +#if os(macOS) +import Darwin +import Foundation +import XCTest +@testable import BitMatchEngine + +/// Deterministic boundaries that lost another writer's bytes under the old +/// empty-claim/rename fallback. All paths are inside a disposable directory. +final class ClaimedNamePublicationRaceTests: XCTestCase { + private var scratch: URL! + + override func setUpWithError() throws { + scratch = FileManager.default.temporaryDirectory + .appendingPathComponent("bitmatch-publication-race-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: scratch, withIntermediateDirectories: false) + } + + override func tearDownWithError() throws { + if let scratch { try FileManager.default.removeItem(at: scratch) } + } + + private func openScratchDirectory() throws -> Int32 { + let fd = Darwin.open(scratch.path, O_RDONLY | O_DIRECTORY | O_CLOEXEC) + guard fd >= 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } + return fd + } + + private func replaceNameWithOtherWriterFile(named name: String, bytes: Data, relativeTo parentFD: Int32) { + XCTAssertEqual(name.withCString { unlinkat(parentFD, $0, 0) }, 0) + let fd = name.withCString { + openat(parentFD, $0, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0o600) + } + XCTAssertGreaterThanOrEqual(fd, 0) + guard fd >= 0 else { return } + defer { _ = Darwin.close(fd) } + XCTAssertEqual(bytes.withUnsafeBytes { Darwin.write(fd, $0.baseAddress, $0.count) }, bytes.count) + XCTAssertEqual(fsync(fd), 0) + } + + func testOtherWriterReplacingFinalNameBeforeCompletionIsNeitherOverwrittenNorDeleted() throws { + let parentFD = try openScratchDirectory() + defer { _ = Darwin.close(parentFD) } + let marker = try TransferStatusMarker.create(rootFD: parentFD, attemptID: UUID(), sourcePaths: ["clip.mov"]) + XCTAssertEqual(try marker.currentStatus(), .incomplete) + let finalName = "clip.mov" + let otherBytes = Data("another writer's file".utf8) + let copiedBytes = Data("original copy".utf8) + let fd = try PinnedDestinationDirectory.createDirectFinalFile(named: finalName, relativeTo: parentFD) + defer { _ = Darwin.close(fd) } + + // Former boundary: another writer replaced the empty claim after the + // identity check and before renameat. Direct writes stay on our FD. + replaceNameWithOtherWriterFile(named: finalName, bytes: otherBytes, relativeTo: parentFD) + XCTAssertEqual(copiedBytes.withUnsafeBytes { Darwin.write(fd, $0.baseAddress, $0.count) }, copiedBytes.count) + XCTAssertThrowsError(try PinnedDestinationDirectory.namedFileMatchesOpenFile(fd, named: finalName, relativeTo: parentFD)) + XCTAssertEqual(try Data(contentsOf: scratch.appendingPathComponent(finalName)), otherBytes) + XCTAssertEqual(try marker.currentStatus(), .incomplete) + } + + func testFailureCleanupNeverUnlinksOtherWritersReplacement() throws { + let parentFD = try openScratchDirectory() + defer { _ = Darwin.close(parentFD) } + let marker = try TransferStatusMarker.create(rootFD: parentFD, attemptID: UUID(), sourcePaths: ["clip.mov"]) + let finalName = "clip.mov" + let otherBytes = Data("another writer's file".utf8) + let fd = try PinnedDestinationDirectory.createDirectFinalFile(named: finalName, relativeTo: parentFD) + replaceNameWithOtherWriterFile(named: finalName, bytes: otherBytes, relativeTo: parentFD) + + // Former boundary: an error after the claim check ran pathname-based + // cleanup. The direct file owns only a descriptor and is never unlinked. + _ = Darwin.close(fd) + XCTAssertEqual(try Data(contentsOf: scratch.appendingPathComponent(finalName)), otherBytes) + XCTAssertEqual(try marker.currentStatus(), .incomplete) + } + + func testMarkerNameCollisionIgnoresCaseAndSourceFileRemainsData() throws { + let parentFD = try openScratchDirectory() + defer { _ = Darwin.close(parentFD) } + let collisionID = UUID() + let existing = "BITMATCH_TRANSFER_STATUS_\(collisionID.uuidString).txt" + let source = scratch.appendingPathComponent("SOURCE", isDirectory: true) + try FileManager.default.createDirectory(at: source, withIntermediateDirectories: true) + try Data("source data".utf8).write(to: source.appendingPathComponent(existing)) + let marker = try TransferStatusMarker.create(rootFD: parentFD, attemptID: UUID(), + sourcePaths: [existing.lowercased()], firstCandidateID: collisionID) + XCTAssertNotEqual(marker.name.lowercased(), existing.lowercased()) + XCTAssertFalse(FileManager.default.fileExists(atPath: scratch.appendingPathComponent(existing).path)) + XCTAssertEqual(try Data(contentsOf: source.appendingPathComponent(existing)), Data("source data".utf8)) + } + + func testMarkerNameCollisionAlsoChecksNestedSourceRootDirectory() throws { + let parentFD = try openScratchDirectory() + defer { _ = Darwin.close(parentFD) } + let collisionID = UUID() + let rootName = "BITMATCH_TRANSFER_STATUS_\(collisionID.uuidString).txt" + let sourceRoot = scratch.appendingPathComponent("SOURCE", isDirectory: true) + let nested = sourceRoot.appendingPathComponent(rootName, isDirectory: true) + try FileManager.default.createDirectory(at: nested, withIntermediateDirectories: true) + let sourceFile = nested.appendingPathComponent("clip.mov") + try Data("source data".utf8).write(to: sourceFile) + + let marker = try TransferStatusMarker.create( + rootFD: parentFD, attemptID: UUID(), + sourcePaths: ["\(rootName.lowercased())/clip.mov"], firstCandidateID: collisionID + ) + XCTAssertNotEqual(marker.name.lowercased(), rootName.lowercased()) + XCTAssertEqual(try Data(contentsOf: sourceFile), Data("source data".utf8)) + XCTAssertEqual(try marker.currentStatus(), .incomplete) + } + + func testTornMarkerCannotAuthorizeVerifiedStatus() throws { + let parentFD = try openScratchDirectory() + defer { _ = Darwin.close(parentFD) } + let marker = try TransferStatusMarker.create(rootFD: parentFD, attemptID: UUID(), sourcePaths: []) + let statusURL = scratch.appendingPathComponent(marker.name) + XCTAssertEqual(TransferStatusMarker.validatedStatus(at: statusURL, relativePath: marker.name), .incomplete) + try marker.append(.filesVerified) + XCTAssertEqual(TransferStatusMarker.validatedStatus(at: statusURL, relativePath: marker.name), .filesVerified) + let fd = Darwin.open(statusURL.path, O_WRONLY | O_APPEND) + XCTAssertGreaterThanOrEqual(fd, 0) + guard fd >= 0 else { return } + let torn = Data("X".utf8) + XCTAssertEqual(torn.withUnsafeBytes { Darwin.write(fd, $0.baseAddress, $0.count) }, 1) + _ = Darwin.close(fd) + XCTAssertNil(TransferStatusMarker.validatedStatus(at: statusURL, relativePath: marker.name)) + XCTAssertThrowsError(try marker.currentStatus()) + } +} +#endif diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DirectFinalPublicationTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DirectFinalPublicationTests.swift new file mode 100644 index 00000000..1cc4c88a --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/DirectFinalPublicationTests.swift @@ -0,0 +1,422 @@ +#if os(macOS) +import CryptoKit +import Darwin +import Foundation +import Synchronization +import XCTest +@testable import BitMatchEngine + +/// Exercises FAT-style publication on disposable APFS folders through a test +/// hook, so the failure boundaries are deterministic without a mounted image. +final class DirectFinalPublicationTests: XCTestCase, @unchecked Sendable { + private var scratch: URL! + private var source: URL! + private var destination: URL! + private let settings = CameraLabelSettings() + + override func setUpWithError() throws { + scratch = FileManager.default.temporaryDirectory + .appendingPathComponent("bitmatch-direct-final-\(UUID().uuidString)", isDirectory: true) + source = scratch.appendingPathComponent("SOURCE", isDirectory: true) + destination = scratch.appendingPathComponent("DEST", isDirectory: true) + try FileManager.default.createDirectory(at: source, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) + } + + override func tearDownWithError() throws { + if let scratch { try FileManager.default.removeItem(at: scratch) } + } + + private var outputRoot: URL { + SafetyValidator.resolvedDestinationRoot(source: source, destination: destination, settings: settings) + } + + private func writeSource(_ bytes: Data) throws { + try bytes.write(to: source.appendingPathComponent("clip.bin")) + } + + private func markers() throws -> [(url: URL, status: TransferStatusMarker.Status?)] { + try FileManager.default.contentsOfDirectory(at: outputRoot, includingPropertiesForKeys: nil) + .filter { TransferStatusMarker.hasExactName($0.lastPathComponent) } + .map { ($0, TransferStatusMarker.validatedStatus(at: $0, relativePath: $0.lastPathComponent)) } + } + + private func perform( + mode: VerificationMode = .standard, + hooks: DestinationWriter.FanOutHooks = .init(forceDirectFinalPublication: { _ in true }) + ) async throws -> FileOperation { + try await TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared, + fanOutHooks: hooks).performFileOperation( + sourceURL: source, destinationURLs: [destination], verificationMode: mode, + settings: settings, progressCallback: { _ in }, onFileResult: nil + ) + } + + func testVerifiedMarkerFollowsIndependentReadback() async throws { + try await FileOperationsTestLock.shared.run { + let bytes = Data(repeating: 0x44, count: 8193) + try writeSource(bytes) + let operation = try await perform() + XCTAssertEqual(operation.results.count, 1) + XCTAssertEqual(operation.results.first?.verificationResult?.isValid, true) + XCTAssertNotNil(operation.results.first?.verificationResult?.destinationReadIdentity) + XCTAssertEqual(try markers().count, 1) + XCTAssertEqual(try markers().first?.status, .filesVerified) + XCTAssertEqual(try Data(contentsOf: outputRoot.appendingPathComponent("clip.bin")), bytes) + } + } + + func testInterruptedSecondChunkLeavesPartialFinalFileAndIncompleteWarning() async throws { + try await FileOperationsTestLock.shared.run { + let bytes = Data(repeating: 0x62, count: 4 * 1024 * 1024 + 97) + try writeSource(bytes) + let operation = try await perform(hooks: .init( + beforeWrite: { _, _, chunk in + if chunk == 1 { throw NSError(domain: "InjectedWrite", code: 1) } + }, + forceDirectFinalPublication: { _ in true } + )) + XCTAssertEqual(operation.results.count, 1) + XCTAssertFalse(operation.results[0].success) + XCTAssertEqual(try markers().first?.status, .incomplete) + XCTAssertEqual(try Data(contentsOf: outputRoot.appendingPathComponent("clip.bin")).count, 4 * 1024 * 1024) + } + } + + func testSourceChangeAfterFirstChunkWarnsAboutPartialFinalFile() async throws { + try await FileOperationsTestLock.shared.run { + try writeSource(Data(repeating: 0x63, count: 4 * 1024 * 1024 + 97)) + let reads = Mutex(0) + let operation = try await perform(hooks: .init( + sourceDidRead: { _, data in + let ordinal = reads.withLock { value in + value += 1 + return value + } + return ordinal == 2 ? Data() : data + }, + forceDirectFinalPublication: { _ in true } + )) + XCTAssertEqual(operation.results.count, 1) + XCTAssertFalse(operation.results[0].success) + XCTAssertTrue(operation.results[0].error?.localizedDescription.contains("partial file") == true) + XCTAssertEqual(try markers().first?.status, .incomplete) + XCTAssertEqual(try Data(contentsOf: outputRoot.appendingPathComponent("clip.bin")).count, + 4 * 1024 * 1024) + } + } + + func testFlushFailureRetainsFinalBytesAndIncompleteWarning() async throws { + try await FileOperationsTestLock.shared.run { + let bytes = Data(repeating: 0x5c, count: 8193) + try writeSource(bytes) + let operation = try await perform(hooks: .init( + beforeFlush: { _, _ in throw NSError(domain: "InjectedFlush", code: 1) }, + forceDirectFinalPublication: { _ in true } + )) + XCTAssertFalse(operation.results[0].success) + XCTAssertEqual(try markers().first?.status, .incomplete) + XCTAssertEqual(try Data(contentsOf: outputRoot.appendingPathComponent("clip.bin")), bytes) + } + } + + func testMarkerFlushFailureOpensNoFinalMediaName() async throws { + try await FileOperationsTestLock.shared.run { + try writeSource(Data("clip data".utf8)) + let operation = try await perform(hooks: .init( + forceDirectFinalPublication: { _ in true }, + beforeMarkerFileFlush: { _ in throw NSError(domain: "InjectedMarkerFlush", code: 1) } + )) + XCTAssertEqual(operation.results.count, 1) + XCTAssertFalse(operation.results[0].success) + XCTAssertFalse(FileManager.default.fileExists(atPath: outputRoot.appendingPathComponent("clip.bin").path)) + } + } + + func testMarkerRootSyncFailureKeepsOtherBackupValid() async throws { + try await FileOperationsTestLock.shared.run { + let fixture = try DisposableTransferFixture(seed: 20_261_017, fileCount: 1, bytesPerFile: 4096) + defer { fixture.cleanup() } + let hooks = DestinationWriter.FanOutHooks( + forceDirectFinalPublication: { _ in true }, + beforeMarkerRootSync: { index in + if index == 0 { throw NSError(domain: "InjectedRootSync", code: 1) } + } + ) + let operation = try await TransferPipeline( + fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared, fanOutHooks: hooks + ).performFileOperation( + sourceURL: fixture.source, destinationURLs: fixture.destinations, + verificationMode: .standard, settings: settings, + progressCallback: { _ in }, onFileResult: nil + ) + let failedRoot = SafetyValidator.resolvedDestinationRoot( + source: fixture.source, destination: fixture.destinations[0], settings: settings + ) + let validRoot = SafetyValidator.resolvedDestinationRoot( + source: fixture.source, destination: fixture.destinations[1], settings: settings + ) + XCTAssertEqual(operation.results.count, fixture.manifest.count * 2) + XCTAssertTrue(operation.results.filter { $0.destinationURL.path.hasPrefix(failedRoot.path + "/") } + .allSatisfy { !$0.success }) + XCTAssertTrue(operation.results.filter { $0.destinationURL.path.hasPrefix(validRoot.path + "/") } + .allSatisfy { $0.success && $0.verificationResult?.isValid == true }) + for path in fixture.manifest.keys { + XCTAssertFalse(FileManager.default.fileExists(atPath: failedRoot.appendingPathComponent(path).path)) + } + let validMarkers = try FileManager.default.contentsOfDirectory(at: validRoot, includingPropertiesForKeys: nil) + .filter { TransferStatusMarker.hasExactName($0.lastPathComponent) } + XCTAssertEqual(validMarkers.count, 1) + XCTAssertEqual(TransferStatusMarker.validatedStatus(at: validMarkers[0], + relativePath: validMarkers[0].lastPathComponent), .filesVerified) + } + } + + func testMarkerRemovedBeforeDirectCreateOpensNoMediaName() async throws { + try await FileOperationsTestLock.shared.run { + try writeSource(Data("clip data".utf8)) + let root = outputRoot + let operation = try await perform(hooks: .init( + forceDirectFinalPublication: { _ in true }, + beforeDirectFinalCreate: { _, _ in + let entries = try FileManager.default.contentsOfDirectory( + at: root, includingPropertiesForKeys: nil + ) + let marker = try XCTUnwrap(entries.first { + TransferStatusMarker.hasExactName($0.lastPathComponent) + }) + try FileManager.default.removeItem(at: marker) + } + )) + XCTAssertTrue(operation.results.allSatisfy { !$0.success }) + XCTAssertFalse(FileManager.default.fileExists( + atPath: root.appendingPathComponent("clip.bin").path + )) + } + } + + func testMarkerRemovedAtPeriodicWriteCheckStopsPartialFile() async throws { + try await FileOperationsTestLock.shared.run { + let boundary = 256 * 1024 * 1024 + let sourceFile = source.appendingPathComponent("clip.bin") + XCTAssertTrue(FileManager.default.createFile(atPath: sourceFile.path, contents: nil)) + let sourceHandle = try FileHandle(forWritingTo: sourceFile) + try sourceHandle.truncate(atOffset: UInt64(boundary + 97)) + try sourceHandle.close() + let root = outputRoot + let operation = try await perform(hooks: .init( + beforeWrite: { _, _, chunk in + guard chunk == 64 else { return } + let entries = try FileManager.default.contentsOfDirectory( + at: root, includingPropertiesForKeys: nil + ) + let marker = try XCTUnwrap(entries.first { + TransferStatusMarker.hasExactName($0.lastPathComponent) + }) + try FileManager.default.removeItem(at: marker) + }, + forceDirectFinalPublication: { _ in true } + )) + XCTAssertTrue(operation.results.allSatisfy { !$0.success }) + let partial = try FileManager.default.attributesOfItem( + atPath: root.appendingPathComponent("clip.bin").path + ) + XCTAssertEqual((partial[.size] as? NSNumber)?.intValue, boundary) + } + } + + func testNestedDirectorySyncFailureOpensNoFinalMediaName() async throws { + try await FileOperationsTestLock.shared.run { + let nested = source.appendingPathComponent("NESTED", isDirectory: true) + try FileManager.default.createDirectory(at: nested, withIntermediateDirectories: true) + try Data("clip data".utf8).write(to: nested.appendingPathComponent("clip.bin")) + let root = outputRoot + let operation = try await perform(hooks: .init( + forceDirectFinalPublication: { _ in true }, + beforeDirectorySync: { _ in + let entries = (try? FileManager.default.contentsOfDirectory( + at: root, includingPropertiesForKeys: nil + )) ?? [] + if entries.contains(where: { TransferStatusMarker.hasExactName($0.lastPathComponent) }) { + throw NSError(domain: "InjectedDirectorySync", code: 1) + } + } + )) + XCTAssertTrue(operation.results.allSatisfy { !$0.success }) + XCTAssertEqual(try markers().first?.status, .incomplete) + XCTAssertFalse(FileManager.default.fileExists( + atPath: root.appendingPathComponent("NESTED/clip.bin").path + )) + } + } + + func testMarkerTamperBeforeFinalGateCannotLeaveGreenResult() async throws { + try await FileOperationsTestLock.shared.run { + try writeSource(Data("clip data".utf8)) + let root = outputRoot + let removed = Mutex(false) + let operation = try await perform(hooks: .init( + forceDirectFinalPublication: { _ in true }, + beforeDestinationRead: { _, _ in + let shouldRemove = removed.withLock { value in + if value { return false } + value = true + return true + } + guard shouldRemove else { return } + let items = try FileManager.default.contentsOfDirectory(at: root, includingPropertiesForKeys: nil) + let marker = try XCTUnwrap(items.first { TransferStatusMarker.hasExactName($0.lastPathComponent) }) + try Data("damaged status".utf8).write(to: marker) + } + )) + XCTAssertEqual(operation.results.count, 1) + XCTAssertFalse(operation.results[0].success) + XCTAssertNil(try markers().first?.status) + } + } + + func testMissingMarkerBeforeFinalGateCannotLeaveGreenResult() async throws { + try await FileOperationsTestLock.shared.run { + try writeSource(Data("clip data".utf8)) + let root = outputRoot + let removed = Mutex(false) + let operation = try await perform(hooks: .init( + forceDirectFinalPublication: { _ in true }, + beforeDestinationRead: { _, _ in + let shouldRemove = removed.withLock { value in + if value { return false } + value = true + return true + } + guard shouldRemove else { return } + let items = try FileManager.default.contentsOfDirectory(at: root, includingPropertiesForKeys: nil) + let marker = try XCTUnwrap(items.first { TransferStatusMarker.hasExactName($0.lastPathComponent) }) + try FileManager.default.removeItem(at: marker) + } + )) + XCTAssertEqual(operation.results.count, 1) + XCTAssertFalse(operation.results[0].success) + XCTAssertTrue(try markers().isEmpty) + } + } + + func testSameSizeReplacementAfterCloseSurvivesWithoutGreenResult() async throws { + try await FileOperationsTestLock.shared.run { + let original = Data("first copy".utf8) + let outsider = Data("other copy".utf8) + XCTAssertEqual(original.count, outsider.count) + try writeSource(original) + let finalURL = outputRoot.appendingPathComponent("clip.bin") + let operation = try await perform(hooks: .init( + beforePublish: { _, _ in + try FileManager.default.removeItem(at: finalURL) + try outsider.write(to: finalURL, options: .withoutOverwriting) + }, + forceDirectFinalPublication: { _ in true } + )) + XCTAssertFalse(operation.results[0].success) + XCTAssertEqual(try Data(contentsOf: finalURL), outsider) + XCTAssertEqual(try markers().first?.status, .incomplete) + } + } + + func testMatchingFileCanBeRecoveredOnlyAfterFreshProof() async throws { + try await FileOperationsTestLock.shared.run { + let bytes = Data("recover after interruption".utf8) + try writeSource(bytes) + let first = try await perform(hooks: .init( + forceDirectFinalPublication: { _ in true }, + beforeDestinationRead: { _, _ in throw NSError(domain: "InjectedRead", code: 1) } + )) + XCTAssertFalse(first.results[0].success) + XCTAssertEqual(try markers().map(\.status), [.incomplete]) + let second = try await perform() + XCTAssertTrue(second.results[0].success) + XCTAssertTrue(second.results[0].wasReused) + XCTAssertEqual(second.results[0].verificationResult?.isValid, true) + XCTAssertEqual(Set(try markers().compactMap(\.status)), Set([.incomplete, .filesVerified])) + XCTAssertEqual(try Data(contentsOf: outputRoot.appendingPathComponent("clip.bin")), bytes) + } + } + + func testQuickModeRefusesExistingFileAndRetainsEarlierWarning() async throws { + try await FileOperationsTestLock.shared.run { + let bytes = Data("quick copy".utf8) + try writeSource(bytes) + let first = try await perform(mode: .quick) + XCTAssertTrue(first.results[0].success) + XCTAssertEqual(try markers().map(\.status), [.copyCompleteUnverified]) + let second = try await perform(mode: .quick) + XCTAssertFalse(second.results[0].success) + XCTAssertEqual(Set(try markers().compactMap(\.status)), Set([.incomplete, .copyCompleteUnverified])) + XCTAssertEqual(try Data(contentsOf: outputRoot.appendingPathComponent("clip.bin")), bytes) + } + } + + func testCompareAndSavedCheckIgnoreOnlyValidatedDestinationOnlyMarker() async throws { + try await FileOperationsTestLock.shared.run { + let bytes = Data("clip data".utf8) + try writeSource(bytes) + let copied = try await perform() + XCTAssertTrue(copied.results[0].success) + let marker = try XCTUnwrap(markers().first?.url) + let comparer = FolderComparer(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + var compared = try await comparer.compare(left: source, right: outputRoot, + verificationMode: .standard, progress: { _ in }) + XCTAssertTrue(compared.isClean) + + let reports = outputRoot.appendingPathComponent("Reports", isDirectory: true) + try FileManager.default.createDirectory(at: reports, withIntermediateDirectories: true) + let report = reports.appendingPathComponent("BitMatch_Report_marker-test.json") + let digest = SHA256.hash(data: bytes).map { String(format: "%02x", $0) }.joined() + let payload: [String: Any] = [ + "timestamp": ISO8601DateFormatter().string(from: Date()), + "mode": "copy-and-verify", + "verification": ["algorithm": "SHA-256"], + "results": [["target": outputRoot.appendingPathComponent("clip.bin").path, + "status": ResultOutcome.verified.statusText, "checksum": digest]], + ] + try JSONSerialization.data(withJSONObject: payload).write(to: report) + let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + let checked = try await checker.check(try await checker.discover(under: outputRoot)) { _ in } + XCTAssertTrue(checked.isIntact) + XCTAssertFalse(checked.newPaths.contains(marker.lastPathComponent)) + + // A same-named SOURCE item is selected data. It must participate + // in content comparison even though destination has a valid marker. + try Data("not status metadata".utf8).write(to: source.appendingPathComponent(marker.lastPathComponent)) + compared = try await comparer.compare(left: source, right: outputRoot, + verificationMode: .standard, progress: { _ in }) + XCTAssertFalse(compared.isClean) + XCTAssertTrue(compared.mismatchedPaths.contains(marker.lastPathComponent)) + try FileManager.default.removeItem(at: source.appendingPathComponent(marker.lastPathComponent)) + + // An exact-looking but malformed destination file is ordinary + // unrecorded data, never silently classified as a status marker. + let bogus = outputRoot.appendingPathComponent("BITMATCH_TRANSFER_STATUS_\(UUID().uuidString).txt") + try Data("bogus".utf8).write(to: bogus) + compared = try await comparer.compare(left: source, right: outputRoot, + verificationMode: .standard, progress: { _ in }) + XCTAssertTrue(compared.onlyInRightPaths.contains(bogus.lastPathComponent)) + let checkedAgain = try await checker.check(try await checker.discover(under: outputRoot)) { _ in } + XCTAssertTrue(checkedAgain.newPaths.contains(bogus.lastPathComponent)) + + // Check must retain a marker-shaped path when the saved evidence + // explicitly names it; its bytes are then checked as ordinary data. + let markerBytes = try Data(contentsOf: marker) + let markerDigest = SHA256.hash(data: markerBytes).map { String(format: "%02x", $0) }.joined() + var withMarker = payload + withMarker["results"] = [ + ["target": outputRoot.appendingPathComponent("clip.bin").path, + "status": ResultOutcome.verified.statusText, "checksum": digest], + ["target": marker.path, + "status": ResultOutcome.verified.statusText, "checksum": markerDigest], + ] + try JSONSerialization.data(withJSONObject: withMarker).write(to: report) + let explicitlyRecorded = try await checker.check(try await checker.discover(under: outputRoot)) { _ in } + XCTAssertEqual(explicitlyRecorded.recordedFileCount, 2) + XCTAssertTrue(explicitlyRecorded.matchingPaths.contains(marker.lastPathComponent)) + } + } +} +#endif diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ExFATDestinationTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ExFATDestinationTests.swift index 5288b6f9..38a9b29a 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ExFATDestinationTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/ExFATDestinationTests.swift @@ -1,9 +1,7 @@ // ExFATDestinationTests.swift -// exFAT has no hard links and no RENAME_EXCL, so the linkat no-replace -// publish failed every file on exFAT backups (errno 45) in 0.1.4 through -// 0.1.6. These tests mount a real exFAT disk image: APFS cannot reproduce -// it, and macOS's exFAT driver also reports a temporary inode for a new -// file until it is fsynced. +// exFAT has no hard links and no RENAME_EXCL. These tests mount a real exFAT +// image to exercise the visible, durable marker and direct final-file path. +// macOS's exFAT driver reports a temporary inode until a file is fsynced. import Darwin import Foundation import XCTest @@ -23,6 +21,13 @@ final class ExFATDestinationTests: XCTestCase { hdiutil(["attach", "-quiet", "-nobrowse", "-mountpoint", mountPoint.path, image.path + ".sparseimage"]) == 0 else { throw XCTSkip("hdiutil could not create or mount an exFAT image") } + var info = statfs() + XCTAssertEqual(statfs(mountPoint.path, &info), 0) + let actual = withUnsafeBytes(of: info.f_fstypename) { bytes in + String(decoding: bytes.prefix(while: { $0 != 0 }), as: UTF8.self) + } + print("EXFAT_DESTINATION mounted f_fstypename=\(actual)") + XCTAssertEqual(actual.lowercased(), "exfat") } override func tearDown() { @@ -53,49 +58,60 @@ final class ExFATDestinationTests: XCTestCase { XCTAssertEqual(fsync(fd), 0) } - /// Fails if `PinnedDestinationDirectory.publishTemporaryFile` loses its fallback for filesystems - /// without hard links (only `linkat`, as in 0.1.4-0.1.6). - func testPublishOnExFATGivesTheFileItsName() throws { + /// The visible warning is durable before a final media name is opened. + func testDirectFinalFileOnExFATHasDurableMarker() throws { let parentFD = try openDirectory(mountPoint) defer { _ = Darwin.close(parentFD) } - try writeTemporary("clip", named: ".bitmatch.tmp.a", in: parentFD) - - try PinnedDestinationDirectory.publishTemporaryFile(named: ".bitmatch.tmp.a", as: "A001C001.MXF", relativeTo: parentFD) + let marker = try TransferStatusMarker.create(rootFD: parentFD, attemptID: UUID(), sourcePaths: ["A001C001.MXF"]) + XCTAssertEqual(try marker.currentStatus(), .incomplete) + let markerFD = Darwin.open(mountPoint.appendingPathComponent(marker.name).path, O_RDWR | O_NOFOLLOW | O_CLOEXEC) + XCTAssertGreaterThanOrEqual(markerFD, 0) + if markerFD >= 0 { + defer { _ = Darwin.close(markerFD) } + let fullSync = fcntl(markerFD, F_FULLFSYNC) + let fullError = fullSync == 0 ? 0 : errno + let fileSync = fsync(markerFD) + let fileError = fileSync == 0 ? 0 : errno + let rootSync = fsync(parentFD) + let rootError = rootSync == 0 ? 0 : errno + print("EXFAT_DESTINATION marker F_FULLFSYNC rc=\(fullSync) errno=\(fullError); marker fsync rc=\(fileSync) errno=\(fileError); root fsync rc=\(rootSync) errno=\(rootError)") + XCTAssertEqual(fileSync, 0) + XCTAssertEqual(rootSync, 0) + } + let fd = try PinnedDestinationDirectory.createDirectFinalFile(named: "A001C001.MXF", relativeTo: parentFD) + defer { _ = Darwin.close(fd) } + let bytes = Data("clip".utf8) + XCTAssertEqual(bytes.withUnsafeBytes { Darwin.write(fd, $0.baseAddress, $0.count) }, bytes.count) + try PinnedDestinationDirectory.flushToMedium(fd) + try PinnedDestinationDirectory.synchronizeDirectory(parentFD) let published = mountPoint.appendingPathComponent("A001C001.MXF") XCTAssertEqual(try String(contentsOf: published, encoding: .utf8), "clip") - XCTAssertFalse(FileManager.default.fileExists(atPath: mountPoint.appendingPathComponent(".bitmatch.tmp.a").path)) + XCTAssertTrue(FileManager.default.fileExists(atPath: mountPoint.appendingPathComponent(marker.name).path)) } - /// Promise 1: the fallback must still never replace an existing file. - /// Fails if the fallback renames over the name without claiming it - /// exclusively first. - func testPublishOnExFATRefusesToReplaceAnExistingFile() throws { + /// Promise 1: O_EXCL refuses an existing destination file. + func testDirectFinalOnExFATRefusesToReplaceAnExistingFile() throws { let parentFD = try openDirectory(mountPoint) defer { _ = Darwin.close(parentFD) } let existing = mountPoint.appendingPathComponent("A001C001.MXF") try Data("someone else's".utf8).write(to: existing) - try writeTemporary("clip", named: ".bitmatch.tmp.b", in: parentFD) - - XCTAssertThrowsError(try PinnedDestinationDirectory.publishTemporaryFile(named: ".bitmatch.tmp.b", as: "A001C001.MXF", relativeTo: parentFD)) + _ = try TransferStatusMarker.create(rootFD: parentFD, attemptID: UUID(), sourcePaths: ["A001C001.MXF"]) + XCTAssertThrowsError(try PinnedDestinationDirectory.createDirectFinalFile(named: "A001C001.MXF", relativeTo: parentFD)) XCTAssertEqual(try String(contentsOf: existing, encoding: .utf8), "someone else's") } - /// The fallback itself must refuse a file that appeared at the name - /// (linkat reports EEXIST first when one is already there, so the test - /// above does not reach the fallback). Fails if `publishByClaimingName` - /// renames without claiming the name exclusively first. - func testClaimFallbackRefusesToReplaceAFileThatAppeared() throws { + /// Unknown filesystems without links cannot take the old claim path. + func testTemporaryPublicationOnExFATFailsClosedWithoutClaimOrRename() throws { let parentFD = try openDirectory(mountPoint) defer { _ = Darwin.close(parentFD) } let existing = mountPoint.appendingPathComponent("A001C002.MXF") - try Data("appeared meanwhile".utf8).write(to: existing) try writeTemporary("clip", named: ".bitmatch.tmp.c", in: parentFD) - - XCTAssertThrowsError(try PinnedDestinationDirectory.publishByClaimingName(temporaryName: ".bitmatch.tmp.c", name: "A001C002.MXF", relativeTo: parentFD)) - - XCTAssertEqual(try String(contentsOf: existing, encoding: .utf8), "appeared meanwhile") + XCTAssertThrowsError(try PinnedDestinationDirectory.publishTemporaryFile( + named: ".bitmatch.tmp.c", as: "A001C002.MXF", relativeTo: parentFD + )) + XCTAssertFalse(FileManager.default.fileExists(atPath: existing.path)) XCTAssertTrue(FileManager.default.fileExists(atPath: mountPoint.appendingPathComponent(".bitmatch.tmp.c").path)) } diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/FanOutTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/FanOutTests.swift index df2000a7..46862ee8 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/FanOutTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/FanOutTests.swift @@ -433,16 +433,16 @@ struct FanOutTests { } @Test - func claimedNamePublishFallbackCompletesThroughFanOut() async throws { + func directFinalPublicationCompletesThroughFanOut() async throws { try await FileOperationsTestLock.shared.run { #if os(macOS) - let fixture = try FanOutFixture(name: "claim-publish") + let fixture = try FanOutFixture(name: "direct-final") defer { fixture.cleanup() } let expected = Data(repeating: 0x6c, count: 4099) try fixture.write(expected, to: "clip.bin") let probe = FanOutProbe() let hooks = DestinationWriter.FanOutHooks( - useClaimedNamePublish: { probe.shouldClaimName(for: $0) } + forceDirectFinalPublication: { probe.shouldForceDirectFinal(for: $0) } ) let operation = try await fixture.pipeline(hooks: hooks).performFileOperation( sourceURL: fixture.source, @@ -455,13 +455,13 @@ struct FanOutTests { #expect(operation.results.count == fixture.destinations.count) #expect(operation.results.allSatisfy { $0.success && $0.verificationResult?.isValid == true }) - #expect(probe.publishDecisions == Set(fixture.destinations.indices)) - let claimedFile = SafetyValidator.resolvedDestinationRoot( + #expect(probe.directFinalDecisions == Set(fixture.destinations.indices)) + let directFile = SafetyValidator.resolvedDestinationRoot( source: fixture.source, destination: fixture.destinations[1], settings: CameraLabelSettings() ).appendingPathComponent("clip.bin") - #expect(try Data(contentsOf: claimedFile) == expected) + #expect(try Data(contentsOf: directFile) == expected) #else #expect(true) #endif @@ -1126,7 +1126,7 @@ private final class FanOutProbe: Sendable { var completedPerDestination: [Int] = [] var noCacheDestinations: Set = [] var allNoCacheDescriptorsWereOpen = true - var publishDecisions: Set = [] + var directFinalDecisions: Set = [] var destinationOpens: [String: Int] = [:] var destinationBytes: [String: Int] = [:] var verificationSourceOpens: [String: Int] = [:] @@ -1200,12 +1200,12 @@ private final class FanOutProbe: Sendable { var noCacheDestinations: Set { state.withLock { $0.noCacheDestinations } } var allNoCacheDescriptorsWereOpen: Bool { state.withLock { $0.allNoCacheDescriptorsWereOpen } } - func shouldClaimName(for destination: Int) -> Bool { - _ = state.withLock { $0.publishDecisions.insert(destination) } + func shouldForceDirectFinal(for destination: Int) -> Bool { + _ = state.withLock { $0.directFinalDecisions.insert(destination) } return destination == 1 } - var publishDecisions: Set { state.withLock { $0.publishDecisions } } + var directFinalDecisions: Set { state.withLock { $0.directFinalDecisions } } private func destinationKey(destination: Int, path: String) -> String { "\(destination):\(path)" } diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Issue10ExFATHandoffTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Issue10ExFATHandoffTests.swift index 160417ec..de4e8cf8 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Issue10ExFATHandoffTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/Issue10ExFATHandoffTests.swift @@ -30,9 +30,10 @@ final class Issue10ExFATHandoffTests: XCTestCase { mounted = true var volume = statfs() XCTAssertEqual(statfs(mount.path, &volume), 0) - let type = withUnsafePointer(to: &volume.f_fstypename) { - $0.withMemoryRebound(to: CChar.self, capacity: 16) { String(cString: $0) } + let type = withUnsafeBytes(of: volume.f_fstypename) { bytes in + String(decoding: bytes.prefix(while: { $0 != 0 }), as: UTF8.self) } + print("ISSUE10 mounted filesystem f_fstypename=\(type)") XCTAssertEqual(type.lowercased(), "exfat", "Must exercise the actual exFAT driver") // Deliberately fail setup rather than skip: no green test without // actually exercising the exFAT filesystem. @@ -173,6 +174,16 @@ final class Issue10ExFATHandoffTests: XCTestCase { XCTAssertTrue(operation.results.allSatisfy { $0.success && $0.verificationResult?.isValid == true }, "Failure occurs before MHL: \(operation.results.map { String(describing: $0.error) })") + let outputRoot = SafetyValidator.resolvedDestinationRoot( + source: fixture.source, destination: destination, settings: settings + ) + let statusFiles = try FileManager.default.contentsOfDirectory(at: outputRoot, includingPropertiesForKeys: nil) + .filter { TransferStatusMarker.hasExactName($0.lastPathComponent) } + XCTAssertEqual(statusFiles.count, 1) + let statusFile = try XCTUnwrap(statusFiles.first) + XCTAssertEqual(TransferStatusMarker.validatedStatus( + at: statusFile, relativePath: statusFile.lastPathComponent + ), .filesVerified) let plan = TransferCompletion.ascmhlPlan( results: operation.results, sourceFiles: operation.sourceManifest, @@ -188,6 +199,9 @@ final class Issue10ExFATHandoffTests: XCTestCase { print("ISSUE10 copy_passed=\(operation.results.allSatisfy(\.success)) mhlIssues=\(issues) taskCancelled=\(Task.isCancelled)") XCTAssertTrue(issues.isEmpty, "Copy/readback passed but MHL handoff failed on real exFAT: \(issues)") + XCTAssertEqual(TransferStatusMarker.validatedStatus( + at: statusFile, relativePath: statusFile.lastPathComponent + ), .filesVerified) XCTAssertEqual(try sourceContents(fixture.source), originalSource, "Source must remain byte-for-byte untouched") let updates = observed.values XCTAssertEqual(updates.first?.bytesProcessed, 0) diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedFilesystemFaultTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedFilesystemFaultTests.swift index e0424176..f30a9eaf 100644 --- a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedFilesystemFaultTests.swift +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedFilesystemFaultTests.swift @@ -23,9 +23,10 @@ final class MountedFilesystemFaultTests: XCTestCase { } var info = statfs() XCTAssertEqual(statfs(mount.path, &info), 0) - let actual = withUnsafePointer(to: &info.f_fstypename) { - $0.withMemoryRebound(to: CChar.self, capacity: 16) { String(cString: $0) } + let actual = withUnsafeBytes(of: info.f_fstypename) { bytes in + String(decoding: bytes.prefix(while: { $0 != 0 }), as: UTF8.self) } + print("MOUNTED_FAULT_MATRIX f_fstypename=\(actual)") XCTAssertEqual(actual, expected) scratch = FileManager.default.temporaryDirectory.appendingPathComponent("bitmatch-real-fault-\(UUID())") source = scratch.appendingPathComponent("Card") @@ -43,6 +44,70 @@ final class MountedFilesystemFaultTests: XCTestCase { } } + func testFATFamilyDirectPublicationAndHandoff() async throws { + let filesystem = ProcessInfo.processInfo.environment["BITMATCH_FAULT_MATRIX_TYPE"] + guard filesystem == "exfat" || filesystem == "msdos" else { + throw XCTSkip("This handoff exercises mounted exFAT and FAT32 only") + } + let directoryFD = Darwin.open(destination.path, O_RDONLY | O_DIRECTORY | O_CLOEXEC) + XCTAssertGreaterThanOrEqual(directoryFD, 0) + guard directoryFD >= 0 else { return } + defer { _ = Darwin.close(directoryFD) } + let probeName = "status-flush-probe-\(UUID().uuidString).txt" + let probeFD = probeName.withCString { + openat(directoryFD, $0, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0o600) + } + XCTAssertGreaterThanOrEqual(probeFD, 0) + guard probeFD >= 0 else { return } + defer { + _ = Darwin.close(probeFD) + try? FileManager.default.removeItem(at: destination.appendingPathComponent(probeName)) + } + let probeBytes = Data("visible incomplete warning".utf8) + XCTAssertEqual(probeBytes.withUnsafeBytes { Darwin.write(probeFD, $0.baseAddress, $0.count) }, probeBytes.count) + let fullSync = fcntl(probeFD, F_FULLFSYNC) + let fullError = fullSync == 0 ? 0 : errno + let fileSync = fsync(probeFD) + let fileError = fileSync == 0 ? 0 : errno + let rootSync = fsync(directoryFD) + let rootError = rootSync == 0 ? 0 : errno + print("MOUNTED_FAULT_MATRIX \(filesystem ?? "unknown") marker F_FULLFSYNC rc=\(fullSync) errno=\(fullError); marker fsync rc=\(fileSync) errno=\(fileError); root fsync rc=\(rootSync) errno=\(rootError)") + XCTAssertEqual(fileSync, 0) + XCTAssertEqual(rootSync, 0) + + let operation = try await run(destination, mode: .standard) + XCTAssertEqual(operation.results.count, 2) + XCTAssertTrue(operation.results.allSatisfy { + $0.success && $0.verificationResult?.isValid == true + }, "\(filesystem ?? "unknown"): \(operation.results.map { String(describing: $0.error) })") + + let output = SafetyValidator.resolvedDestinationRoot( + source: source, destination: destination, settings: CameraLabelSettings() + ) + let statuses = try FileManager.default.contentsOfDirectory(at: output, includingPropertiesForKeys: nil) + .filter { TransferStatusMarker.hasExactName($0.lastPathComponent) } + XCTAssertEqual(statuses.count, 1) + let status = try XCTUnwrap(statuses.first) + XCTAssertEqual(TransferStatusMarker.validatedStatus( + at: status, relativePath: status.lastPathComponent + ), .filesVerified) + + let handoff = TransferCompletion.ascmhlPlan( + results: operation.results, sourceFiles: operation.sourceManifest, + destinations: [destination], source: source, settings: CameraLabelSettings() + ) + XCTAssertEqual(handoff.jobs.count, 1) + XCTAssertTrue(handoff.issues.isEmpty) + let issues = try TransferCompletion.writeASCMHL( + handoff.jobs, planIssues: handoff.issues, startTime: operation.startTime, + source: source, toolVersion: "fat-family-probe", progress: { _ in } + ) + XCTAssertTrue(issues.isEmpty, "\(filesystem ?? "unknown") handoff: \(issues)") + XCTAssertEqual(TransferStatusMarker.validatedStatus( + at: status, relativePath: status.lastPathComponent + ), .filesVerified) + } + func testConflictingFilePreservedAcrossModes() async throws { for mode in VerificationMode.allCases { let root = try freshDestination(mode.rawValue) diff --git a/docs/EXFAT_PUBLICATION_DESIGN.md b/docs/EXFAT_PUBLICATION_DESIGN.md new file mode 100644 index 00000000..3545d3c1 --- /dev/null +++ b/docs/EXFAT_PUBLICATION_DESIGN.md @@ -0,0 +1,172 @@ +# exFAT/FAT media-file publication design + +This is the design for the exFAT/FAT media-file publication-race change. +Source cards remain read-only; a media file that BitMatch did not create is +never replaced or removed. + +## What is measured + +On a disposable mounted exFAT image, each of these calls preserved an existing +sentinel (`EEXIST`, errno 17), but failed against a vacant **file** name with +`ENOTSUP` (errno 45): + +| Call | Vacant target | Existing target | +| --- | --- | --- | +| `renameatx_np(..., RENAME_EXCL)` | `ENOTSUP` | `EEXIST`; unchanged | +| `linkat` | `ENOTSUP` | `EEXIST`; unchanged | +| `clonefileat(..., CLONE_NOFOLLOW)` | `ENOTSUP` | `EEXIST`; unchanged | + +The image was detached and deleted. The former `publishByClaimingName` path +had two separate deterministic red tests on APFS with the fallback forced: +replacing the empty claim immediately before `renameat` loses the other +writer's bytes, and replacing it immediately before failure cleanup loses +those bytes through `unlinkat`. These tests demonstrate the race in the code; +they do not establish that a customer has hit either race. + +Without a supported late atomic no-replace operation, exFAT cannot provide +all three properties at once: a normal final name that stays invisible until +fully written, no replacement of a concurrent writer's name, and cleanup of +only the file BitMatch owns. The fallback chooses no replacement and honest +incomplete-output evidence. Its normal final name may be visible while bytes +are still being written. + +## On-disk attempt marker + +For each exFAT/FAT destination root and transfer attempt, create exactly one +visible, warning-oriented status file directly under the pinned root: + +`BITMATCH_TRANSFER_STATUS_.txt` + +The name is generated independently of the transfer ID. Before creating it, +reject or regenerate any exact collision with the source manifest. Create it +using descriptor-relative `openat(O_CREAT|O_EXCL|O_NOFOLLOW|O_CLOEXEC)`. The +file starts with a strict magic/version and an explicit warning, the +transfer-attempt UUID and a random marker UUID, followed by numbered, +checksummed status records. It holds no footage names or paths. `incomplete` +is the first status. Write and flush +the marker, then fsync the destination-root directory. Only after both flushes +succeed may a final media name be created. If either flush fails, no final +media name is created. Never remove or rename the marker automatically. +For newly created output-root or nested folders on this direct-write path, +fsync each parent after `mkdirat` before writing media beneath the new name. +If a parent sync fails, that destination remains failed for the attempt. +These calls request durability from macOS and the drive; they cannot prove how +a particular device behaves after sudden power loss. + +BitMatch retains the open marker descriptor for that attempt and appends +statuses through it. A torn, missing, malformed, or unreadable record is +interpreted as incomplete, never complete. A marker is a warning/recovery +signal, **not** proof of copied or verified bytes. A file result and safe +verdict still require the existing independent readback, exact selected-file +coverage, ASC MHL/report work and durable transfer journal outcome. + +The marker only records file-evidence status, not a whole-transfer safe +verdict: + +1. `incomplete`: created and flushed before any direct final-name output. +2. `filesVerified`: append/fsync only when every selected file for this + destination has a valid independent readback result and exact coverage. +3. `copyCompleteUnverified`: Quick mode only; never safe to erase. +A copy or readback failure leaves `incomplete` as the last valid status. Later +MHL, report or journal failure can leave `filesVerified` while the whole +attempt remains unsafe. A failed append or torn record also leaves it unsafe. + +A successful `filesVerified` marker cannot turn a failed report, missing MHL, +cancelled run or failed journal commit into success. Conversely, an unreadable +or incomplete marker for the **current** attempt blocks a safe verdict, even +when readback results exist; the output's state would otherwise contradict +BitMatch's evidence. Previous attempt markers stay immutable and may still +say `incomplete` after a new attempt recovers the files. + +Marker names have an exact UUID shape and strict content format. Do not +blanket-filter `._*`, `.bitmatch*`, or an arbitrary hidden file from selected +source coverage. If a source file happens to have the chosen marker name, +choose another UUID before writing. Existing unrelated names are never +overwritten. The destination's final coverage check may enumerate marker +files; it must still check every selected source-relative path exactly and +must not use marker presence to excuse a missing or unexpected selected row. + +## Final file and recovery + +On exFAT/FAT, once the durable marker exists, create each new final media +name with `openat(O_CREAT|O_EXCL|O_NOFOLLOW|O_CLOEXEC)` below a pinned parent. +Keep that descriptor through the copy; set `F_NOCACHE`, write chunks, preserve +required metadata, flush the file, and sync its parent directory. Never +plain-rename a temporary file over a claimed final name. Never pathname-unlink +the final file after an error or cancellation. Leave uncertain partial bytes +in place, marked by the attempt status and an unsafe History result. A +replacement of the final name while the descriptor is open must fail the +attempt's identity/coverage checks; BitMatch must not delete or replace the +new occupant. APFS's current hard-link publication path remains separate. +The retained marker is checked immediately before each direct final-name +create, at the start of a file, and at 256 MiB write intervals. A missing or +changed marker stops further writes at the next check, and final coverage +checks it again. A deliberate deletion between checks cannot be made atomic +with a write on this filesystem; do not treat the marker as proof of complete +media. + +An unknown filesystem that rejects hard links fails closed with an actionable +message. It does not fall back to the old claim-and-rename path. This change +closes the confirmed final-name races. Existing cleanup of random +`.bitmatch.tmp.*` names on the APFS/hard-link path is best-effort: Darwin has +no atomic unlink-if-owned primitive, so a deliberate replacement of a temp +name at the exact cleanup boundary remains a separate design problem. Do not +describe this change as a universal namespace-race proof. + +The existing ASC MHL directory fallback is separate from media-file +publication. On exFAT it preserves a nonempty valid history, but a concurrent +writer could substitute an empty directory at its claim boundary. That +directory race needs its own ownership design and regression work; this media +protocol does not close it. + +The separate named-readback identity fix landed in PR #51 before this change. +The reader checks that the public name still identifies its open file, and +final coverage compares the current named file with the file used for +readback. This identity proof must remain ahead of status advancement; the +exFAT marker cannot substitute for it. + +On a new run or retry, a final name that already exists is never overwritten. +Standard/Thorough/Paranoid may reuse it only after the existing whole-file +source/destination proof and the normal independent readback. The result's +existing `wasReused` flag and "already existed and were verified" wording +describe that proof; a distinct "rechecked after an incomplete attempt" label +remains future UX work. Quick mode refuses existing-file reuse. A short or +mismatched file remains untouched and unsafe; guide the operator to choose a +new destination folder or resolve the old output manually. The earlier +History attempt remains interrupted/failed even if a later one succeeds. + +If the final name is absent, a new attempt may create its own marker and then +its own O_EXCL final file. An old marker remains as evidence of its old run. +The app may inspect prior marker records to explain recovery, but may not +trust them to skip verification. A marker moved or replaced by an external +actor is detected where possible through its retained descriptor and pinned +path identity; a hostile actor replacing it at the exact moment of a process +death cannot be fully defeated without a filesystem transaction. This marker +policy targets ordinary interruption and accidental concurrent file writes, +while no-overwrite of media names holds through O_EXCL. + +## Regression matrix before merge + +| Scenario | Required observation | +| --- | --- | +| Claim swapped just before old `renameat` | Red on current code; replacement bytes survive new code. | +| Claim swapped just before old cleanup | Red on current code; replacement bytes survive new code. | +| Marker creation/flush fails | No normal final media name created. | +| Crash/cancel after durable marker, before final create | Marker remains incomplete; no final file. | +| Crash/cancel during direct final write | Partial file and incomplete marker remain; no green result. | +| Crash after file flush, before readback/status | Existing file is rechecked, not silently treated as complete. | +| Final name swapped during direct write/readback | Other writer's bytes survive; result fails closed. | +| Existing mismatched or short file | No overwrite or delete; clear unsafe History/retry guidance. | +| Existing matching file in Standard | Independent whole-file proof, reused/recovered note, new attempt only may succeed. | +| Existing file in Quick | Reuse refused. | +| Same source name as a would-be marker | Exact source file still selected; marker name regenerated or setup fails safely. | +| Corrupt/torn/missing current marker | Never a safe verdict. | +| Mounted exFAT and APFS transfers | exFAT direct final FD, APFS hard-link path, full readback, MHL/report, cancellation and journal coverage. Mounted FAT32 transfer and sync behavior remain a follow-up because this Mac could not attach FAT images. | +| Deliberately swapped random temp name on APFS | Characterize separately; do not claim a conditional-unlink guarantee. | + +The original two red cases now exercise direct-final publication boundaries. +Mounted exFAT file and root-directory flushes, full Standard readback and ASC +MHL handoff passed on the test Mac. FAT32 remains untested here because this +Mac's `hdiutil` returned “Device not configured” while creating or attaching +FAT images, before the test could reach the filesystem. This is a test-host +limitation, not evidence that FAT32 supports or rejects these sync calls. diff --git a/docs/HARDWARE_COMPATIBILITY.md b/docs/HARDWARE_COMPATIBILITY.md index 8826a4b2..105467b2 100644 --- a/docs/HARDWARE_COMPATIBILITY.md +++ b/docs/HARDWARE_COMPATIBILITY.md @@ -1,6 +1,6 @@ # Hardware validation status -No physical-device test reports are recorded in this matrix yet. Mounted disk-image tests exercise APFS, HFS+, exFAT and FAT32, but they do not validate physical enclosures, hubs, readers or cables. Those combinations remain **unverified here** until a reproducible report is published. This is an evidence register, not a certification or a list of unsupported hardware. +No physical-device test reports are recorded in this matrix yet. The mounted disk-image matrix covers APFS, HFS+, exFAT and FAT32 where the test host can attach those images; FAT32 could not be mounted on this Mac for the current publication change. Disk images do not validate physical enclosures, hubs, readers or cables. Those combinations remain **unverified here** until a reproducible report is published. This is an evidence register, not a certification or a list of unsupported hardware. | Evidence type | Available workflow | What it establishes | | --- | --- | --- | @@ -31,7 +31,7 @@ What the code on `main` does for these cases, with where it lives. These are cod | Sony VENICE SxS card (UDF) on a Mac | If Disk Arbitration reports a removable disk with no readable filesystem and its description matches SxS, BitMatch shows a setup notice with Sony driver guidance. This notice is a heuristic, not a diagnosis of every Sony card. | `BitMatch/Core/Services/UnreadableMediaMonitor.swift` (`UnreadableMediaNotice.make`), shown by `BitMatch/Views/CopyAndVerify/UnreadableMediaBanner.swift`; tests `UnreadableMediaNoticeTests`, `UnreadableMediaMonitorTests` | None recorded. Detection matches "sxs", or "sony" with "card", in the disk's vendor, model or media name. | | Sony AXS card on a Mac | If an unreadable disk's description matches AXS, the same notice path links to Sony reader-software support. | same as above | None recorded. Matches "axs" in the disk's description. | | Any other card macOS sees but cannot read | A generic notice: it may need its maker's driver or use an unsupported format. | same as above | None recorded. | -| exFAT (and FAT) backup drive | Copy publication was fixed in 0.1.7; ASC MHL directory publication was fixed in 0.2.2 and is covered by the [handoff matrix](TRANSFER_MATRIX.md). exFAT has no hard links, so the `linkat` publish failed every file with "Destination file appeared during copy" in 0.1.4–0.1.6. On `ENOTSUP`/`EOPNOTSUPP` BitMatch now claims the final name with an exclusive create and renames the verified temp file onto that empty claim; an existing file still fails the file and is never replaced. | `Packages/BitMatchEngine/Sources/BitMatchEngine/File/DestinationWriter.swift` (`PinnedDestinationDirectory.publishTemporaryFile`, `publishByClaimingName`); test `Packages/BitMatchEngine/Tests/BitMatchEngineTests/ExFATDestinationTests.swift`, which mounts a real exFAT disk image | The bug was first found on real exFAT hardware (lukasisar/Bitmatch#4). The fix is tested on an exFAT disk image, not yet on a physical drive. | +| exFAT (and FAT) backup drive | BitMatch saves a visible `BITMATCH_TRANSFER_STATUS_.txt` warning before creating media files at their final names with exclusive create. It flushes new folder entries, each media file, and its name. An interrupted write leaves its partial file and incomplete warning in place; an existing file is never replaced. The warning alone does not mean a backup is safe to erase; check BitMatch History. A filesystem without hard links that BitMatch does not recognize fails closed with a clear error. ASC MHL directory publication remains covered by the [handoff matrix](TRANSFER_MATRIX.md). | `Packages/BitMatchEngine/Sources/BitMatchEngine/File/DestinationWriter.swift` and `TransferStatusMarker.swift`; mounted-image tests `ExFATDestinationTests` and `Issue10ExFATHandoffTests` | The original bug was found on real exFAT hardware (lukasisar/Bitmatch#4). Marker flush, root-folder sync, full Standard readback and ASC MHL handoff passed on a mounted exFAT image. No physical drive has been validated. FAT32 mounting failed on this test Mac before its sync behavior could be measured. | | A folder on the Mac's startup disk as a backup | Allowed when you pick it (for example in your home folder). The startup disk itself ("/", which `/Volumes/Macintosh HD` resolves to), anything under `/System`, and the root of an internal volume with a system name (Recovery, "Recovery 2", Preboot, any letter case) are refused with a reason. Drive discovery and the launch-time restore never add them. The engine's own preflight also refuses `/Library`, `/usr`, `/bin`, `/sbin`, `/private`, `/var` and `/etc`. | `Packages/BitMatchEngine/Sources/BitMatchEngine/File/BackupTargetPolicy.swift` (`refusal(for:origin:source:)`), `Packages/BitMatchEngine/Sources/BitMatchEngine/File/SafetyValidator.swift` (`isProtectedSystemPath`); tests `BackupTargetPolicyTests`, `BackupTargetPolicyRealVolumeTests` (real paths on the test Mac) | A mounted internal Recovery volume being offered as a backup was confirmed and fixed on a Mac (see [THESIS.md](THESIS.md), "Must fix before the next release"). | | iPad/iPhone backup chosen in Files (On My iPad, iCloud Drive, an external drive) | Works from 0.1.7. On a device these folders live below `/private/var/mobile`, which the system-folder rule refused, so every backup was refused as a "system folder". `/var/mobile` and `/private/var/mobile` are now user storage on iOS; other system folders stay protected. | `Packages/BitMatchEngine/Sources/BitMatchEngine/File/SafetyValidator.swift` (`isProtectedSystemPath`, `iOSUserStorageRoots`; commit `31bc940`); test `BitMatch-iPadTests/IOSStoragePathTests.swift` | **Not yet confirmed on a physical iPhone or iPad.** The simulator cannot show the bug: its paths are under the Mac's `/Users`. | diff --git a/docs/TODO.md b/docs/TODO.md index 3cbb4870..dca67dea 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -22,7 +22,7 @@ Unreleased work on `main`: terminal/History presentation, Mac card-event reconci | 4 | Distinguish local-source, local-backup and SFTP assurance (upload guards merged; broader assurance audit open) | High | Low–medium | Low–medium | Quick | | 5 | Expand the MHL interoperability corpus (rename/move fixture merged; classic interop open) | High | Medium | Low | Scoped | | 6 | Reconcile missed Mac card events (merged; physical check open) | Medium–high | Low | Low | Quick | -| 7 | Eliminate the exFAT/FAT publication race | Very high | High | High | Long; protected safety work | +| 7 | Eliminate the exFAT/FAT media-file publication race (implemented; FAT32 and physical checks open) | Very high | High | High | Field validation open | | 8 | Mark failed readback output unmistakably on disk | High | High | High | After publication ownership | | 9 | Mac menu bar companion and icon (merged; signed-app walkthrough open) | High | Medium | Medium | Scoped | | 10 | Per-drive copy/readback speed (merged; signed-app walkthrough open) | Medium–high | Medium | Low–medium | Scoped | @@ -86,7 +86,9 @@ The queue/recovery items marked complete below were merged in #20 for 0.2.8; see The priority buckets are not a strict implementation sequence: investigate P0 reports and audit assurance wording first, use physical benchmarks for default and scheduling decisions, and validate ASC MHL chains read-only before extending them. Source-read retry depends on fault classification and source identity; destination re-copy or repair additionally requires proof of output ownership. Neither retry path depends on MHL validation. -- [ ] **Close the exFAT/FAT media-file publication race.** The current no-hard-link fallback exclusively claims the final name, checks that it still owns the claim, then uses `renameat` to publish the verified temporary file. Another writer could replace that name between the check and rename; the failure cleanup has a similar check-then-unlink window. First turn adversarial publication cases into regressions for the current engine, then choose a no-overwrite design. Copying into the claimed descriptor is one candidate, but it makes an incomplete final-name file visible during interruption and needs an explicit recovery/presentation policy. Preserve the [thesis](THESIS.md)'s partial-copy promise. **Acceptance:** deterministic claim/replacement/rollback/interruption tests preserve any other writer's bytes and existing files; owned partial output stays unmistakably incomplete; actual exFAT/FAT and APFS runs confirm publication and recovery behavior; source media stays untouched and every uncertain result fails closed. +- [x] **Close the exFAT/FAT media-file publication race in code.** The no-hard-link FAT-family path now durably creates a visible incomplete-status warning before opening each media file at its final name with exclusive create. It never renames or removes that final name. An interrupted copy leaves its partial bytes and warning in place; existing files and another writer's bytes remain untouched. Exact marker validation, independent readback and named-file identity checks guard status advancement and the safe verdict. Deterministic replacement, cleanup, interruption and recovery regressions pass on the forced-direct path; mounted exFAT marker/root sync and full Standard-to-ASC-MHL handoff pass. See the [design and limits](EXFAT_PUBLICATION_DESIGN.md). **Remaining validation:** FAT32 images could not be mounted on this Mac, and no physical drive or sudden-power-loss result is recorded. Confirm FAT32 sync and transfer behavior on another host, then test representative physical drives without treating OS flush success as a power-loss guarantee. + +- [ ] **Close the ASC MHL exFAT empty-directory claim race.** This is separate from media-file publication. The current directory fallback preserves nonempty valid histories, but a concurrent writer could substitute an empty directory at the claim boundary. Add a deterministic replacement regression, then design a no-replacement directory handoff or fail closed while preserving the other writer's directory. Verify on mounted exFAT and retain independent MHL validation before accepting the new history. - [ ] **Make failed readback output unmistakable on disk.** BitMatch publishes a copied file at its normal destination name before readback; the existing corruption regression confirms that the bad file remains there even though the app verdict correctly fails closed. Investigate marking or moving only an attempt-owned, identity-checked file that fails verification. A read error does not prove the bytes are corrupt, and even bad bytes may be worth preserving. **Acceptance:** injected mismatch, read error, external replacement, interrupted marking and marking failure never overwrite, delete or relabel another writer's file; the real on-disk path and uncertainty appear in History/reports and recovery; source media stays untouched.