diff --git a/BitMatchTests/SafetyValidatorTests.swift b/BitMatchTests/SafetyValidatorTests.swift index f4d521ce..a781852a 100644 --- a/BitMatchTests/SafetyValidatorTests.swift +++ b/BitMatchTests/SafetyValidatorTests.swift @@ -269,7 +269,7 @@ final class SafetyValidatorTests: XCTestCase { guard case FileOperationError.unsafeOperation(let message) = error else { return XCTFail("Expected unsafeOperation, got \(error)") } - XCTAssertTrue(message.contains("collide on case-insensitive filesystems")) + XCTAssertTrue(message.contains("differ only by case or Unicode form")) } } diff --git a/CHANGELOG.md b/CHANGELOG.md index a7655251..c250543b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +- Fixed crashes caused by Unicode-equivalent filenames. Conflicting source names are refused before copying; conflicting existing backup names prevent a successful result. + - Check no longer reports preserved Finder metadata as missing when it is listed in the saved checksums. - Mac: Corrected physical-drive checks for selected subfolders, so two folders on one drive count as one backup. - Cleaned up long names and page breaks in PDFs; metadata companions no longer inflate the clip-preview count. Job details now share the page footer instead of creating a nearly empty last page. diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/SafetyValidator.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/SafetyValidator.swift index f3d5c93d..a3a9de15 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/File/SafetyValidator.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/File/SafetyValidator.swift @@ -428,9 +428,11 @@ public final class SafetyValidator { .precomposedStringWithCanonicalMapping .lowercased(with: locale) - if let original = seen[portableKey], original != relativePath { + // Swift String equality includes canonical equivalence. Two distinct + // directory entries (for example K and K on HFS+) can compare equal. + if let original = seen[portableKey] { throw FileOperationError.unsafeOperation( - "Source contains paths that collide on case-insensitive filesystems: \(original) and \(relativePath)" + "Source contains names that differ only by case or Unicode form: \(original) and \(relativePath)" ) } seen[portableKey] = relativePath diff --git a/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferPipeline.swift b/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferPipeline.swift index d9426677..d40289c9 100644 --- a/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferPipeline.swift +++ b/Packages/BitMatchEngine/Sources/BitMatchEngine/TransferPipeline.swift @@ -247,6 +247,18 @@ private func completionGateError(_ message: String) -> NSError { ) } +func uniqueDestinationMetadata(_ entries: [FileEntry]) throws -> [String: FileEntry] { + var byPath: [String: FileEntry] = [:] + for entry in entries { + // Some filesystems allow distinct names that Swift compares as equal. + // Never trap or discard an entry while checking authoritative results. + guard byPath.updateValue(entry, forKey: entry.relativePath) == nil else { + throw completionGateError("Destination contains identical or Unicode-equivalent names. Keep the source media.") + } + } + return byPath +} + /// Owns the single operation admitted by a service instance. /// Cancellation never releases the slot; only the matching operation's exit does. public final class ActiveOperationRegistry: Sendable { @@ -499,6 +511,9 @@ public final class TransferPipeline: FileOperationsService, Sendable { // Step 2: Build one fail-closed source manifest before safety validation. SharedLogger.debug("Prep: enumerating source manifest at \(operation.sourceURL.path)", category: .transfer) let completeSourceManifest = try CardSource.enumerateRegularFiles(base: operation.sourceURL) + // Validate these captured paths before exclusions or String-keyed maps. + // A later live scan cannot prove this manifest has unambiguous names. + try SafetyValidator.validatePortableRelativePaths(completeSourceManifest.map(\.relativePath)) let excluded = try AppleDoubleSelection.excluded(in: completeSourceManifest, reviewedPaths: operation.settings.excludedAppleDoublePaths) if !excluded.isEmpty { SharedLogger.transferEvent(.appleDoubleExcluded, run: TransferDiagnostics.runID ?? operation.id, code: excluded.count) @@ -1059,9 +1074,7 @@ public final class TransferPipeline: FileOperationsService, Sendable { guard let pinned = pinnedDestinations[destIndex] else { continue } let actual: [String: FileEntry] do { - actual = Dictionary( - uniqueKeysWithValues: try pinned.regularFileMetadata().map { ($0.relativePath, $0) } - ) + actual = try uniqueDestinationMetadata(pinned.regularFileMetadata()) } catch { let reason = "\(destinationURL.lastPathComponent)'s files could not be checked after verification: \(error.localizedDescription)" let failure = completionGateError(reason) diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedUnicodeSourceTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedUnicodeSourceTests.swift new file mode 100644 index 00000000..a62d1d8b --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/MountedUnicodeSourceTests.swift @@ -0,0 +1,303 @@ +#if os(macOS) +import CryptoKit +import Darwin +import Foundation +import XCTest +@testable import BitMatchEngine + +/// HFS+ can hold K and the canonically equivalent Kelvin sign simultaneously. +/// Raw UTF-8 Data keys keep the fixture's own source checks from merging them. +final class MountedUnicodeSourceTests: XCTestCase { + func testKelvinNamesFailBeforeChangingPriorMediaOrHistory() async throws { + guard ProcessInfo.processInfo.environment["BITMATCH_UNICODE_HFS"] == "1" else { + throw XCTSkip("Set BITMATCH_UNICODE_HFS=1 for a disposable HFS+ image") + } + try await FileOperationsTestLock.shared.run { + let fixture = try UnicodeSourceFixture() + do { + try await fixture.exercise() + try fixture.finish(remove: true) + } catch { + try? fixture.finish(remove: false) + print("UNICODE_SOURCE retained=\(fixture.root.path) error=\(error)") + throw error + } + } + } + + func testExistingKelvinDestinationFailsClosedAndPreservesPriorHistory() async throws { + guard ProcessInfo.processInfo.environment["BITMATCH_UNICODE_HFS"] == "1" else { + throw XCTSkip("Set BITMATCH_UNICODE_HFS=1 for a disposable HFS+ image") + } + try await FileOperationsTestLock.shared.run { + let fixture = try UnicodeSourceFixture(size: "2g") + do { + try await fixture.exerciseAmbiguousDestination() + try fixture.finish(remove: true) + } catch { + try? fixture.finish(remove: false) + print("UNICODE_DESTINATION retained=\(fixture.root.path) error=\(error)") + throw error + } + } + } +} + +private final class UnicodeSourceFixture { + struct Entry: Equatable { + let device: dev_t + let inode: ino_t + let kind: mode_t + let size: Int64 + let modifiedSeconds: Int + let modifiedNanos: Int + let sha256: String + } + let root: URL + private let mount: URL + private let image: URL + private let nonce = UUID().uuidString + private var rootIdentity: (dev_t, ino_t)? + private var imageIdentity: (dev_t, ino_t)? + private var volumeUUID: String? + private let fm = FileManager.default + + init(size: String = "256m") throws { + root = URL(fileURLWithPath: "/private/tmp/bitmatch-unicode-source-\(UUID())", isDirectory: true) + mount = root.appendingPathComponent("mount") + image = root.appendingPathComponent("source.sparseimage") + try fm.createDirectory(at: root, withIntermediateDirectories: false) + var owned = stat() + try require(lstat(root.path, &owned) == 0 && owned.st_mode & S_IFMT == S_IFDIR, "Invalid owned root") + rootIdentity = (owned.st_dev, owned.st_ino) + try Data(nonce.utf8).write(to: root.appendingPathComponent(".owned"), options: .withoutOverwriting) + try fm.createDirectory(at: mount, withIntermediateDirectories: false) + _ = try command("/usr/bin/hdiutil", ["create", "-quiet", "-size", size, "-type", "SPARSE", "-fs", "Journaled HFS+", "-volname", "BMUNI", "-o", image.path]) + var imageStat = stat() + try require(lstat(image.path, &imageStat) == 0 && imageStat.st_mode & S_IFMT == S_IFREG, "Invalid new image") + imageIdentity = (imageStat.st_dev, imageStat.st_ino) + do { + let attached = try plist(command("/usr/bin/hdiutil", ["attach", "-plist", "-nobrowse", "-mountpoint", mount.path, image.path])) + let points = (attached["system-entities"] as? [[String: Any]] ?? []).compactMap { $0["mount-point"] as? String } + try require(points == [mount.path], "Unexpected image mountpoints") + let disk = try plist(command("/usr/sbin/diskutil", ["info", "-plist", mount.path])) + try require(disk["FilesystemType"] as? String == "hfs" && disk["MountPoint"] as? String == mount.path, "Expected actual HFS+ driver") + volumeUUID = disk["VolumeUUID"] as? String + try require(volumeUUID != nil, "Missing mounted volume identity") + } catch { + try? finish(remove: false) + print("UNICODE_SOURCE setup_retained=\(root.path)") + throw error + } + } + + func exercise() async throws { + let previous = root.appendingPathComponent("previous/Card") + let backup = root.appendingPathComponent("backup") + try fm.createDirectory(at: previous, withIntermediateDirectories: true) + try fm.createDirectory(at: backup, withIntermediateDirectories: false) + let priorBytes = Data("prior verified content; must survive".utf8) + try priorBytes.write(to: previous.appendingPathComponent("prior.bin")) + let baseline = try await transfer(previous, backup) + try require(baseline.results.count == 1 && baseline.results.allSatisfy { $0.success && $0.verificationResult?.isValid == true }, "Baseline transfer failed") + let plan = TransferCompletion.ascmhlPlan(results: baseline.results, sourceFiles: baseline.sourceManifest, destinations: [backup], source: previous, settings: .init()) + try require(plan.jobs.count == 1 && plan.issues.isEmpty, "Baseline handoff missing") + let issues = try TransferCompletion.writeASCMHL(plan.jobs, planIssues: plan.issues, startTime: baseline.startTime, source: previous, toolVersion: "unicode-source-regression") + try require(issues.isEmpty, "Baseline MHL failed") + let copied = SafetyValidator.resolvedDestinationRoot(source: previous, destination: backup, settings: .init()) + let destinationBefore = try snapshot(copied) + let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + let checkedBefore = try await checker.check(checker.discover(under: copied)) { _ in } + try require(checkedBefore.isIntact && checkedBefore.recordedFileCount == 1, "Baseline history not intact") + + let source = mount.appendingPathComponent("Card") + try fm.createDirectory(at: source, withIntermediateDirectories: false) + try writeKelvinNames(in: source) + let before = try snapshot(source) + let expected = Set([Data("K.mov".utf8), Data("\u{212A}.mov".utf8)]) + try require(Set(before.keys) == expected && before.count == 2, "Raw source names did not remain distinct") + try require(Set(before.values.map(\.inode)).count == 2, "Driver names alias one inode") + try require("K.mov" == "\u{212A}.mov", "Fixture must exercise Swift canonical equality") + let captured = try CardSource.enumerateRegularFiles(base: source) + try require(captured.count == 2 && Set(captured.map { Data($0.relativePath.utf8) }) == expected, "Manifest lost raw filename coverage") + do { + _ = try await transfer(source, backup) + throw failure("Ambiguous source returned a completed operation") + } catch FileOperationError.unsafeOperation(let reason) { + try require(reason.contains("Unicode"), "Unexpected preflight refusal") + } + try require(try snapshot(source) == before, "Source bytes, names or identities changed") + try require(try snapshot(copied) == destinationBefore, "Prior media, history or destination inventory changed") + let checkedAfter = try await checker.check(checker.discover(under: copied)) { _ in } + try require(checkedAfter.isIntact && checkedAfter.recordedFileCount == 1, "Prior history no longer passes Check") + print("UNICODE_SOURCE PASS rawDistinct=2 swiftEquivalent=true refusedBeforeCopy=true priorHistoryPreserved=true sourcePreserved=true") + } + + func exerciseAmbiguousDestination() async throws { + let previous = root.appendingPathComponent("previous/Card") + let backup = mount.appendingPathComponent("backup") + try fm.createDirectory(at: previous, withIntermediateDirectories: true) + try fm.createDirectory(at: backup, withIntermediateDirectories: false) + try Data("prior verified content; must survive".utf8).write(to: previous.appendingPathComponent("prior.bin")) + let baseline = try await transfer(previous, backup) + try require(baseline.results.count == 1 && baseline.results.allSatisfy { $0.success && $0.verificationResult?.isValid == true }, "Baseline HFS+ transfer failed") + let plan = TransferCompletion.ascmhlPlan(results: baseline.results, sourceFiles: baseline.sourceManifest, destinations: [backup], source: previous, settings: .init()) + try require(plan.jobs.count == 1 && plan.issues.isEmpty, "Baseline HFS+ handoff missing") + let issues = try TransferCompletion.writeASCMHL(plan.jobs, planIssues: plan.issues, startTime: baseline.startTime, source: previous, toolVersion: "unicode-destination-regression") + try require(issues.isEmpty, "Baseline HFS+ MHL failed") + let copied = SafetyValidator.resolvedDestinationRoot(source: previous, destination: backup, settings: .init()) + let checker = SavedChecksumCheck(fileAccess: LocalFileAccess(), checksum: ChecksumEngine.shared) + let checked = try await checker.check(checker.discover(under: copied)) { _ in } + try require(checked.isIntact && checked.recordedFileCount == 1, "Baseline HFS+ history not intact") + + try writeKelvinNames(in: copied) + let before = try snapshot(copied) + let first = try unwrap(before[Data("K.mov".utf8)], "Missing ASCII destination") + let second = try unwrap(before[Data("\u{212A}.mov".utf8)], "Missing Kelvin destination") + try require(first.inode != second.inode, "Destination fixture must contain two actual entries") + let source = root.appendingPathComponent("fresh/Card") + try fm.createDirectory(at: source, withIntermediateDirectories: true) + let sourceBytes = Data("unambiguous fresh source".utf8) + try sourceBytes.write(to: source.appendingPathComponent("new.mov")) + let sourceBefore = try snapshot(source) + let operation = try await transfer(source, backup) + try require(operation.results.count == 1, "Missing selected-file result") + for row in operation.results { + try require(!row.success && row.verificationResult == nil, "Ambiguous destination received a successful result") + let error = row.error as NSError? + try require(error?.domain == "TransferPipeline" && error?.code == NSFileReadUnknownError && error?.localizedDescription.contains("Unicode-equivalent") == true, "Wrong authoritative-results failure") + } + let rejected = TransferCompletion.ascmhlPlan(results: operation.results, sourceFiles: operation.sourceManifest, destinations: [backup], source: source, settings: .init()) + try require(rejected.jobs.isEmpty && !rejected.issues.isEmpty, "Ambiguous destination received a complete-history job") + let rejectedIssues = try TransferCompletion.writeASCMHL(rejected.jobs, planIssues: rejected.issues, startTime: operation.startTime, source: source, toolVersion: "unicode-destination-regression") + let verdict = TransferCompletion.verdict(rows: TransferCompletion.rows(from: operation), sourceFiles: operation.sourceManifest, destinations: [backup], source: source, settings: .init(), mode: .standard, generateASCMHL: true, handoffIssues: rejectedIssues, reportIssue: nil, project: .init(didPersist: true, locallySafe: nil)) + try require(!verdict.success, "Ambiguous destination received a safe verdict") + let after = try snapshot(copied) + try require(before.allSatisfy { after[$0.key] == $0.value }, "Prior media or history changed") + try require(after.count == before.count + 1 && after[Data("new.mov".utf8)] != nil, "Unexpected destination inventory change") + try require(try Data(contentsOf: copied.appendingPathComponent("new.mov")) == sourceBytes, "Retained copied bytes differ") + try require(try snapshot(source) == sourceBefore, "Unambiguous source changed") + print("UNICODE_DESTINATION PASS rawDistinct=2 failedAuthoritativeGate=true noNewMHL=true priorHistoryPreserved=true sourcePreserved=true") + } + + private func writeKelvinNames(in directoryURL: URL) throws { + let directory = open(directoryURL.path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW) + try require(directory >= 0, "Cannot open owned fixture directory") + defer { close(directory) } + for (name, bytes) in [("K.mov", Data("ASCII K original".utf8)), ("\u{212A}.mov", Data("Kelvin original".utf8))] { + let fd = name.withCString { openat(directory, $0, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, 0o600) } + try require(fd >= 0, "Actual driver did not create distinct Kelvin names") + let wrote = bytes.withUnsafeBytes { Darwin.write(fd, $0.baseAddress, $0.count) } + let synced = fsync(fd) + close(fd) + try require(wrote == bytes.count && synced == 0, "Synthetic fixture write failed") + } + } + + private func transfer(_ source: URL, _ backup: URL) async throws -> FileOperation { + try await TransferPipeline(fileSystem: LocalFileAccess(), checksum: ChecksumEngine.shared) + .performFileOperation(sourceURL: source, destinationURLs: [backup], verificationMode: .standard, settings: .init(), progressCallback: { _ in }, onFileResult: nil) + } + + private func snapshot(_ url: URL) throws -> [Data: Entry] { + let fd = open(url.path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW) + try require(fd >= 0, "Snapshot root unavailable") + defer { close(fd) } + var entries: [Data: Entry] = [:] + try walk(fd, prefix: Data(), entries: &entries) + return entries + } + + private func walk(_ fd: Int32, prefix: Data, entries: inout [Data: Entry]) throws { + let scan = dup(fd) + try require(scan >= 0, "Snapshot descriptor unavailable") + guard let directory = fdopendir(scan) else { close(scan); throw failure("Snapshot enumeration failed") } + defer { closedir(directory) } + while true { + errno = 0 + guard let raw = readdir(directory) else { try require(errno == 0, "Snapshot enumeration interrupted"); break } + var item = raw.pointee + let bytes = withUnsafeBytes(of: &item.d_name) { Data($0.prefix { $0 != 0 }) } + guard bytes != Data(".".utf8) && bytes != Data("..".utf8) else { continue } + let name = try unwrap(String(data: bytes, encoding: .utf8), "Invalid synthetic UTF-8") + var info = stat() + try require(name.withCString { fstatat(fd, $0, &info, AT_SYMLINK_NOFOLLOW) } == 0, "Snapshot entry disappeared") + let kind = info.st_mode & S_IFMT + try require(kind == S_IFREG || kind == S_IFDIR, "Unexpected symlink or special entry") + let relative = prefix + bytes + var sha = "" + if kind == S_IFDIR { + let sub = name.withCString { openat(fd, $0, O_RDONLY | O_DIRECTORY | O_NOFOLLOW) } + try require(sub >= 0, "Snapshot directory unavailable") + defer { close(sub) } + try walk(sub, prefix: relative + Data([47]), entries: &entries) + } else { + try require(info.st_size <= 1_048_576, "Unexpected large fixture file") + let file = name.withCString { openat(fd, $0, O_RDONLY | O_NOFOLLOW) } + try require(file >= 0, "Snapshot file unavailable") + defer { close(file) } + var data = Data(count: Int(info.st_size)) + let count = data.withUnsafeMutableBytes { pread(file, $0.baseAddress, $0.count, 0) } + try require(count == data.count, "Incomplete snapshot read") + sha = SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined() + } + entries[relative] = Entry(device: info.st_dev, inode: info.st_ino, kind: kind, size: Int64(info.st_size), modifiedSeconds: info.st_mtimespec.tv_sec, modifiedNanos: info.st_mtimespec.tv_nsec, sha256: sha) + } + } + + func finish(remove: Bool) throws { + let info = try plist(command("/usr/bin/hdiutil", ["info", "-plist"])) + let matches = (info["images"] as? [[String: Any]] ?? []).filter { $0["image-path"] as? String == image.path } + if !matches.isEmpty { + try require(matches.count == 1, "Ambiguous backing image") + let points = (matches[0]["system-entities"] as? [[String: Any]] ?? []).compactMap { $0["mount-point"] as? String } + try require(points == [mount.path], "Owned mount mapping changed") + var named = stat() + try require(lstat(image.path, &named) == 0 && named.st_dev == imageIdentity?.0 && named.st_ino == imageIdentity?.1, "Owned image replaced") + if let volumeUUID { + let disk = try plist(command("/usr/sbin/diskutil", ["info", "-plist", mount.path])) + try require(disk["VolumeUUID"] as? String == volumeUUID, "Owned volume replaced") + } + _ = try command("/usr/bin/hdiutil", ["detach", "-quiet", mount.path]) + } + let final = try plist(command("/usr/bin/hdiutil", ["info", "-plist"])) + try require(!(final["images"] as? [[String: Any]] ?? []).contains { $0["image-path"] as? String == image.path }, "Image remains attached") + if remove { + var mounts: UnsafeMutablePointer? + let count = getmntinfo(&mounts, MNT_NOWAIT) + try require(count > 0 && mounts != nil, "Cannot establish mount absence") + let remaining = (0.. Data { + let name = root.appendingPathComponent("command-\(UUID())") + let stdout = name.appendingPathExtension("stdout"), stderr = name.appendingPathExtension("stderr") + try Data().write(to: stdout); try Data().write(to: stderr) + let output = try FileHandle(forWritingTo: stdout), errors = try FileHandle(forWritingTo: stderr) + defer { try? output.close(); try? errors.close() } + let process = Process() + process.executableURL = URL(fileURLWithPath: executable); process.arguments = arguments + process.standardOutput = output; process.standardError = errors + try process.run(); process.waitUntilExit() + try require(process.terminationStatus == 0, "Command failed; logs retained") + try output.synchronize() + return try Data(contentsOf: stdout) + } + private func plist(_ data: Data) throws -> [String: Any] { + try unwrap(PropertyListSerialization.propertyList(from: data, format: nil) as? [String: Any], "Malformed command plist") + } + private func require(_ value: Bool, _ message: String) throws { if !value { throw failure(message) } } + private func failure(_ message: String) -> NSError { NSError(domain: "MountedUnicodeSource", code: 1, userInfo: [NSLocalizedDescriptionKey: message]) } + private func unwrap(_ value: T?, _ message: String) throws -> T { guard let value else { throw failure(message) }; return value } +} +#endif diff --git a/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SourceFilenameAmbiguityTests.swift b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SourceFilenameAmbiguityTests.swift new file mode 100644 index 00000000..d86b63e1 --- /dev/null +++ b/Packages/BitMatchEngine/Tests/BitMatchEngineTests/SourceFilenameAmbiguityTests.swift @@ -0,0 +1,70 @@ +import Foundation +import XCTest +@testable import BitMatchEngine + +final class SourceFilenameAmbiguityTests: XCTestCase { + func testCanonicallyEqualByteDistinctPathsAreRefusedInEitherOrder() { + let pairs = [("K.mov", "\u{212A}.mov"), ("caf\u{E9}.mov", "cafe\u{301}.mov")] + for (first, second) in pairs { + XCTAssertEqual(first, second, "This regression needs Swift-equal names") + XCTAssertNotEqual(Data(first.utf8), Data(second.utf8)) + for paths in [[first, second], [second, first]] { + assertUnsafe(paths) + } + } + } + + func testRepeatedIdenticalPathCannotBeAnUnambiguousManifest() { + assertUnsafe(["clip.mov", "clip.mov"]) + } + + func testAsciiCaseAliasesAreRefusedInEitherOrder() { + for paths in [["clip.mov", "CLIP.mov"], ["CLIP.mov", "clip.mov"]] { + assertUnsafe(paths) + } + } + + func testEachSingleCanonicalSpellingIsAllowed() { + for path in ["K.mov", "\u{212A}.mov", "caf\u{E9}.mov", "cafe\u{301}.mov"] { + XCTAssertNoThrow(try SafetyValidator.validatePortableRelativePaths([path])) + } + } + + func testDistinctEmojiAndPathsInSeparateDirectoriesRemainAllowed() { + XCTAssertNoThrow(try SafetyValidator.validatePortableRelativePaths([ + "\u{1F469}\u{200D}\u{1F4BB}.mov", "\u{1F469}\u{1F4BB}.mov", + "\u{1F3AC}.mov", "\u{1F3AC}\u{FE0F}.mov", "日本語.mov", + "nfc/caf\u{E9}.mov", "nfd/cafe\u{301}.mov" + ])) + } + + func testDestinationCanonicalAliasesAreRefusedWithoutDiscardingAnEntry() { + for names in [["K.mov", "\u{212A}.mov"], ["\u{212A}.mov", "K.mov"], ["caf\u{E9}.mov", "cafe\u{301}.mov"]] { + XCTAssertThrowsError(try uniqueDestinationMetadata(names.map(entry))) { error in + let failure = error as NSError + XCTAssertEqual(failure.domain, "TransferPipeline") + XCTAssertEqual(failure.code, NSFileReadUnknownError) + XCTAssertTrue(failure.localizedDescription.contains("Unicode-equivalent")) + } + } + } + + func testDestinationCaseVariantsAndUnrelatedUnicodeExtrasRemainDistinct() throws { + let names = ["clip.mov", "CLIP.mov", "nfc/caf\u{E9}.mov", "nfd/cafe\u{301}.mov", "\u{1F469}\u{200D}\u{1F4BB}.mov", "\u{1F469}\u{1F4BB}.mov"] + let actual = try uniqueDestinationMetadata(names.map(entry)) + XCTAssertEqual(actual.count, names.count) + for name in names { XCTAssertEqual(actual[name]?.relativePath, name) } + } + + private func entry(_ path: String) -> FileEntry { + FileEntry(url: URL(fileURLWithPath: "/synthetic").appendingPathComponent(path), relativePath: path, size: 1) + } + + private func assertUnsafe(_ paths: [String], file: StaticString = #filePath, line: UInt = #line) { + XCTAssertThrowsError(try SafetyValidator.validatePortableRelativePaths(paths), file: file, line: line) { error in + guard case FileOperationError.unsafeOperation = error else { + return XCTFail("Expected an unsafe source error, got \(error)", file: file, line: line) + } + } + } +} diff --git a/docs/validation/UNICODE_SOURCE_PREFLIGHT_20261020.md b/docs/validation/UNICODE_SOURCE_PREFLIGHT_20261020.md new file mode 100644 index 00000000..f90ede05 --- /dev/null +++ b/docs/validation/UNICODE_SOURCE_PREFLIGHT_20261020.md @@ -0,0 +1,62 @@ +# Ambiguous filename handling — October 20, 2026 + +A disposable journaled HFS+ image reproduced a crash before copying: it held both `K.mov` and `K.mov` (the Kelvin sign), with different contents and distinct raw UTF-8 names and inodes. Swift compares those names as equal. The existing portable-name check exempted Swift-equal names, and a later unique-key dictionary trapped. A contained command-line worker built from the unchanged engine at `953c779f` exited by SIGTRAP with “Duplicate values for key.” Source hashes and the existing destination sentinel stayed unchanged. + +The shared engine now rejects repeated portable-name keys and checks the captured **complete** manifest immediately after enumeration, before AppleDouble selection, sets, fingerprints or path dictionaries. Existing live-tree safety checks remain. A later live scan cannot establish that an earlier captured list was unambiguous. This is a preflight refusal, not a filename rewrite or a change to verification defaults. + +The same source in the patched worker returned a normal unsafe-source error, preserving both source files and the destination sentinel. The original crashing binary, source hashes, byte-exact snapshots and failed-worker logs are retained privately. No app host or physical drive was used to reproduce the crash. + +A second contained worker reproduced the same kind of trap in the final destination-metadata check. An unambiguous source was copied into an existing HFS+ folder containing `K.mov` and `K.mov`. Both prior files survived, but the unique-key dictionary trapped instead of returning a result. The engine now builds that metadata map with an explicit duplicate check. A duplicate fails the existing authoritative-results gate: selected files receive failed results, no complete MHL job is created, and the verdict remains unsafe. This check uses Swift string equality only; it does not impose a new case-folding rule on unrelated destination files. Copying may already have happened when this final check refuses completion. + +## Coverage + +Five ordinary regressions require the specific unsafe-source error for canonically equivalent or repeated paths, retain the existing ASCII-case refusal, and allow individual Unicode spellings, distinct emoji sequences and equivalent spellings in different directories. + +Two further ordinary regressions require the precise destination completion error for canonical aliases and preserve distinct case-only or unrelated Unicode destination entries. + +Two opt-in HFS+ regressions create the actual Kelvin pair with exclusive POSIX writes. Snapshots use raw UTF-8 `Data` keys rather than `String`, so the tests cannot accidentally merge the two names. Each first completes a tiny Standard transfer, ASC MHL and Saved Check. The ambiguous-source case must fail before returning an operation, preserve every source entry and prior destination/history snapshot, and leave the prior history passing a fresh Check. The ambiguous-destination case must return a failed selected-file result, create no new MHL history, preserve all prior media and history, retain the newly copied bytes and leave the source unchanged. It does not require a new Check to treat the deliberately added ambiguous files as intact. Required assertions throw before successful fixture cleanup. The final focused run passed all nine methods with zero failures or skips. + +```sh +BITMATCH_UNICODE_HFS=1 swift test \ + --package-path Packages/BitMatchEngine --jobs 4 \ + --disable-swift-testing \ + --filter 'SourceFilenameAmbiguityTests|MountedUnicodeSourceTests' +``` + +Without the opt-in, both image tests skip. Unsupported setup fails rather than being counted as a passing mounted case. The fixture normally detaches only its identity-checked image and refuses recursive cleanup while any filesystem remains mounted beneath its owned root. Failed fixtures remain available. Native filesystem calls are not forcibly preempted. + +Full engine gates passed separately in Debug and Release, with all environment-controlled stress, hardware and mounted-image cases disabled: + +| Configuration | Framework | Passed | Skipped | Failed | +| --- | --- | ---: | ---: | ---: | +| Debug | XCTest | 195 | 43 | 0 | +| Debug | Swift Testing | 217 | 6 | 0 | +| Release | XCTest | 195 | 43 | 0 | +| Release | Swift Testing | 217 | 6 | 0 | + +XCTest scheduled 238 methods in each configuration. Swift Testing reported 223 declarations in 40 suites, including its six explicit skips. The enabled nine-method focused run above supplies the real mounted HFS+ evidence; the ordinary full runs do not repeat it. + +## Application gates + +The full Mac app suite passed 1,282 tests, with 23 skips and zero failures. The first run failed one assertion that expected the previous error message; after updating that assertion, the fresh full run passed. The original failed result remains retained. Both structured and raw runtime UI-warning checks were clear; compiler warnings are retained separately, without an all-warnings-free claim. + +Unsigned shared iOS-device and universal Mac Release builds passed. A local universal diagnostic archive also passed deep, strict signature verification using the unchanged signing configuration. It was not installed, exported, notarized or published. Before/after hashes bind all 444 Swift files to the tested code in `546c3d0d`; later validation-document commits do not change that code. These gates do not establish a physical mobile-device or final signed-UI walkthrough. + +A fresh x86_64-only Release bundle also passed the same nine explicitly selected XCTest methods under Rosetta: nine passed, zero skipped or failed. Both disposable HFS+ cases ran and normally cleaned their fixtures. Xcode's signed universal XCTest host loaded the translated bundle; no private unsigned test loader or signing credentials were needed. This is translated engine coverage on Apple Silicon, not a physical Intel Mac, older macOS runtime or application walkthrough. + +## Driver observations and limits + +A separate raw POSIX probe attempted 13 name pairs on each of APFS, case-sensitive APFS, journaled HFS+, case-sensitive journaled HFS+, exFAT and FAT32: **78 pair probes**, not 78 application transfers or XCTest methods. Receipt keys are filename bytes. OS-created AppleDouble entries are counted separately from principal files. + +- Every tested Mac driver rejected the second exclusive create for NFC `café.mov` versus NFD `café.mov`. These attempts do not prove that foreign-created media can never contain problematic names. +- HFS+, case-sensitive HFS+ and FAT32 allowed the Kelvin pair. APFS, case-sensitive APFS and exFAT refused it. +- Emoji joiners, variation selectors and non-ASCII case mappings did not behave uniformly across drivers. These observations are not a general filename-portability guarantee. +- A case-sensitive APFS source with `clip.mov` and `CLIP.mov`, containing identical bytes, was refused by the existing case-collision preflight. Sigma/final-sigma sources with identical and different bytes both settled unsafe on the case-insensitive APFS destination, with no complete MHL or green verdict. They did not establish a second unsafe-outcome bug. + +Initial engine probes used 256 MiB destination images and correctly stopped at the existing 1 GB free-space reserve. Those setup refusals were retained; the crash and normal-refusal controls used an owned host APFS destination with sufficient real free space. No reserve or other safety rule was bypassed. + +The destination crash control used a separate 2 GiB HFS+ image with sufficient free space. Its patched worker returned normally with failed results and unsafe verdicts both with and without MHL generation. An initial rebuild of that private worker omitted its testability compiler flag and failed compilation; the corrected build and actual worker run are retained separately from engine test results. + +All seven research images were normally detached after identity checks, and their exact backing files were removed. Small receipts, logs, binaries and snapshots remain. A first cleanup comparison stopped before detaching anything because the initial driver probe had hashed only the first 1,024 bytes of each file. The corrected comparison matches that original scope: complete contents for the tiny principal payloads, a prefix for 4 KiB AppleDouble companions. Full current companion hashes were also saved; they do not establish full original-companion preservation. The separate application regressions use full-file snapshots for their source and history assertions. + +These are generated tiny files on host-backed images, not camera footage, USB/dock reliability or a physical-device walkthrough. The disappearing-between-scans race was not directly injected; validating the captured manifest before its consumers closes that gap by construction. This crash is **not established as the cause of issue #10 or #23**; both field investigations remain separate.