diff --git a/README.md b/README.md index c52b8d3..1bb4f52 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,7 @@ services: #- PIPELINE_SKIP_DOWNLOADS=false #- PIPELINE_SKIP_DOWNLOADS_IMPORTS=false #- PIPELINE_FORCE_REDOWNLOAD=false + #- PIPELINE_FORCE_REIMPORT_EXISTING=false #- PIPELINE_SKIP_POSTERS=false #- PIPELINE_INCLUDE_SPECIALS=false @@ -328,6 +329,7 @@ Here is a breakdown of key optional variables you can adjust in your | 🍏 `PIPELINE_SKIP_DOWNLOADS` | `false` | If `true`, skips download. Use if you only want to organize your current files. | | 🍏 `PIPELINE_SKIP_DOWNLOADS_IMPORTS` | `false` | If `true`, skips updating posters when updating metadata. | | `PIPELINE_FORCE_REDOWNLOAD` | `false` | If `true`, downloads and imports files even if already present. | +| `PIPELINE_FORCE_REIMPORT_EXISTING` | `false` | If `true`, when an episode that already exists is downloaded again, it reimports it overwriting the file. | | `PIPELINE_SKIP_POSTERS` | `false` | If `true`, skips updating posters when updating metadata. | | --- | --- | --- | | `PIPELINE_INCLUDE_SPECIALS` | `false` | Set to `true` to also process specials. | diff --git a/sample.env b/sample.env index 07a617d..c72bbe9 100644 --- a/sample.env +++ b/sample.env @@ -25,6 +25,8 @@ PIPELINE_SKIP_DOWNLOADS=false PIPELINE_SKIP_DOWNLOADS_IMPORTS=false # If true, downloads and imports files even if already present. PIPELINE_FORCE_REDOWNLOAD=false +# If true, when an episode that already exists is downloaded again, it reimports it overwriting the file. +PIPELINE_FORCE_REIMPORT_EXISTING=false # If true, skips updating posters when updating metadata. PIPELINE_SKIP_POSTERS=false diff --git a/src/environment.ts b/src/environment.ts index 35950ce..542808b 100644 --- a/src/environment.ts +++ b/src/environment.ts @@ -48,6 +48,9 @@ export default { PIPELINE_FORCE_REDOWNLOAD: /true/i.test( process.env.PIPELINE_FORCE_REDOWNLOAD || 'false', ), + PIPELINE_FORCE_REIMPORT_EXISTING: /true/i.test( + process.env.PIPELINE_FORCE_REIMPORT_EXISTING || 'false', + ), PIPELINE_SKIP_POSTERS: /true/i.test( process.env.PIPELINE_SKIP_POSTERS || process.env.SKIP_POSTERS || 'false', ), diff --git a/src/index.ts b/src/index.ts index de33291..52df556 100644 --- a/src/index.ts +++ b/src/index.ts @@ -59,6 +59,8 @@ const startApp = async () => { PIPELINE_SKIP_DOWNLOADS_IMPORTS: environment.PIPELINE_SKIP_DOWNLOADS_IMPORTS, PIPELINE_FORCE_REDOWNLOAD: environment.PIPELINE_FORCE_REDOWNLOAD, + PIPELINE_FORCE_REIMPORT_EXISTING: + environment.PIPELINE_FORCE_REIMPORT_EXISTING, PIPELINE_SKIP_POSTERS: environment.PIPELINE_SKIP_POSTERS, PIPELINE_INCLUDE_SPECIALS: environment.PIPELINE_INCLUDE_SPECIALS, PIPELINE_PREFER_EXTENDED: environment.PIPELINE_PREFER_EXTENDED, diff --git a/src/pipeline/pipeline.model.ts b/src/pipeline/pipeline.model.ts index d856fbe..9488ec5 100644 --- a/src/pipeline/pipeline.model.ts +++ b/src/pipeline/pipeline.model.ts @@ -8,6 +8,7 @@ export type PipelineControllerConfig = { PIPELINE_SKIP_DOWNLOADS: boolean PIPELINE_SKIP_DOWNLOADS_IMPORTS: boolean PIPELINE_FORCE_REDOWNLOAD: boolean + PIPELINE_FORCE_REIMPORT_EXISTING: boolean PIPELINE_SKIP_POSTERS: boolean PIPELINE_INCLUDE_SPECIALS: boolean PIPELINE_PREFER_EXTENDED: boolean diff --git a/src/torrent/torrent.controller.test.ts b/src/torrent/torrent.controller.test.ts new file mode 100644 index 0000000..36d379d --- /dev/null +++ b/src/torrent/torrent.controller.test.ts @@ -0,0 +1,122 @@ +import { + linkSync, + mkdirSync, + mkdtempSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import environment from '../environment.js' +import type { LibraryController } from '../library/library.controller.js' +import type { TargetLibraryFile } from '../library/library.model.js' +import type { EpisodeMetadata } from '../metadata/metadata.model.js' +import type { MetadataController } from '../metadata/metadata.controller.js' +import type { PipelineController } from '../pipeline/pipeline.controller.js' +import { Context } from '../util/context.js' +import type { ITorrentController, Torrent } from './torrent.model.js' +import { TorrentController } from './torrent.controller.js' + +const CRC32 = 'ABCD1234' +const DESTINATION_NAME = 'One Pace - S01E01.mkv' +const SOURCE_NAME = `One Pace - S01E01 [${CRC32}].mkv` + +describe('Torrent import', () => { + let root: string + let downloads: string + let library: string + let source: string + let destination: string + + let controller: TorrentController + let updatemetadata: ReturnType + let updateTorrentCategory: ReturnType + + const torrent: Torrent = { + hash: 'torrent-hash', + content_path: '', + name: SOURCE_NAME, + } + + const episode = { + arc: 1, + episode: 1, + title: 'Romance Dawn', + } as EpisodeMetadata + + beforeEach(() => { + root = mkdtempSync(path.join(tmpdir(), 'onepacerr-import-')) + downloads = path.join(root, 'downloads') + library = path.join(root, 'library') + source = path.join(downloads, SOURCE_NAME) + destination = path.join(library, DESTINATION_NAME) + + mkdirSync(downloads, { recursive: true }) + mkdirSync(library, { recursive: true }) + + torrent.content_path = downloads + + environment.MOUNT_DOWNLOADS_TORRENT = '' + environment.MOUNT_DOWNLOADS_ONEPACERR = '' + environment.MOUNT_LIBRARY_MEDIA_SERVER = '' + environment.MOUNT_LIBRARY_ONEPACERR = '' + environment.LIBRARY_USE_HARDLINKS = true + environment.PIPELINE_SKIP_DOWNLOADS = false + environment.PIPELINE_SKIP_DOWNLOADS_IMPORTS = false + + updatemetadata = vi.fn().mockResolvedValue(undefined) + Context.pipeline = { + updatemetadata, + } as unknown as PipelineController + + Context.metadata = { + findEpisodeByCRC32: vi.fn().mockResolvedValue(episode), + findCRC32: vi.fn().mockResolvedValue(CRC32), + } as unknown as MetadataController + + Context.library = { + getTargetLibraryEpisodeFile: vi.fn().mockResolvedValue({ + path: `${library}${path.sep}`, + filename: DESTINATION_NAME, + } as TargetLibraryFile), + } as unknown as LibraryController + + updateTorrentCategory = vi.fn().mockResolvedValue(undefined) + const client = { + torrentClient: 'qbittorrent', + updateTorrentCategory, + } as unknown as ITorrentController + + controller = new TorrentController() + ;(controller as unknown as { client: ITorrentController }).client = client + }) + + afterEach(() => { + rmSync(root, { recursive: true, force: true }) + vi.clearAllMocks() + }) + + const importTorrent = () => + ( + controller as unknown as { + importTorrentFiles: (torrent: Torrent) => Promise + } + ).importTorrentFiles(torrent) + + it('Does not recreate an episode that is already imported', async () => { + writeFileSync(source, 'episode payload') + linkSync(source, destination) // the episode was imported on a previous run + const importedAt = statSync(destination).ctimeMs + + await importTorrent() + + expect(statSync(destination).ctimeMs).toBe(importedAt) + expect(updatemetadata).not.toHaveBeenCalled() + expect(updateTorrentCategory).toHaveBeenCalledWith( + torrent, + environment.TORRENT_CATEGORY_ONCE_COMPLETED, + ) + }) +}) diff --git a/src/torrent/torrent.controller.ts b/src/torrent/torrent.controller.ts index d7ace55..c3be0a7 100644 --- a/src/torrent/torrent.controller.ts +++ b/src/torrent/torrent.controller.ts @@ -1,5 +1,11 @@ import { Logger } from 'ez-ts-logger' -import { existsSync, mkdirSync, readdirSync, unlinkSync } from 'node:fs' +import { + existsSync, + mkdirSync, + readdirSync, + statSync, + unlinkSync, +} from 'node:fs' import path from 'node:path' import environment from '../environment.js' import { TargetLibraryFile } from '../library/library.model.js' @@ -27,6 +33,30 @@ import { TorrentConnectionError, } from './torrent.model.js' +/** + * Reports whether the source file is already present in the library. + * + * Importing is not idempotent: a re-run would delete and recreate the library + * file, touching it and making the media server rescan and regenerate + * trickplay/chapter images on every pass. When the destination is already the + * same hardlink, or an identical-size copy if hardlinks are disabled, the file + * can be treated as imported and left untouched. + */ +function isAlreadyImported(source: string, destination: string): boolean { + try { + const sourceStat = statSync(source) + const destinationStat = statSync(destination) + if (environment.LIBRARY_USE_HARDLINKS) + return ( + sourceStat.ino === destinationStat.ino && + sourceStat.dev === destinationStat.dev + ) + return sourceStat.size === destinationStat.size + } catch { + return false + } +} + export class TorrentController { private client: ITorrentController @@ -354,6 +384,18 @@ export class TorrentController { Logger.debug( `File for S${String(episode.arc).padStart(2, '0')}-${String(episode.episode).padStart(2, '0')} detected`, ) + + if ( + !environment.PIPELINE_FORCE_REIMPORT_EXISTING && + isAlreadyImported(source, destination) + ) { + Logger.debug( + `File for S${String(episode.arc).padStart(2, '0')}-${String(episode.episode).padStart(2, '0')} already imported, skipping re-import`, + ) + processed = true + continue + } + if (!environment.PIPELINE_SKIP_DOWNLOADS_IMPORTS) { if (previousLibraryFileName) { const toDelete = path.resolve(