Added a Setting to set the Model Download Folder

appinfo/info.xml

@@ -101,6 +101,7 @@ The app does not send any sensitive data to cloud providers or similar services.
 	<repair-steps>
 		<post-migration>
 			<step>OCA\Recognize\Migration\InstallDeps</step>
+			<step>OCA\Recognize\Migration\MoveDefaultModelFolder</step>
 		</post-migration>
 		<live-migration>
 			<step>OCA\Recognize\Migration\RemoveDuplicateFaceDetections</step>

lib/Classifiers/Audio/MusicnnClassifier.php

@@ -10,6 +10,7 @@
 use OCA\Recognize\Classifiers\Classifier;
 use OCA\Recognize\Service\Logger;
 use OCA\Recognize\Service\QueueService;
+use OCA\Recognize\Service\SettingsService;
 use OCA\Recognize\Service\TagManager;
 use OCP\AppFramework\Services\IAppConfig;
 use OCP\Files\IRootFolder;
@@ -24,8 +25,8 @@ final class MusicnnClassifier extends Classifier {
 
 	private TagManager $tagManager;
 
-	public function __construct(Logger $logger, IAppConfig $config, TagManager $tagManager, QueueService $queue, IRootFolder $rootFolder, ITempManager $tempManager, IPreview $previewProvider) {
-		parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider);
+	public function __construct(Logger $logger, IAppConfig $config, TagManager $tagManager, QueueService $queue, IRootFolder $rootFolder, ITempManager $tempManager, IPreview $previewProvider, SettingsService $settingsService) {
+		parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider, $settingsService);
 		$this->tagManager = $tagManager;
 	}
 

lib/Classifiers/Classifier.php

@@ -13,6 +13,7 @@
 use OCA\Recognize\Constants;
 use OCA\Recognize\Db\QueueFile;
 use OCA\Recognize\Service\QueueService;
+use OCA\Recognize\Service\SettingsService;
 use OCP\AppFramework\Services\IAppConfig;
 use OCP\DB\Exception;
 use OCP\Encryption\Exceptions\GenericEncryptionException;
@@ -41,14 +42,16 @@ abstract class Classifier {
 	private ITempManager $tempManager;
 	private IPreview $previewProvider;
 	private int $maxExecutionTime = self::MAX_EXECUTION_TIME;
+	private SettingsService $settingsService;
 
-	public function __construct(LoggerInterface $logger, IAppConfig $config, IRootFolder $rootFolder, QueueService $queue, ITempManager $tempManager, IPreview  $previewProvider) {
+	public function __construct(LoggerInterface $logger, IAppConfig $config, IRootFolder $rootFolder, QueueService $queue, ITempManager $tempManager, IPreview  $previewProvider, SettingsService $settingsService) {
 		$this->logger = $logger;
 		$this->config = $config;
 		$this->rootFolder = $rootFolder;
 		$this->queue = $queue;
 		$this->tempManager = $tempManager;
 		$this->previewProvider = $previewProvider;
+		$this->settingsService = $settingsService;
 	}
 
 	public function setMaxExecutionTime(int $time): void {
@@ -187,6 +190,7 @@ public function classifyFiles(string $model, array $queueFiles, int $timeout): \
 		if ($cores !== '0') {
 			$env['RECOGNIZE_CORES'] = $cores;
 		}
+		$env['MODEL_DIR'] = $this->settingsService->getSetting('models_target_path').'/models';
 		$proc->setEnv($env);
 		$proc->setTimeout(count($paths) * $timeout);
 		$proc->setInput(implode("\n", $paths));

lib/Classifiers/Images/ClusteringFaceClassifier.php

@@ -13,6 +13,7 @@
 use OCA\Recognize\Db\FaceDetectionMapper;
 use OCA\Recognize\Service\Logger;
 use OCA\Recognize\Service\QueueService;
+use OCA\Recognize\Service\SettingsService;
 use OCP\AppFramework\Services\IAppConfig;
 use OCP\BackgroundJob\IJobList;
 use OCP\DB\Exception;
@@ -43,8 +44,8 @@ public function __construct(
 		private FaceDetectionMapper $faceDetections,
 		QueueService $queue, IRootFolder $rootFolder,
 		private IUserMountCache $userMountCache, private IJobList $jobList, ITempManager $tempManager, IPreview $previewProvider,
-		private IManager $shareManager) {
-		parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider);
+		private IManager $shareManager, SettingsService $settingsService) {
+		parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider, $settingsService);
 	}
 
 	/**

lib/Classifiers/Images/ImagenetClassifier.php

@@ -10,6 +10,7 @@
 use OCA\Recognize\Classifiers\Classifier;
 use OCA\Recognize\Service\Logger;
 use OCA\Recognize\Service\QueueService;
+use OCA\Recognize\Service\SettingsService;
 use OCA\Recognize\Service\TagManager;
 use OCP\AppFramework\Services\IAppConfig;
 use OCP\Files\IRootFolder;
@@ -25,8 +26,8 @@ final class ImagenetClassifier extends Classifier {
 	private TagManager $tagManager;
 	protected QueueService $queue;
 
-	public function __construct(Logger $logger, IAppConfig $config, TagManager $tagManager, QueueService $queue, IRootFolder $rootFolder, ITempManager $tempManager, IPreview $previewProvider) {
-		parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider);
+	public function __construct(Logger $logger, IAppConfig $config, TagManager $tagManager, QueueService $queue, IRootFolder $rootFolder, ITempManager $tempManager, IPreview $previewProvider, SettingsService $settingsService) {
+		parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider, $settingsService);
 		$this->tagManager = $tagManager;
 		$this->queue = $queue;
 	}

lib/Classifiers/Images/LandmarksClassifier.php

@@ -11,6 +11,7 @@
 use OCA\Recognize\Db\QueueFile;
 use OCA\Recognize\Service\Logger;
 use OCA\Recognize\Service\QueueService;
+use OCA\Recognize\Service\SettingsService;
 use OCA\Recognize\Service\TagManager;
 use OCP\AppFramework\Services\IAppConfig;
 use OCP\Files\IRootFolder;
@@ -26,8 +27,8 @@ final class LandmarksClassifier extends Classifier {
 
 	private TagManager $tagManager;
 
-	public function __construct(Logger $logger, IAppConfig $config, TagManager $tagManager, QueueService $queue, IRootFolder $rootFolder, ITempManager $tempManager, IPreview $previewProvider) {
-		parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider);
+	public function __construct(Logger $logger, IAppConfig $config, TagManager $tagManager, QueueService $queue, IRootFolder $rootFolder, ITempManager $tempManager, IPreview $previewProvider, SettingsService $settingsService) {
+		parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider, $settingsService);
 		$this->tagManager = $tagManager;
 	}
 

lib/Classifiers/Video/MovinetClassifier.php

@@ -11,6 +11,7 @@
 use OCA\Recognize\Exception\Exception;
 use OCA\Recognize\Service\Logger;
 use OCA\Recognize\Service\QueueService;
+use OCA\Recognize\Service\SettingsService;
 use OCA\Recognize\Service\TagManager;
 use OCP\AppFramework\Services\IAppConfig;
 use OCP\Files\IRootFolder;
@@ -24,8 +25,8 @@ final class MovinetClassifier extends Classifier {
 
 	private TagManager $tagManager;
 
-	public function __construct(Logger $logger, IAppConfig $config, TagManager $tagManager, QueueService $queue, IRootFolder $rootFolder, ITempManager $tempManager, IPreview $previewProvider) {
-		parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider);
+	public function __construct(Logger $logger, IAppConfig $config, TagManager $tagManager, QueueService $queue, IRootFolder $rootFolder, ITempManager $tempManager, IPreview $previewProvider, SettingsService $settingsService) {
+		parent::__construct($logger, $config, $rootFolder, $queue, $tempManager, $previewProvider, $settingsService);
 		$this->tagManager = $tagManager;
 	}
 

lib/Migration/MoveDefaultModelFolder.php

@@ -0,0 +1,70 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2020, Joas Schilling <coding@schilljs.com>
+ * @copyright Copyright (c) 2021, Marcel Klehr <mklehr@gmx.net>
+ *
+ * @author Joas Schilling <coding@schilljs.com>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+namespace OCA\Recognize\Migration;
+
+use OCA\Recognize\Service\SettingsService;
+use OCP\Migration\IOutput;
+use OCP\Migration\IRepairStep;
+use Psr\Log\LoggerInterface;
+
+final class MoveDefaultModelFolder implements IRepairStep {
+
+	public function __construct(
+		private SettingsService $settingsService,
+		private LoggerInterface $logger,
+	) {
+	}
+
+	public function getName(): string {
+		return 'Try to move the default Model Folder';
+	}
+
+	public function run(IOutput $output): void {
+		$oldModelTargetPath = __DIR__ . '/../../models';
+		$oldModelArchivePath = __DIR__ . '/../../models.tar.gz';
+		$newPath = $this->settingsService->getSetting('models_target_path');
+		$newModelTargetPath = $newPath . '/models';
+		$newModelArchivePath = $newPath . '/models.tar.gz';
+
+		if (is_dir($oldModelTargetPath)) {
+			/** @var array<string> $filesToMove */
+			$filesToMove = scandir($oldModelTargetPath);
+			$filesToMove = array_filter($filesToMove, fn (string $value) => $value !== '.' && $value !== '..');
+			$filesToMove = array_map(fn ($value) => $oldModelTargetPath.'/'.$value, $filesToMove);
+			mkdir($newModelTargetPath, recursive: true);
+			foreach ($filesToMove as $file) {
+				// Moving files across Partitions will fail, so instead we copy and delete see https://www.php.net/manual/en/function.rename.php#113943
+				copy($file, $newModelTargetPath.'/'.basename($file));
+				unlink($file);
+
+			}
+		}
+
+		if (is_file($oldModelArchivePath)) {
+			rename($oldModelArchivePath, $newModelArchivePath);
+		}
+	}
+}

lib/Service/DownloadModelsService.php

@@ -12,29 +12,37 @@
 use OCP\Http\Client\IClientService;
 use RecursiveDirectoryIterator;
 use RecursiveIteratorIterator;
+use function Safe\mkdir;
 
 final class DownloadModelsService {
 	private IClientService $clientService;
 	private bool $isCLI;
+	private SettingsService $settingsService;
 
-	public function __construct(IClientService $clientService, bool $isCLI) {
+	public function __construct(IClientService $clientService, bool $isCLI, SettingsService $settingsService) {
 		$this->clientService = $clientService;
 		$this->isCLI = $isCLI;
+		$this->settingsService = $settingsService;
 	}
 
 	/**
 	 * @param callable(string): void|null $log
 	 * @return void
 	 * @throws \Exception
 	 */
-	public function download(?callable $log = null) : void {
+
+	public function download() : void {
 		$log ??= static function (string $message): void {
 		};
-		$targetPath = __DIR__ . '/../../models';
-		if (file_exists($targetPath)) {
+		$targetPath = $this->settingsService->getSetting('models_target_path');
+		if (!file_exists($targetPath)) {
+			mkdir($targetPath, recursive: true);
+		}
+		$modelPath = $targetPath . '/models';
+		if (file_exists($modelPath)) {
 			$log('Removing existing models directory at ' . $targetPath);
 			// remove models directory
-			$it = new RecursiveDirectoryIterator($targetPath, FilesystemIterator::SKIP_DOTS);
+			$it = new RecursiveDirectoryIterator($modelPath, FilesystemIterator::SKIP_DOTS);
 			$files = new RecursiveIteratorIterator($it,
 				RecursiveIteratorIterator::CHILD_FIRST);
 			foreach ($files as $file) {
@@ -44,11 +52,12 @@ public function download(?callable $log = null) : void {
 					unlink($file->getRealPath());
 				}
 			}
-			rmdir($targetPath);
+			rmdir($modelPath);
 		}
 
 		$archiveUrl = $this->getArchiveUrl($this->getNeededArchiveRef());
-		$archivePath = __DIR__ . '/../../models.tar.gz';
+
+		$archivePath = $targetPath . '/models.tar.gz';
 		$log('Downloading models archive from ' . $archiveUrl);
 		$log('Saving archive to ' . $archivePath);
 		$timeout = $this->isCLI ? 0 : 480;

lib/Service/SettingsService.php

@@ -61,6 +61,7 @@ final class SettingsService {
 		'nice_value' => '0',
 		'concurrency.enabled' => 'false',
 		'ffmpeg_binary' => '',
+		'models_target_path' => '../../models_cache',
 	];
 
 	/** @var array<string,string>  */
@@ -94,7 +95,13 @@ final class SettingsService {
 		'landmarks.batchSize',
 		'movinet.batchSize',
 		'musicnn.batchSize',
-		'concurrency.enabled'
+		'concurrency.enabled',
+		'models_target_path',
+		'models_archive_file',
+	];
+
+	private const PATH_SETTINGS = [
+		'models_target_path',
 	];
 
 	private IAppConfig $config;
@@ -121,6 +128,14 @@ public function getSetting(string $key): string {
 		if (in_array($key, self::LAZY_SETTINGS, true)) {
 			$lazy = true;
 		}
+
+		if (in_array($key, self::PATH_SETTINGS, true)) {
+			$path = $this->config->getAppValueString($key, self::DEFAULTS[$key], lazy: $lazy);
+			if (!$this->isPathAbsolute($path)) {
+				$path = __DIR__ .'/'. $path;
+			}
+			return $path;
+		}
 		return $this->config->getAppValueString($key, self::DEFAULTS[$key], lazy: $lazy);
 	}
 
@@ -182,6 +197,9 @@ public function setSetting(string $key, string $value): void {
 		if (in_array($key, self::LAZY_SETTINGS, true)) {
 			$lazy = true;
 		}
+		if (in_array($key, self::PATH_SETTINGS) && $value === '') {
+			$value = self::DEFAULTS[$key];
+		}
 		$this->config->setAppValueString($key, $value, lazy: $lazy);
 	}
 
@@ -195,4 +213,23 @@ public function getAll(): array {
 		}
 		return $settings;
 	}
+
+	private function isPathAbsolute(string $path): bool {
+		if ($path === '') {
+			return false;
+		}
+		if ($path[0] === '/') {
+			return true;
+		}
+
+		if (ctype_alpha($path[0]) && $path[1] === ':') {
+			return true;
+		}
+
+		if ($path[0] === '\\' && $path[1] === '\\') {
+			return true;
+		}
+
+		return false;
+	}
 }

lib/Settings/AdminSettings.php

@@ -28,9 +28,11 @@ public function getForm(): TemplateResponse {
 		$settings = $this->settingsService->getAll();
 		$this->initialState->provideInitialState('settings', $settings);
 
-		$modelsPath = __DIR__ . '/../../models';
-		$modelsDownloaded = file_exists($modelsPath);
+		$targetPath = $this->settingsService->getSetting('models_target_path');
+		$modelsDownloaded = file_exists($targetPath .'/models');
+		$modelsTargetPathWritable = is_writable($targetPath);
 		$this->initialState->provideInitialState('modelsDownloaded', $modelsDownloaded);
+		$this->initialState->provideInitialState('modelsTargetPathWritable', $modelsTargetPathWritable);
 
 		$tagsEnabled = $this->appManager->isEnabledForAnyone('systemtags');
 		$this->initialState->provideInitialState('tagsEnabled', $tagsEnabled);

src/classifier_imagenet.js

@@ -4,9 +4,11 @@ const YAML = require('yaml')
 const _ = require('lodash')
 const rules = YAML.parse(fsSync.readFileSync(path.join(__dirname, 'rules.yml')).toString('utf8'))
 const { IMAGENET_CLASSES } = require('./efficientnet/classes')
+const MODEL_DIR = process.env.MODEL_DIR
 
 let tf, getPort, StaticServer
 let PUREJS = false
+
 if (process.env.RECOGNIZE_PUREJS === 'true') {
 	tf = require('@tensorflow/tfjs')
 	require('@tensorflow/tfjs-backend-wasm')
@@ -58,7 +60,7 @@ if (process.argv.length < 3) throw new Error('Incorrect arguments: node classify
  * @param minInput
  */
 async function main(modelName, imgSize, minInput) {
-	const modelPath = path.resolve(__dirname, '..', 'models', modelName)
+	const modelPath = path.resolve(MODEL_DIR, modelName)
 
 	const modelFileName = 'model.json'
 	let modelUrl

src/classifier_landmarks.js

@@ -9,6 +9,7 @@ const LABELS = {
 	landmarks_south_america: require('./landmarks/south_america.json').name,
 	landmarks_oceania: require('./landmarks/oceania.json').name,
 }
+const MODEL_DIR = process.env.MODEL_DIR
 
 let tf, getPort, StaticServer
 let PUREJS = false
@@ -48,7 +49,7 @@ if (process.argv.length < 3) throw new Error('Incorrect arguments: node classify
  * @param paths
  */
 async function main(modelName, imgSize, minInput, paths) {
-	const modelPath = path.resolve(__dirname, '..', 'models', modelName)
+	const modelPath = path.resolve(MODEL_DIR, modelName)
 
 	const modelFileName = 'model.json'
 	let modelUrl