diff --git a/agent/androidapp/build.gradle.kts b/agent/androidapp/build.gradle.kts index 7f150f1b..ef2b5e6a 100644 --- a/agent/androidapp/build.gradle.kts +++ b/agent/androidapp/build.gradle.kts @@ -11,8 +11,6 @@ kotlin { androidTarget() sourceSets { - // M28/M31 platform-neutral logic lives in commonMain so that when androidTarget() - // is wired (needs AGP from dl.google.com), no file moves are required. commonMain.dependencies { implementation(project(":kernel")) implementation(project(":api")) @@ -31,10 +29,6 @@ kotlin { implementation("app.cash.sqldelight:runtime:2.0.2") implementation("app.cash.sqldelight:sqlite-driver:2.0.2") implementation("org.xerial:sqlite-jdbc:3.45.2.0") - // jvmMain has no Compose UI of its own, but the Compose Compiler Gradle plugin - // (kotlin.plugin.compose) runs its version check against every compilation in - // this module, androidTarget included — it fails hard without the runtime on - // the classpath even here. Unused at runtime, only satisfies that check. implementation("androidx.compose.runtime:runtime:1.6.0") } } @@ -50,14 +44,14 @@ kotlin { } } - // androidMain sourceSet — uncomment when androidTarget() is wired. val androidMain by getting { dependencies { implementation(project(":kernel")) implementation(project(":api")) + implementation(project(":storage")) implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.9.0") implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.6.1") - + // Compose and Material Design 3 implementation("androidx.compose.ui:ui:1.6.0") implementation("androidx.compose.material3:material3:1.1.0") @@ -65,19 +59,16 @@ kotlin { implementation("androidx.activity:activity-compose:1.8.0") implementation("androidx.lifecycle:lifecycle-viewmodel-compose:2.6.1") implementation("androidx.lifecycle:lifecycle-runtime-compose:2.6.1") - + // Navigation implementation("androidx.navigation:navigation-compose:2.7.0") - + // Android core implementation("androidx.core:core-ktx:1.10.1") implementation("androidx.appcompat:appcompat:1.6.1") implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.6.1") - // AidosService : LifecycleService (RFC-0050) — the foreground service hosting - // RuntimeServiceHost. implementation("androidx.lifecycle:lifecycle-service:2.6.1") - - // For scoped storage and file access + implementation("androidx.documentfile:documentfile:1.0.1") } } @@ -93,11 +84,10 @@ sqldelight { } } -// android { } — uncomment when androidTarget() is wired. android { namespace = "fi.italeino.aidos" compileSdk = 34 - + defaultConfig { applicationId = "fi.italeino.aidos" minSdk = 26 @@ -105,17 +95,12 @@ android { versionCode = 1 versionName = "0.1.0" } - + compileOptions { sourceCompatibility = JavaVersion.VERSION_21 targetCompatibility = JavaVersion.VERSION_21 } - // A release build type may only reference a signingConfig that is actually - // signing-ready (AGP's packageRelease requires storeFile once one is attached) - so the - // keystore's presence gates both populating the config below and attaching it to the - // release build type. Absent (e.g. no KEYSTORE_BASE64 secret in CI), the release build - // proceeds unsigned, matching the workflow's own documented fallback. val releaseKeystorePath = System.getenv("KEYSTORE_FILE") ?: System.getenv("HOME")?.let { "$it/.android/aidos-keystore.jks" } val hasReleaseKeystore = releaseKeystorePath != null && File(releaseKeystorePath).exists() diff --git a/agent/androidapp/src/androidMain/kotlin/fi/italeino/aidos/AndroidRuntimeClientFactory.kt b/agent/androidapp/src/androidMain/kotlin/fi/italeino/aidos/AndroidRuntimeClientFactory.kt new file mode 100644 index 00000000..63d9a70c --- /dev/null +++ b/agent/androidapp/src/androidMain/kotlin/fi/italeino/aidos/AndroidRuntimeClientFactory.kt @@ -0,0 +1,41 @@ +package fi.italeino.aidos + +import android.content.Context +import dev.aidos.api.RealRuntimeClient +import dev.aidos.storage.AndroidAidosStorage +import kotlinx.datetime.Clock +import java.io.File + +/** + * Process-scoped Android RuntimeClient composition root. + * + * The activity and foreground service deliberately share this instance: recreating a + * RealRuntimeClient in either component would recreate the in-memory session/event state and would + * make the persistence seams appear wired while the UI was still talking to a different client. + * Project locking stays unset until Android file-lock semantics have an instrumentation test. + */ +object AndroidRuntimeClientFactory { + @Volatile + private var instance: RealRuntimeClient? = null + + fun get(context: Context): RealRuntimeClient { + instance?.let { return it } + return synchronized(this) { + instance ?: create(context.applicationContext).also { instance = it } + } + } + + private fun create(context: Context): RealRuntimeClient { + val nowIso = { Clock.System.now().toString() } + val userDriver = AndroidAidosStorage.openUser(context, nowIso) + val projectsRoot = File(context.filesDir, "projects").apply { mkdirs() } + + return RealRuntimeClient().apply { + this.userDriver = userDriver + projectDbFactory = { projectRoot -> + AndroidAidosStorage.openProject(context, projectRoot, nowIso) + } + runtimeManagedProjectsRoot = projectsRoot.absolutePath + } + } +} diff --git a/agent/androidapp/src/androidMain/kotlin/fi/italeino/aidos/MainActivity.kt b/agent/androidapp/src/androidMain/kotlin/fi/italeino/aidos/MainActivity.kt index e921500a..d282d6cd 100644 --- a/agent/androidapp/src/androidMain/kotlin/fi/italeino/aidos/MainActivity.kt +++ b/agent/androidapp/src/androidMain/kotlin/fi/italeino/aidos/MainActivity.kt @@ -6,13 +6,11 @@ import androidx.activity.compose.setContent import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.material3.Surface import androidx.compose.ui.Modifier -import androidx.lifecycle.ViewModelProvider import androidx.navigation.compose.rememberNavController -import dev.aidos.api.RealRuntimeClient +import dev.aidos.androidapp.ui.diff.CommitPresenter import dev.aidos.androidapp.ui.projects.ProjectsPresenter import dev.aidos.androidapp.ui.runs.RunListPresenter import dev.aidos.androidapp.ui.sessions.SessionListPresenter -import dev.aidos.androidapp.ui.diff.CommitPresenter import fi.italeino.aidos.navigation.AidosNavHost import fi.italeino.aidos.theme.AidosTheme import kotlinx.coroutines.CoroutineScope @@ -21,21 +19,13 @@ import kotlinx.coroutines.SupervisorJob /** * Main activity for Aidos Android app (M27, M28, RFC-0050). * - * Hosts the Compose UI and wires the runtime presenters. - * The runtime itself runs in a foreground service (RuntimeServiceHost, M27). - * - * This activity only manages the UI — state, navigation, and presenter lifecycle. - * All business logic remains in presenters (commonMain), which are platform-neutral - * and testable on the JVM. + * Hosts the Compose UI and wires the runtime presenters. The runtime client is the same + * process-scoped, persistence-backed instance used by the foreground service. */ class MainActivity : ComponentActivity() { - + private val applicationScope = CoroutineScope(SupervisorJob()) - - // RFC-0050 MVP item 2 puts the runtime in-process in a foreground service; binding this - // activity to that service (RuntimeServiceHost) and injecting its client is the next step. - // Until that binding exists, each activity instance owns its own RealRuntimeClient — real - // command/event semantics, but state doesn't yet survive the activity being destroyed. + private lateinit var projectsPresenter: ProjectsPresenter private lateinit var sessionListPresenter: SessionListPresenter private lateinit var runListPresenter: RunListPresenter @@ -44,10 +34,8 @@ class MainActivity : ComponentActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - // TODO(RFC-0050 MVP item 2): replace with the RuntimeClient bound from the foreground - // service (RuntimeServiceHost) once the Service subclass exists. - val runtimeClient = RealRuntimeClient() - + val runtimeClient = AndroidRuntimeClientFactory.get(this) + projectsPresenter = ProjectsPresenter(runtimeClient, applicationScope) sessionListPresenter = SessionListPresenter(runtimeClient, applicationScope) runListPresenter = RunListPresenter(runtimeClient, applicationScope) @@ -70,7 +58,6 @@ class MainActivity : ComponentActivity() { } } - // Load projects on app start projectsPresenter.loadProjects() } } diff --git a/agent/androidapp/src/androidMain/kotlin/fi/italeino/aidos/service/AidosService.kt b/agent/androidapp/src/androidMain/kotlin/fi/italeino/aidos/service/AidosService.kt index 74bab2d3..1f9f73e9 100644 --- a/agent/androidapp/src/androidMain/kotlin/fi/italeino/aidos/service/AidosService.kt +++ b/agent/androidapp/src/androidMain/kotlin/fi/italeino/aidos/service/AidosService.kt @@ -9,7 +9,7 @@ import androidx.core.app.NotificationCompat import androidx.lifecycle.LifecycleService import dev.aidos.androidapp.notification.NotificationManager as AidosNotificationManager import dev.aidos.androidapp.service.RuntimeServiceHost -import dev.aidos.api.RealRuntimeClient +import fi.italeino.aidos.AndroidRuntimeClientFactory import fi.italeino.aidos.MainActivity import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -19,36 +19,19 @@ import kotlinx.coroutines.flow.collectLatest import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking -/** - * Android foreground service hosting the runtime (RFC-0050's `AidosService : LifecycleService`, - * M27, D24). - * - * Wires the platform-neutral [RuntimeServiceHost] into the Android service lifecycle: - * [onStartCommand] starts hosting a run and calls `startForeground` with the ongoing - * notification (RFC-0050 "Notifications" item 1 — "what is running, and Cancel"); [onDestroy] - * shuts the host down. Notification *content* decisions (what text, whether a given - * notification should fire at all) stay in the platform-neutral - * `dev.aidos.androidapp.notification.NotificationManager` — this class only performs the actual - * Android `NotificationManager`/`NotificationChannel` calls RFC-0050 says belong to the service - * wrapper. - * - * Owns its own [RealRuntimeClient] for now, separate from [MainActivity]'s — binding the two so - * they share one client (and its state survives activity recreation) is the next step, tracked - * in PIPELINE.md's Group 2 checklist. - * - * The Cancel action and the wake lock RFC-0050's D24(a) calls for are not wired yet — this link - * covers the service lifecycle and the ongoing notification itself; both are flagged as - * follow-up in PIPELINE.md rather than guessed at here. - */ +/** Android foreground service hosting the runtime (RFC-0050's AidosService, M27, D24). */ class AidosService : LifecycleService() { private val serviceScope = CoroutineScope(SupervisorJob() + Dispatchers.Default) - private val runtimeClient = RealRuntimeClient() - private val runtimeServiceHost = RuntimeServiceHost(client = runtimeClient, scope = serviceScope) + private lateinit var runtimeServiceHost: RuntimeServiceHost private val notificationDecisions = AidosNotificationManager() override fun onCreate() { super.onCreate() + runtimeServiceHost = RuntimeServiceHost( + client = AndroidRuntimeClientFactory.get(this), + scope = serviceScope, + ) createNotificationChannel() serviceScope.launch { @@ -67,26 +50,19 @@ class AidosService : LifecycleService() { runtimeServiceHost.startRun(runId, runDescription) } - // A Run reaching a model call needs the foreground window regardless of what started - // this call (D24(a)) — startForeground on every onStartCommand keeps that window open. startForeground(NOTIFICATION_ID, buildNotification()) - return START_NOT_STICKY } override fun onDestroy() { - // RuntimeServiceHost.shutdown() cancels the active run's job and joins it so the - // recovery checkpoint (RFC-0009) is the one actually written, not whatever was in - // flight — serviceScope is cancelled separately below rather than racing this suspend - // call against Service teardown. runBlocking { runtimeServiceHost.shutdown() } serviceScope.cancel() super.onDestroy() } private fun updateNotification() { - val manager = getSystemService(AndroidNotificationManager::class.java) - manager.notify(NOTIFICATION_ID, buildNotification()) + getSystemService(AndroidNotificationManager::class.java) + .notify(NOTIFICATION_ID, buildNotification()) } private fun buildNotification(): Notification { @@ -100,8 +76,6 @@ class AidosService : LifecycleService() { return NotificationCompat.Builder(this, CHANNEL_ID) .setContentTitle(content.title) .setContentText(content.body) - // TODO: real app icon (no drawable resources exist in this module yet — tracked - // separately from the service wiring, not blocking it). .setSmallIcon(android.R.drawable.ic_dialog_info) .setContentIntent(openAppIntent) .setOngoing(true) @@ -116,7 +90,8 @@ class AidosService : LifecycleService() { ).apply { description = "Shows what Aidos is doing while a run is in progress" } - getSystemService(AndroidNotificationManager::class.java).createNotificationChannel(channel) + getSystemService(AndroidNotificationManager::class.java) + .createNotificationChannel(channel) } companion object { diff --git a/agent/storage/build.gradle.kts b/agent/storage/build.gradle.kts index 72cbbcd5..5a28c84b 100644 --- a/agent/storage/build.gradle.kts +++ b/agent/storage/build.gradle.kts @@ -12,9 +12,6 @@ kotlin { implementation(project(":kernel")) implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.9.0") implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.6.1") - // D35: the driver only, not SQLDelight's .sq schema codegen. schema/ stays the one - // canonical DDL (RFC-0040); this is the KMP SqlDriver interface gitsema-kotlin also - // uses, so both libraries share one SQLite build per process on Android. implementation("app.cash.sqldelight:runtime:2.0.2") } @@ -23,16 +20,10 @@ kotlin { } val jvmMain by getting { - // schema/ is read directly from its one canonical location (RFC-0040) rather than - // copied into this module, so there is exactly one file a change to the DDL touches. resources.srcDir(rootProject.projectDir.resolve("../schema")) resources.include("*.sql") dependencies { implementation("app.cash.sqldelight:sqlite-driver:2.0.2") - // sqlite-driver depends on this at runtime only; SQLiteConfig (used to bake WAL / - // synchronous / foreign_keys / busy_timeout into every connection the driver - // opens, since PRAGMA after the fact only reaches one connection) needs it at - // compile time too. implementation("org.xerial:sqlite-jdbc:3.45.2.0") } } @@ -42,6 +33,12 @@ kotlin { implementation(kotlin("test-junit")) } } + + val androidMain by getting { + dependencies { + implementation("app.cash.sqldelight:android-driver:2.0.2") + } + } } compilerOptions { @@ -57,6 +54,10 @@ android { minSdk = 26 } + // The canonical DDL remains in the repository-level schema/ directory. Package those files + // as Android assets so AndroidStorage can feed the same MigrationRunner as the JVM target. + sourceSets["main"].assets.srcDir(rootProject.projectDir.resolve("../schema")) + compileOptions { sourceCompatibility = JavaVersion.VERSION_21 targetCompatibility = JavaVersion.VERSION_21 diff --git a/agent/storage/src/androidMain/kotlin/dev/aidos/storage/AndroidStorage.kt b/agent/storage/src/androidMain/kotlin/dev/aidos/storage/AndroidStorage.kt new file mode 100644 index 00000000..07730aae --- /dev/null +++ b/agent/storage/src/androidMain/kotlin/dev/aidos/storage/AndroidStorage.kt @@ -0,0 +1,55 @@ +package dev.aidos.storage + +import android.content.Context +import app.cash.sqldelight.db.AfterVersion +import app.cash.sqldelight.db.QueryResult +import app.cash.sqldelight.db.SqlDriver +import app.cash.sqldelight.db.SqlSchema +import app.cash.sqldelight.driver.android.AndroidSqliteDriver +import java.io.File + +/** + * Android counterpart to [AidosStorage]. + * + * The canonical SQL lives in the repository-level `schema/` directory and is packaged as Android + * assets. SQLDelight owns the SQLite connection; [MigrationRunner] remains the single bootstrap + * and migration state machine for the actual schema. + */ +object AndroidAidosStorage { + private const val RUNTIME_VERSION = "0.1.0-alpha" + + fun openUser(context: Context, nowIso: () -> String): SqlDriver = + open(context, DatabaseKind.USER, File(context.filesDir, "user.db"), nowIso) + + fun openProject(context: Context, projectRoot: String, nowIso: () -> String): SqlDriver = + open(context, DatabaseKind.PROJECT, File(projectRoot, ".aidos/state.db"), nowIso) + + private fun open( + context: Context, + kind: DatabaseKind, + path: File, + nowIso: () -> String, + ): SqlDriver { + path.parentFile?.mkdirs() + val driver = AndroidSqliteDriver( + schema = RawSchema(kind.currentVersion), + context = context, + name = path.absolutePath, + ) + val schemaSql = context.assets.open(kind.schemaResource).bufferedReader().use { it.readText() } + MigrationRunner.open(driver, kind, schemaSql, RUNTIME_VERSION, nowIso) + return driver + } + + /** SQLDelight lifecycle hook is intentionally empty; MigrationRunner owns the real DDL. */ + private class RawSchema(override val version: Long) : SqlSchema> { + override fun create(driver: SqlDriver): QueryResult.Value = QueryResult.Value(Unit) + + override fun migrate( + driver: SqlDriver, + oldVersion: Long, + newVersion: Long, + vararg callbacks: AfterVersion, + ): QueryResult.Value = QueryResult.Value(Unit) + } +} diff --git a/engine/androidapp/src/androidMain/kotlin/fi/italeino/aidos/engine/EngineService.kt b/engine/androidapp/src/androidMain/kotlin/fi/italeino/aidos/engine/EngineService.kt index 7976da42..4832d6e0 100644 --- a/engine/androidapp/src/androidMain/kotlin/fi/italeino/aidos/engine/EngineService.kt +++ b/engine/androidapp/src/androidMain/kotlin/fi/italeino/aidos/engine/EngineService.kt @@ -137,9 +137,19 @@ class EngineService : LifecycleService() { // Android uses the native llama.cpp binding directly. The JVM-only backend in // :modelruntime remains the desktop implementation; both share GlobalModelRuntime. - val runtime = GlobalModelRuntime(AndroidLlamaCppInferenceBackend(this@EngineService)) + // The persistent catalog is authoritative for model identity, kind and expected + // digest; filesDir/models is only the artifact store. + val runtime = GlobalModelRuntime( + AndroidLlamaCppInferenceBackend( + context = this@EngineService, + catalogManager = catalog, + ) + ) modelRuntime = runtime + // Reconcile stale installed rows before the service becomes available to clients. + runtime.catalog() + httpServer = EngineHttpServer(tokenManager, runtime) httpServer.start() val boundPort = httpServer.getBoundPort() @@ -224,6 +234,35 @@ class EngineService : LifecycleService() { return HttpModelClient(port = port, token = token) } + /** + * Force-load a model into memory without performing inference. The model remains loaded until + * explicitly closed, deleted, or the engine shuts down. + */ + suspend fun openModel(modelId: String): Result { + if (!isRunning) return Result.failure(IllegalStateException("Engine is not running")) + val runtime = modelRuntime ?: return Result.failure(IllegalStateException("Model runtime is not initialized")) + return runtime.load(modelId).map { Unit } + } + + /** + * Interrupt active inference, cancel queued requests for the model, wait for all admitted + * work to stop, then unload the model. The installed model artifact remains available for + * the next inference request. + */ + suspend fun closeModel(modelId: String): Result { + if (!isRunning) return Result.failure(IllegalStateException("Engine is not running")) + return httpServer.closeModel(modelId) + } + + /** + * Closes (interrupts + unloads) a model through the Engine admission gate before deleting its + * installed artifact. + */ + suspend fun deleteModel(modelId: String): Result { + if (!isRunning) return Result.failure(IllegalStateException("Engine is not running")) + return httpServer.deleteModel(modelId) + } + private fun createNotificationChannel() { val channel = NotificationChannel( NOTIFICATION_CHANNEL_ID, diff --git a/engine/androidapp/src/androidMain/kotlin/fi/italeino/aidos/engine/inference/AndroidLlamaCppAdapter.kt b/engine/androidapp/src/androidMain/kotlin/fi/italeino/aidos/engine/inference/AndroidLlamaCppAdapter.kt index deb69425..e5b95eb3 100644 --- a/engine/androidapp/src/androidMain/kotlin/fi/italeino/aidos/engine/inference/AndroidLlamaCppAdapter.kt +++ b/engine/androidapp/src/androidMain/kotlin/fi/italeino/aidos/engine/inference/AndroidLlamaCppAdapter.kt @@ -1,10 +1,12 @@ package fi.italeino.aidos.engine.inference import de.kherud.llama.InferenceParameters +import de.kherud.llama.LlamaIterator import de.kherud.llama.LlamaModel import de.kherud.llama.ModelParameters +import dev.aidos.kernel.CancellableModelAdapter import dev.aidos.kernel.ContentBlock -import dev.aidos.kernel.ModelAdapter +import dev.aidos.kernel.EmbeddingModelAdapter import dev.aidos.kernel.ModelRef import dev.aidos.kernel.ModelRequest import dev.aidos.kernel.ModelResponse @@ -24,16 +26,19 @@ class AndroidLlamaCppAdapter( private val modelFile: File, override val contextWindow: Int, private val threads: Int = 4, -) : ModelAdapter { + private val embeddingMode: Boolean = false, +) : CancellableModelAdapter, EmbeddingModelAdapter { override val providerId: String = "llama.cpp.android" override val modelVersion: String = "java-llama.cpp-4.2.0" override val isLocal: Boolean = true private val model: LlamaModel + private val inferenceLock = Any() + @Volatile private var activeIterator: LlamaIterator? = null @Volatile private var closed = false init { - val parameters = ModelParameters() + var parameters = ModelParameters() .setModel(modelFile.absolutePath) .setCtxSize(contextWindow) .setThreads(threads) @@ -41,11 +46,24 @@ class AndroidLlamaCppAdapter( .setBatchSize(512) .setUbatchSize(512) .setGpuLayers(0) + if (embeddingMode) parameters = parameters.enableEmbedding() model = LlamaModel(parameters) } override fun supportsNativeToolCalls(): Boolean = false + override suspend fun embed(text: String): Result = try { + if (closed) return Result.failure(IllegalStateException("Model $modelId is unloaded")) + if (!embeddingMode) return Result.failure( + UnsupportedOperationException("Model $modelId is not configured for embeddings") + ) + val vector = model.embed(text) + if (vector.isEmpty()) Result.failure(IllegalStateException("Embedding model returned an empty vector")) + else Result.success(vector) + } catch (e: Throwable) { + Result.failure(e) + } + override suspend fun invoke(request: ModelRequest): Result { var response: ModelResponse? = null return try { @@ -69,6 +87,10 @@ class AndroidLlamaCppAdapter( emit(ModelStreamEvent.Failed(IllegalStateException("Model $modelId is unloaded"))) return@flow } + if (embeddingMode) { + emit(ModelStreamEvent.Failed(UnsupportedOperationException("Embedding model cannot perform chat generation"))) + return@flow + } if (request.tools.isNotEmpty()) { emit(ModelStreamEvent.Failed(UnsupportedOperationException("Android llama.cpp tool calling is not enabled yet"))) return@flow @@ -84,11 +106,20 @@ class AndroidLlamaCppAdapter( val output = StringBuilder() var tokenCount = 0 - for (token in model.generate(parameters)) { - if (tokenCount++ >= request.maxOutputTokens) break - output.append(token.text) - emit(ModelStreamEvent.Delta(token.text)) - if (request.stopConditions.any(output::contains)) break + val iterator = model.generate(parameters).iterator() + synchronized(inferenceLock) { activeIterator = iterator } + try { + while (iterator.hasNext()) { + if (tokenCount++ >= request.maxOutputTokens) break + val token = iterator.next() + output.append(token.text) + emit(ModelStreamEvent.Delta(token.text)) + if (request.stopConditions.any(output::contains)) break + } + } finally { + synchronized(inferenceLock) { + if (activeIterator === iterator) activeIterator = null + } } val text = output.toString() @@ -111,9 +142,16 @@ class AndroidLlamaCppAdapter( } } + override fun cancelCurrentInference() { + synchronized(inferenceLock) { + activeIterator?.cancel() + } + } + fun close() { if (closed) return closed = true + cancelCurrentInference() model.close() } diff --git a/engine/androidapp/src/androidMain/kotlin/fi/italeino/aidos/engine/inference/AndroidLlamaCppInferenceBackend.kt b/engine/androidapp/src/androidMain/kotlin/fi/italeino/aidos/engine/inference/AndroidLlamaCppInferenceBackend.kt index fbaf64c7..96fe25fa 100644 --- a/engine/androidapp/src/androidMain/kotlin/fi/italeino/aidos/engine/inference/AndroidLlamaCppInferenceBackend.kt +++ b/engine/androidapp/src/androidMain/kotlin/fi/italeino/aidos/engine/inference/AndroidLlamaCppInferenceBackend.kt @@ -5,64 +5,108 @@ import dev.aidos.kernel.ModelAdapter import dev.aidos.kernel.ModelDescriptor import dev.aidos.kernel.ModelKind import dev.aidos.modelruntime.InferenceBackend +import dev.aidos.models.CatalogEntry +import dev.aidos.models.ModelCatalogManager +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive import java.io.File import java.security.MessageDigest /** - * Android inference backend. Models live in the engine-private `files/models` directory, - * the same location used by the download/install workflow. - * - * The installer verifies the publisher digest before installation. The backend re-hashes - * the installed artifact when the runtime admits it. + * Android inference backend. Model identity and expected digests come from the persistent model + * catalog; installed metadata comes from the same catalog database. The filesystem is treated as + * an artifact store, not as an authoritative model catalog. */ class AndroidLlamaCppInferenceBackend( context: Context, + private val catalogManager: ModelCatalogManager, private val threads: Int = 4, ) : InferenceBackend { private val modelsDir = File(context.filesDir, "models").apply { mkdirs() } private val liveAdapters = mutableMapOf() - override suspend fun catalog(): List = installed() + override suspend fun catalog(): List { + reconcileInstalledState().getOrThrow() + return catalogManager.listCatalog().getOrElse { throw it }.map { entry -> descriptorFromCatalog(entry) } + } override suspend fun installed(): List = - modelsDir.listFiles() - ?.asSequence() - ?.filter { it.isFile && it.extension.equals("gguf", ignoreCase = true) } - ?.map { file -> - ModelDescriptor( - id = file.nameWithoutExtension, - name = file.nameWithoutExtension, - kind = ModelKind.LLM, - providerId = "llama.cpp.android", - isLocal = true, - contextWindow = DEFAULT_CONTEXT, - sizeBytes = file.length(), - digest = sha256(file), - ) + catalogManager.listInstalled().getOrElse { throw it } + .filter { File(it.path).isFile } + .mapNotNull { installed -> + val catalog = catalogManager.getCatalog(installed.modelId).getOrElse { throw it } + catalog?.let { descriptorFromCatalog(it, installed.sizeBytes, installed.digest) } } - ?.toList() - ?: emptyList() - override suspend fun computeDigest(modelId: String): String = - sha256(resolveModelFile(modelId)) + /** + * Reconciles persistent installed state with the artifact store and catalog authority. + * Orphan files are intentionally ignored: they have no trusted model identity and cannot + * become executable merely by existing in filesDir/models. + */ + suspend fun reconcileInstalledState(): Result = runCatching { + val catalogEntries = catalogManager.listCatalog().getOrThrow().associateBy { it.id } + val installedEntries = catalogManager.listInstalled().getOrThrow() + for (installed in installedEntries) { + val catalog = catalogEntries[installed.modelId] + val file = File(installed.path) + val expectedDigest = catalog?.let(::expectedDigest) + val valid = catalog != null && + file.isFile && + installed.digest.isNotBlank() && + (expectedDigest == null || expectedDigest.equals(installed.digest, ignoreCase = true)) + if (!valid) { + liveAdapters.remove(installed.modelId)?.close() + catalogManager.uninstall(installed.modelId).getOrThrow() + } + } + } + + override suspend fun computeDigest(modelId: String): String = sha256(resolveModelFile(modelId)) override suspend fun delete(modelId: String) { liveAdapters.remove(modelId)?.close() - resolveModelFile(modelId).delete() + val file = resolveModelFile(modelId) + if (file.isFile) file.delete() + catalogManager.uninstall(modelId).getOrThrow() } override suspend fun load(modelId: String): Result { - val file = resolveModelFile(modelId) + val catalog = catalogManager.getCatalog(modelId).getOrElse { return Result.failure(it) } + ?: return Result.failure(IllegalStateException("Model $modelId is not in the model catalog")) + val installed = catalogManager.listInstalled().getOrElse { return Result.failure(it) } + .firstOrNull { it.modelId == modelId } + ?: return Result.failure(IllegalStateException("Model $modelId is not installed")) + val file = File(installed.path) if (!file.isFile) return Result.failure( - IllegalStateException("Model file not found for '$modelId' in ${modelsDir.absolutePath}") + IllegalStateException("MODEL_NOT_INSTALLED: model file not found for '$modelId': ${file.absolutePath}") ) + return try { + if (installed.digest.isBlank()) { + return Result.failure(IllegalStateException("MODEL_INTEGRITY_MISSING: no installed digest for '$modelId'")) + } + val expectedDigest = expectedDigest(catalog) + if (expectedDigest != null && !expectedDigest.equals(installed.digest, ignoreCase = true)) { + return Result.failure( + IllegalStateException("MODEL_INTEGRITY_MISMATCH: installed metadata for '$modelId' does not match catalog") + ) + } + val actualDigest = sha256(file) + if (!actualDigest.equals(installed.digest, ignoreCase = true)) { + return Result.failure( + IllegalStateException("MODEL_INTEGRITY_MISMATCH: installed digest for '$modelId' does not match the model file") + ) + } + val adapter = AndroidLlamaCppAdapter( modelId = modelId, modelFile = file, - contextWindow = DEFAULT_CONTEXT, + contextWindow = contextWindow(catalog), threads = threads, + embeddingMode = catalog.kind == ModelKind.EMBEDDING, ) + liveAdapters[modelId]?.close() liveAdapters[modelId] = adapter Result.success(adapter) } catch (e: Throwable) { @@ -74,14 +118,50 @@ class AndroidLlamaCppInferenceBackend( liveAdapters.remove(modelId)?.close() } - /** Supports both exact ids and the `_.gguf` installer naming scheme. */ - private fun resolveModelFile(modelId: String): File { - val exact = File(modelsDir, "$modelId.gguf") - if (exact.isFile) return exact - val safeId = modelId.replace(Regex("[^A-Za-z0-9._-]"), "_") - return modelsDir.listFiles() - ?.firstOrNull { it.isFile && it.extension.equals("gguf", true) && it.nameWithoutExtension.startsWith("${safeId}_") } - ?: exact + private fun descriptorFromCatalog( + entry: CatalogEntry, + sizeBytes: Long? = null, + installedDigest: String? = null, + ): ModelDescriptor { + val metadata = runCatching { Json.parseToJsonElement(entry.propertiesJson).jsonObject }.getOrNull() + val expectedDigest = metadata?.get("sha256")?.jsonPrimitive?.content + val format = metadata?.get("format")?.jsonPrimitive?.content + val quantization = metadata?.get("quantization")?.jsonPrimitive?.content + val extraMetadata = metadata?.entries + ?.filter { it.value is kotlinx.serialization.json.JsonPrimitive } + ?.associate { it.key to it.value.jsonPrimitive.content } + ?: emptyMap() + return ModelDescriptor( + id = entry.id, + name = entry.name, + kind = entry.kind, + providerId = entry.provider, + isLocal = true, + contextWindow = contextWindow(entry), + sizeBytes = sizeBytes, + digest = installedDigest ?: expectedDigest, + format = format, + quantization = quantization, + metadata = extraMetadata, + ) + } + + private fun expectedDigest(entry: CatalogEntry): String? = + runCatching { + Json.parseToJsonElement(entry.propertiesJson).jsonObject["sha256"]?.jsonPrimitive?.content + }.getOrNull()?.takeIf { it.isNotBlank() } + + private fun contextWindow(entry: CatalogEntry): Int = + runCatching { + Json.parseToJsonElement(entry.propertiesJson).jsonObject["context_window"]?.jsonPrimitive?.content?.toIntOrNull() + ?: DEFAULT_CONTEXT + }.getOrDefault(DEFAULT_CONTEXT) + + /** Uses the persistent installed path; filename scanning is only a legacy fallback. */ + private suspend fun resolveModelFile(modelId: String): File { + val installed = catalogManager.listInstalled().getOrThrow().firstOrNull { it.modelId == modelId } + if (installed != null) return File(installed.path) + return File(modelsDir, "$modelId.gguf") } private fun sha256(file: File): String { diff --git a/engine/androidapp/src/androidMain/kotlin/fi/italeino/aidos/engine/ui/ModelsViewModel.kt b/engine/androidapp/src/androidMain/kotlin/fi/italeino/aidos/engine/ui/ModelsViewModel.kt index 010f429e..bd0102ce 100644 --- a/engine/androidapp/src/androidMain/kotlin/fi/italeino/aidos/engine/ui/ModelsViewModel.kt +++ b/engine/androidapp/src/androidMain/kotlin/fi/italeino/aidos/engine/ui/ModelsViewModel.kt @@ -110,13 +110,13 @@ class ModelsViewModel : ViewModel() { } fun deleteModel(modelId: String) { - val runtime = EngineService.instance?.modelRuntime ?: return + val service = EngineService.instance ?: return viewModelScope.launch { try { - runtime.delete(modelId) - refresh() // Refresh list after deletion + service.deleteModel(modelId).getOrThrow() + refresh() } catch (e: Exception) { - // Handle error + _errorMessage.value = "Delete failed: ${e.message ?: "Model may be busy"}" } } } diff --git a/engine/androidapp/src/jvmAndAndroidMain/kotlin/fi/italeino/aidos/engine/http/EngineHttpServer.kt b/engine/androidapp/src/jvmAndAndroidMain/kotlin/fi/italeino/aidos/engine/http/EngineHttpServer.kt index e750210e..f90fa98f 100644 --- a/engine/androidapp/src/jvmAndAndroidMain/kotlin/fi/italeino/aidos/engine/http/EngineHttpServer.kt +++ b/engine/androidapp/src/jvmAndAndroidMain/kotlin/fi/italeino/aidos/engine/http/EngineHttpServer.kt @@ -3,6 +3,7 @@ package fi.italeino.aidos.engine.http import dev.aidos.kernel.* import dev.aidos.kernel.ToolCall as KernelToolCall import fi.italeino.aidos.engine.inference.EngineBusyException +import fi.italeino.aidos.engine.inference.EngineModelBusyException import fi.italeino.aidos.engine.inference.EngineShuttingDownException import fi.italeino.aidos.engine.inference.InferenceRequestManager import io.ktor.http.* @@ -22,9 +23,6 @@ import kotlinx.serialization.json.JsonObject import java.util.* import kotlin.time.Duration.Companion.milliseconds -/** - * Ktor HTTP server for Aidos Engine local inference (RFC-0103). - */ class EngineHttpServer( private val tokenManager: TokenManager, private val modelRuntime: ModelRuntime, @@ -52,12 +50,14 @@ class EngineHttpServer( suspend fun waitUntilModelIdle(modelId: String, timeoutMs: Long = 5_000L): Boolean = inferenceManager.waitUntilModelIdle(modelId = modelId, timeout = timeoutMs.milliseconds) - /** - * Installs content negotiation, bearer auth, and routing onto [application] — factored out - * of [start] so tests can mount the real handlers via Ktor's `testApplication` instead of - * duplicating routing/auth setup per test (as the pre-S4 test suite did, which meant it never - * actually exercised this class's own code). - */ + /** Interrupt active inference, wait for it to stop, then unload the model while keeping it installed. */ + suspend fun closeModel(modelId: String): Result = + inferenceManager.closeModel(modelId) + + /** Close (interrupt + unload) the model before removing its installed artifact. */ + suspend fun deleteModel(modelId: String): Result = + inferenceManager.deleteModel(modelId) + internal fun installInto(application: Application) { with(application) { setupContentNegotiation() @@ -211,13 +211,6 @@ class EngineHttpServer( } } - /** - * Streams real per-token deltas from [ModelAdapter.invokeStreaming] as SSE frames (RFC-0021 - * "Streaming"; Dictator plan S4) — unlike the previous implementation, which called the - * non-streaming [ModelAdapter.invoke], waited for the complete response, and only then chopped - * it into fake chunks. Time-to-first-token now reflects real generation, not full-response - * latency. - */ private suspend fun streamChatCompletions( call: ApplicationCall, adapter: ModelAdapter, @@ -253,11 +246,6 @@ class EngineHttpServer( writeStringUtf8("data: ${Json.encodeToString(finalChunk)}\n\n") } is ModelStreamEvent.Failed -> { - // Headers and a 200 status are already committed by the time streaming can - // fail mid-generation, so there is no clean HTTP error to fall back to — - // send the error as its own SSE frame instead of silently truncating the - // stream, then still terminate with [DONE] below so a client's SSE parser - // doesn't hang waiting for a terminator that never comes. writeStringUtf8( "data: ${Json.encodeToString(ErrorResponse(ErrorDetail(event.error.message ?: "Inference failed", "inference_error")))}\n\n" ) @@ -296,12 +284,13 @@ class EngineHttpServer( kind = model.kind.name.lowercase(), capabilities = capabilitiesFor(model.kind), context_window = model.contextWindow, - format = if (model.isLocal) "gguf" else null, + format = model.format, size_bytes = installedModel?.sizeBytes ?: catalogModel?.sizeBytes, - quantization = deriveQuantization(model.id), + quantization = model.quantization, installed = installedModel != null, loaded = loadedIds.contains(model.id), metadata = buildMap { + putAll(model.metadata) model.digest?.let { put("digest", it) } put("is_local", model.isLocal.toString()) put("provider_id", model.providerId) @@ -329,43 +318,58 @@ class EngineHttpServer( ModelKind.TRANSLATION -> listOf("translation") } - private fun deriveQuantization(modelId: String): String? { - val lowered = modelId.lowercase() - return when { - lowered.contains("q2_k") -> "q2_k" - lowered.contains("q3_k") -> "q3_k" - lowered.contains("q4_k_m") -> "q4_k_m" - lowered.contains("q4_k_s") -> "q4_k_s" - lowered.contains("q4_0") -> "q4_0" - lowered.contains("q5_k_m") -> "q5_k_m" - lowered.contains("q5_k_s") -> "q5_k_s" - lowered.contains("q6_k") -> "q6_k" - lowered.contains("q8_0") -> "q8_0" - else -> null - } - } - private suspend fun handleEmbeddings(call: ApplicationCall) { try { val request = call.receive() + if (request.input.isEmpty()) { + call.respond( + HttpStatusCode.BadRequest, + ErrorResponse(ErrorDetail("input must contain at least one string", "invalid_request_error", param = "input")) + ) + return + } + if (request.encoding_format != null && request.encoding_format != "float") { + call.respond( + HttpStatusCode.BadRequest, + ErrorResponse(ErrorDetail("only float embeddings are supported", "invalid_request_error", param = "encoding_format")) + ) + return + } + val embeddingsResult = inferenceManager.execute(request.model) { adapter -> + val embeddingAdapter = adapter as? EmbeddingModelAdapter + ?: throw UnsupportedOperationException("Model ${request.model} does not support embeddings") + request.input.mapIndexed { index, text -> - val modelRequest = ModelRequest( - messages = listOf(Turn.User(listOf(ContentBlock.Text(text)), TrustLevel.TRUSTED)), - tools = emptyList(), - toolChoice = ToolChoice.None, - maxOutputTokens = 0, - stopConditions = emptyList() - ) - adapter.invoke(modelRequest).getOrThrow() - Embedding(embedding = listOf(0.0f), index = index) + val vector = embeddingAdapter.embed(text).getOrThrow() + if (vector.isEmpty()) { + throw IllegalStateException("Embedding model returned an empty vector") + } + Embedding(embedding = vector.toList(), index = index) } } if (embeddingsResult.isFailure) { respondClassifiedError(call, embeddingsResult.exceptionOrNull()) return } - call.respond(EmbeddingsResponse(data = embeddingsResult.getOrThrow(), model = request.model, usage = TokenUsage(0, 0, 0))) + + val embeddings = embeddingsResult.getOrThrow() + val dimension = embeddings.firstOrNull()?.embedding?.size ?: 0 + if (dimension == 0 || embeddings.any { it.embedding.size != dimension }) { + call.respond( + HttpStatusCode.InternalServerError, + ErrorResponse(ErrorDetail("Embedding model returned inconsistent vector dimensions", "embedding_error")) + ) + return + } + + call.respond( + EmbeddingsResponse( + data = embeddings, + model = request.model, + usage = TokenUsage(0, 0, 0), + ) + ) } catch (e: Exception) { call.respond(HttpStatusCode.BadRequest, ErrorResponse(ErrorDetail(e.message ?: "Unknown error", "invalid_request_error"))) } @@ -411,15 +415,19 @@ class EngineHttpServer( ErrorDetail("Engine is busy; retry later", "engine_busy", code = "queue_saturated") error is EngineShuttingDownException -> HttpStatusCode.ServiceUnavailable to ErrorDetail("Engine is shutting down", "engine_stopping", code = "shutdown") + error is EngineModelBusyException -> HttpStatusCode.Conflict to + ErrorDetail("Model is busy; wait for inference to finish", "model_busy", code = "model_busy") message.contains("MODEL_NOT_INSTALLED") || message.contains("not installed", ignoreCase = true) -> HttpStatusCode.NotFound to ErrorDetail("Model is not installed", "model_error", code = "model_not_installed") + message.contains("MODEL_INTEGRITY_MISSING") -> + HttpStatusCode.UnprocessableEntity to ErrorDetail("Model integrity metadata is missing", "model_error", code = "model_integrity_missing") + message.contains("MODEL_INTEGRITY_MISMATCH") -> + HttpStatusCode.UnprocessableEntity to ErrorDetail("Model integrity verification failed", "model_error", code = "model_integrity_mismatch") message.contains("INVALID_GGUF") || message.contains("unsupported", ignoreCase = true) -> HttpStatusCode.BadRequest to ErrorDetail("Model file is invalid or unsupported", "model_error", code = "invalid_model") message.contains("INCOMPATIBLE_GGUF") || message.contains("incompatible", ignoreCase = true) -> HttpStatusCode.UnprocessableEntity to ErrorDetail("Model is incompatible with current runtime", "model_error", code = "incompatible_model") - message.contains("MODEL_LOAD_FAILED") -> - HttpStatusCode.InternalServerError to ErrorDetail("Model failed to load", "model_error", code = "model_load_failed") - else -> HttpStatusCode.InternalServerError to ErrorDetail("Inference failed", "inference_error") + else -> HttpStatusCode.InternalServerError to ErrorDetail(message, "inference_error") } } } diff --git a/engine/androidapp/src/jvmAndAndroidMain/kotlin/fi/italeino/aidos/engine/inference/InferenceRequestManager.kt b/engine/androidapp/src/jvmAndAndroidMain/kotlin/fi/italeino/aidos/engine/inference/InferenceRequestManager.kt index e724fa20..abb9084d 100644 --- a/engine/androidapp/src/jvmAndAndroidMain/kotlin/fi/italeino/aidos/engine/inference/InferenceRequestManager.kt +++ b/engine/androidapp/src/jvmAndAndroidMain/kotlin/fi/italeino/aidos/engine/inference/InferenceRequestManager.kt @@ -1,5 +1,6 @@ package fi.italeino.aidos.engine.inference +import dev.aidos.kernel.CancellableModelAdapter import dev.aidos.kernel.ModelAdapter import dev.aidos.kernel.ModelRuntime import kotlinx.coroutines.CancellationException @@ -12,14 +13,7 @@ import kotlinx.coroutines.sync.withLock import kotlin.time.Duration import kotlin.time.Duration.Companion.milliseconds -enum class InferenceRequestState { - QUEUED, - LOADING, - RUNNING, - COMPLETED, - CANCELLED, - FAILED, -} +enum class InferenceRequestState { QUEUED, LOADING, RUNNING, COMPLETED, CANCELLED, FAILED } data class InferenceLifecycleMetrics( val queueDepth: Int, @@ -32,15 +26,9 @@ data class InferenceLifecycleMetrics( class EngineBusyException(message: String) : RuntimeException(message) class EngineShuttingDownException(message: String) : RuntimeException(message) +class EngineModelBusyException(message: String) : RuntimeException(message) -/** - * Serializes model inference and keeps request lifecycle state explicit. - * - * - Bounded admission (`maxConcurrentRequests` + `maxQueuedRequests`) - * - Per-model generation serialization - * - Metrics for queue/running/fail/cancel counts - * - Explicit cancellation of all admitted work during service shutdown - */ +/** Coordinates bounded inference admission with safe model open/close/delete lifecycle operations. */ class InferenceRequestManager( private val modelRuntime: ModelRuntime, private val maxConcurrentRequests: Int = 1, @@ -50,7 +38,11 @@ class InferenceRequestManager( private val stateMutex = Mutex() private val modelLocks = mutableMapOf() private val activeByModel = mutableMapOf() - private val requestJobs = mutableSetOf() + private val admittedByModel = mutableMapOf() + private val closingModels = mutableSetOf() + private val deletingModels = mutableSetOf() + private val activeAdapters = mutableMapOf() + private val requestJobsByModel = mutableMapOf>() private var queueDepth = 0 private var runningRequests = 0 @@ -63,8 +55,7 @@ class InferenceRequestManager( suspend fun execute(modelId: String, block: suspend (ModelAdapter) -> T): Result { val requestJob = currentCoroutineContext()[Job] ?: return Result.failure(IllegalStateException("Inference request requires a coroutine Job")) - - val admitted = acquireAdmissionSlot(requestJob) + val admitted = acquireAdmissionSlot(modelId, requestJob) if (admitted.isFailure) return Result.failure(admitted.exceptionOrNull()!!) try { @@ -72,12 +63,10 @@ class InferenceRequestManager( val adapter = modelRuntime.load(modelId).getOrElse { return Result.failure(it) } val modelMutex = stateMutex.withLock { modelLocks.getOrPut(modelId) { Mutex() } } return modelMutex.withLock { - markActive(modelId, +1) + markActive(modelId, +1, adapter) try { updateState(InferenceRequestState.RUNNING) - val result = block(adapter) - updateState(InferenceRequestState.COMPLETED) - Result.success(result) + Result.success(block(adapter)).also { updateState(InferenceRequestState.COMPLETED) } } catch (e: CancellationException) { updateState(InferenceRequestState.CANCELLED) Result.failure(e) @@ -85,117 +74,182 @@ class InferenceRequestManager( updateState(InferenceRequestState.FAILED) Result.failure(e) } finally { - markActive(modelId, -1) + markActive(modelId, -1, adapter) } } } finally { - releaseAdmissionSlot(requestJob) + releaseAdmissionSlot(modelId, requestJob) + } + } + + /** Force-load a model into memory without performing inference. */ + suspend fun openModel(modelId: String): Result { + val allowed = stateMutex.withLock { + when { + shuttingDown -> Result.failure(EngineShuttingDownException("Engine is shutting down")) + closingModels.contains(modelId) || deletingModels.contains(modelId) -> + Result.failure(EngineModelBusyException("Model $modelId is being closed or deleted")) + else -> Result.success(Unit) + } + } + if (allowed.isFailure) return allowed + + return try { + modelRuntime.load(modelId).map { Unit } + } catch (e: Exception) { + Result.failure(e) + } + } + + /** Interrupt active and queued work, then unload the model while keeping it installed. */ + suspend fun closeModel(modelId: String): Result { + val jobsAndAdapter = stateMutex.withLock { + if (shuttingDown) return@withLock Result.failure(EngineShuttingDownException("Engine is shutting down")) + if (!closingModels.add(modelId)) { + return@withLock Result.failure(EngineModelBusyException("Model $modelId is already being closed")) + } + requestJobsByModel[modelId]?.toList().orEmpty() to activeAdapters[modelId] + } + + return try { + jobsAndAdapter.second?.cancelCurrentInference() + jobsAndAdapter.first.forEach { it.cancel() } + val drained = waitUntilModelAdmissionsDrained(modelId, 5_000.milliseconds) + if (!drained) return Result.failure( + EngineModelBusyException("Model $modelId did not stop within the close timeout") + ) + modelRuntime.unload(modelId) + Result.success(Unit) + } catch (e: Exception) { + Result.failure(e) + } finally { + stateMutex.withLock { closingModels.remove(modelId) } + } + } + + /** Close (and therefore interrupt) the model before removing its installed artifact. */ + suspend fun deleteModel(modelId: String): Result { + val close = closeModel(modelId) + if (close.isFailure) return close + val canDelete = stateMutex.withLock { + if (shuttingDown) return@withLock Result.failure(EngineShuttingDownException("Engine is shutting down")) + if (!deletingModels.add(modelId)) { + return@withLock Result.failure(EngineModelBusyException("Model $modelId is already being deleted")) + } + Result.success(Unit) + } + if (canDelete.isFailure) return canDelete + return try { + modelRuntime.delete(modelId) + Result.success(Unit) + } catch (e: Exception) { + Result.failure(e) + } finally { + stateMutex.withLock { deletingModels.remove(modelId) } } } suspend fun snapshotMetrics(): InferenceLifecycleMetrics = stateMutex.withLock { - InferenceLifecycleMetrics( - queueDepth = queueDepth, - runningRequests = runningRequests, - totalRequests = totalRequests, - completedRequests = completedRequests, - cancelledRequests = cancelledRequests, - failedRequests = failedRequests, - ) + InferenceLifecycleMetrics(queueDepth, runningRequests, totalRequests, completedRequests, cancelledRequests, failedRequests) } suspend fun waitUntilModelIdle(modelId: String, timeout: Duration = 5_000.milliseconds): Boolean { val deadline = System.currentTimeMillis() + timeout.inWholeMilliseconds while (System.currentTimeMillis() < deadline) { - val active = stateMutex.withLock { activeByModel[modelId] ?: 0 } - if (active == 0) return true + if (stateMutex.withLock { (activeByModel[modelId] ?: 0) == 0 }) return true delay(25.milliseconds) } return stateMutex.withLock { (activeByModel[modelId] ?: 0) == 0 } } - /** - * Stops admitting new work and cancels every currently admitted request. - * Cancellation is propagated through the request coroutine to the adapter. The native - * adapter is responsible for observing cancellation at its deepest supported boundary. - */ suspend fun cancelAll() { - val jobs = stateMutex.withLock { + val jobsAndAdapters = stateMutex.withLock { shuttingDown = true - requestJobs.toList() + requestJobsByModel.values.flatten() to activeAdapters.values.toList() } - jobs.forEach { it.cancel() } + jobsAndAdapters.second.forEach { it.cancelCurrentInference() } + jobsAndAdapters.first.forEach { it.cancel() } } - /** - * Prevents new work, cancels admitted requests, and waits for their cleanup to finish. - * This is the shutdown path used by the Android service before native model disposal. - */ suspend fun shutdownAndDrain(timeout: Duration = 5_000.milliseconds): Boolean { cancelAll() val deadline = System.currentTimeMillis() + timeout.inWholeMilliseconds while (System.currentTimeMillis() < deadline) { - val drained = stateMutex.withLock { queueDepth == 0 && runningRequests == 0 } - if (drained) return true + if (stateMutex.withLock { queueDepth == 0 && runningRequests == 0 }) return true delay(25.milliseconds) } return stateMutex.withLock { queueDepth == 0 && runningRequests == 0 } } - private suspend fun acquireAdmissionSlot(requestJob: Job): Result { + private suspend fun acquireAdmissionSlot(modelId: String, requestJob: Job): Result { val admitted = stateMutex.withLock { - if (shuttingDown) { - return@withLock Result.failure( - EngineShuttingDownException("Engine is shutting down") - ) + if (shuttingDown) return@withLock Result.failure(EngineShuttingDownException("Engine is shutting down")) + if (closingModels.contains(modelId) || deletingModels.contains(modelId)) { + return@withLock Result.failure(EngineModelBusyException("Model $modelId is being closed or deleted")) } val inSystem = queueDepth + runningRequests if (inSystem >= maxConcurrentRequests + maxQueuedRequests) { - return@withLock Result.failure( - EngineBusyException( - "Engine is busy (running=$runningRequests, queued=$queueDepth, max=$maxConcurrentRequests+$maxQueuedRequests)" - ) - ) + return@withLock Result.failure(EngineBusyException("Engine is busy (running=$runningRequests, queued=$queueDepth, max=$maxConcurrentRequests+$maxQueuedRequests)")) } queueDepth++ totalRequests++ - requestJobs += requestJob + admittedByModel[modelId] = (admittedByModel[modelId] ?: 0) + 1 + requestJobsByModel.getOrPut(modelId) { mutableSetOf() }.add(requestJob) Result.success(Unit) } if (admitted.isFailure) return admitted - return try { updateState(InferenceRequestState.QUEUED) capacity.acquire() - stateMutex.withLock { - queueDepth-- - runningRequests++ - } + stateMutex.withLock { queueDepth--; runningRequests++ } Result.success(Unit) } catch (e: CancellationException) { stateMutex.withLock { queueDepth = (queueDepth - 1).coerceAtLeast(0) + decrementAdmitted(modelId) cancelledRequests++ - requestJobs.remove(requestJob) + requestJobsByModel[modelId]?.remove(requestJob) + if (requestJobsByModel[modelId].isNullOrEmpty()) requestJobsByModel.remove(modelId) } Result.failure(e) } } - private suspend fun releaseAdmissionSlot(requestJob: Job) { + private suspend fun releaseAdmissionSlot(modelId: String, requestJob: Job) { stateMutex.withLock { runningRequests = (runningRequests - 1).coerceAtLeast(0) - requestJobs.remove(requestJob) + decrementAdmitted(modelId) + requestJobsByModel[modelId]?.remove(requestJob) + if (requestJobsByModel[modelId].isNullOrEmpty()) requestJobsByModel.remove(modelId) } capacity.release() } - private suspend fun markActive(modelId: String, delta: Int) { + private fun decrementAdmitted(modelId: String) { + val next = ((admittedByModel[modelId] ?: 0) - 1).coerceAtLeast(0) + if (next == 0) admittedByModel.remove(modelId) else admittedByModel[modelId] = next + } + + private suspend fun markActive(modelId: String, delta: Int, adapter: ModelAdapter) { stateMutex.withLock { - val current = activeByModel[modelId] ?: 0 - val next = (current + delta).coerceAtLeast(0) - if (next == 0) activeByModel.remove(modelId) else activeByModel[modelId] = next + val next = ((activeByModel[modelId] ?: 0) + delta).coerceAtLeast(0) + if (next == 0) { + activeByModel.remove(modelId) + activeAdapters.remove(modelId) + } else { + activeByModel[modelId] = next + if (adapter is CancellableModelAdapter) activeAdapters[modelId] = adapter + } + } + } + + private suspend fun waitUntilModelAdmissionsDrained(modelId: String, timeout: Duration): Boolean { + val deadline = System.currentTimeMillis() + timeout.inWholeMilliseconds + while (System.currentTimeMillis() < deadline) { + if (stateMutex.withLock { (admittedByModel[modelId] ?: 0) == 0 }) return true + delay(25.milliseconds) } + return stateMutex.withLock { (admittedByModel[modelId] ?: 0) == 0 } } private suspend fun updateState(state: InferenceRequestState) { diff --git a/engine/androidapp/src/jvmTest/kotlin/fi/italeino/aidos/engine/http/EngineHttpServerEmbeddingsTest.kt b/engine/androidapp/src/jvmTest/kotlin/fi/italeino/aidos/engine/http/EngineHttpServerEmbeddingsTest.kt new file mode 100644 index 00000000..05e3b415 --- /dev/null +++ b/engine/androidapp/src/jvmTest/kotlin/fi/italeino/aidos/engine/http/EngineHttpServerEmbeddingsTest.kt @@ -0,0 +1,155 @@ +package fi.italeino.aidos.engine.http + +import dev.aidos.kernel.EmbeddingModelAdapter +import dev.aidos.kernel.ModelAdapter +import dev.aidos.kernel.ModelDescriptor +import dev.aidos.kernel.ModelKind +import dev.aidos.kernel.ModelRequest +import dev.aidos.kernel.ModelResponse +import dev.aidos.kernel.ModelRuntime +import dev.aidos.kernel.ModelStreamEvent +import dev.aidos.kernel.ModelRef +import dev.aidos.kernel.StopReason +import dev.aidos.kernel.TextOutput +import dev.aidos.kernel.Usage +import io.ktor.client.request.* +import io.ktor.client.statement.* +import io.ktor.http.* +import io.ktor.server.testing.* +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotEquals +import kotlin.test.assertTrue + +private val embeddingTestJson = Json { encodeDefaults = true } + +class EngineHttpServerEmbeddingsTest { + @Test + fun embeddings_returnsRealVectorsAndPreservesInputOrder() = testApplication { + val runtime = EmbeddingTestRuntime() + val tokenManager = TokenManager() + val server = EngineHttpServer(tokenManager, runtime) + application { server.installInto(this) } + val token = tokenManager.generateNewToken() + + val response = client.post("/v1/embeddings") { + bearerAuth(token.token) + contentType(ContentType.Application.Json) + setBody(embeddingTestJson.encodeToString( + EmbeddingsRequest( + model = "test-embedding", + input = listOf("alpha", "beta"), + ) + )) + } + + assertEquals(HttpStatusCode.OK, response.status) + val body = response.bodyAsText() + assertTrue(body.contains("\"index\":0")) + assertTrue(body.contains("\"index\":1")) + assertTrue(body.contains("0.5")) + assertTrue(body.contains("-0.25")) + assertTrue(!body.contains("0.0\"]"), "response must not be the old placeholder vector") + } + + @Test + fun embeddings_rejectsEmptyInput() = testApplication { + val runtime = EmbeddingTestRuntime() + val tokenManager = TokenManager() + val server = EngineHttpServer(tokenManager, runtime) + application { server.installInto(this) } + val token = tokenManager.generateNewToken() + + val response = client.post("/v1/embeddings") { + bearerAuth(token.token) + contentType(ContentType.Application.Json) + setBody(embeddingTestJson.encodeToString( + EmbeddingsRequest(model = "test-embedding", input = emptyList()) + )) + } + + assertEquals(HttpStatusCode.BadRequest, response.status) + assertTrue(response.bodyAsText().contains("input must contain at least one string")) + } + + @Test + fun embeddings_rejectsNonFloatEncodingFormat() = testApplication { + val runtime = EmbeddingTestRuntime() + val tokenManager = TokenManager() + val server = EngineHttpServer(tokenManager, runtime) + application { server.installInto(this) } + val token = tokenManager.generateNewToken() + + val response = client.post("/v1/embeddings") { + bearerAuth(token.token) + contentType(ContentType.Application.Json) + setBody(embeddingTestJson.encodeToString( + EmbeddingsRequest( + model = "test-embedding", + input = listOf("alpha"), + encoding_format = "base64", + ) + )) + } + + assertEquals(HttpStatusCode.BadRequest, response.status) + assertTrue(response.bodyAsText().contains("only float embeddings are supported")) + } +} + +private class EmbeddingTestRuntime : ModelRuntime { + private val adapter = EmbeddingTestAdapter() + + override suspend fun catalog(): List = listOf( + ModelDescriptor( + id = "test-embedding", + name = "Test Embedding", + kind = ModelKind.EMBEDDING, + providerId = "test", + isLocal = true, + contextWindow = 2048, + sizeBytes = 1024L, + digest = "test-embedding-digest", + ) + ) + + override suspend fun installed(): List = catalog() + + override suspend fun load(modelId: String): Result = + if (modelId == "test-embedding") Result.success(adapter) + else Result.failure(IllegalStateException("Model $modelId is not installed")) + + override suspend fun unload(modelId: String) = Unit + + override fun loaded(): List = listOf("test-embedding") +} + +private class EmbeddingTestAdapter : EmbeddingModelAdapter { + override val providerId = "test" + override val modelId = "test-embedding" + override val modelVersion = "1.0" + override val contextWindow = 2048 + override val isLocal = true + + override fun supportsNativeToolCalls() = false + + override suspend fun embed(text: String): Result = + Result.success( + when (text) { + "alpha" -> floatArrayOf(0.5f, -0.25f, 0.125f, 0.75f) + "beta" -> floatArrayOf(-0.5f, 0.25f, 0.375f, -0.75f) + else -> floatArrayOf(0.1f, 0.2f, 0.3f, 0.4f) + } + ) + + override suspend fun invoke(request: ModelRequest): Result = + Result.failure(UnsupportedOperationException("embedding test adapter does not chat")) + + override suspend fun invokeStreaming(request: ModelRequest): Flow = flow { + emit(ModelStreamEvent.Failed(UnsupportedOperationException("embedding test adapter does not stream chat"))) + } +} diff --git a/engine/androidapp/src/jvmTest/kotlin/fi/italeino/aidos/engine/http/EngineHttpServerIntegrityTest.kt b/engine/androidapp/src/jvmTest/kotlin/fi/italeino/aidos/engine/http/EngineHttpServerIntegrityTest.kt new file mode 100644 index 00000000..c10c2a76 --- /dev/null +++ b/engine/androidapp/src/jvmTest/kotlin/fi/italeino/aidos/engine/http/EngineHttpServerIntegrityTest.kt @@ -0,0 +1,130 @@ +package fi.italeino.aidos.engine.http + +import dev.aidos.kernel.ModelAdapter +import dev.aidos.kernel.ModelDescriptor +import dev.aidos.kernel.ModelKind +import dev.aidos.kernel.ModelRuntime +import io.ktor.client.request.* +import io.ktor.client.statement.* +import io.ktor.http.* +import io.ktor.server.testing.* +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +private val integrityTestJson = Json { encodeDefaults = true } + +class EngineHttpServerIntegrityTest { + @Test + fun chatCompletions_integrityMismatchReturnsUnprocessableEntity() = testApplication { + val tokenManager = TokenManager() + val runtime = IntegrityFailureRuntime("MODEL_INTEGRITY_MISMATCH: test-model") + val server = EngineHttpServer(tokenManager, runtime) + application { server.installInto(this) } + val token = tokenManager.generateNewToken() + + val response = client.post("/v1/chat/completions") { + bearerAuth(token.token) + contentType(ContentType.Application.Json) + setBody(integrityTestJson.encodeToString( + ChatCompletionRequest( + model = "test-model", + messages = listOf(ChatMessage(role = "user", content = "hello")), + ) + )) + } + + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + val body = response.bodyAsText() + assertTrue(body.contains("model_integrity_mismatch")) + assertTrue(!body.contains("MODEL_INTEGRITY_MISMATCH")) + } + + @Test + fun chatCompletions_missingIntegrityReturnsUnprocessableEntity() = testApplication { + val tokenManager = TokenManager() + val runtime = IntegrityFailureRuntime("MODEL_INTEGRITY_MISSING: test-model") + val server = EngineHttpServer(tokenManager, runtime) + application { server.installInto(this) } + val token = tokenManager.generateNewToken() + + val response = client.post("/v1/chat/completions") { + bearerAuth(token.token) + contentType(ContentType.Application.Json) + setBody(integrityTestJson.encodeToString( + ChatCompletionRequest( + model = "test-model", + messages = listOf(ChatMessage(role = "user", content = "hello")), + ) + )) + } + + assertEquals(HttpStatusCode.UnprocessableEntity, response.status) + assertTrue(response.bodyAsText().contains("model_integrity_missing")) + } + + @Test + fun models_exposesCatalogMetadataWithoutFilenameInference() = testApplication { + val tokenManager = TokenManager() + val runtime = MetadataRuntime() + val server = EngineHttpServer(tokenManager, runtime) + application { server.installInto(this) } + val token = tokenManager.generateNewToken() + + val response = client.get("/v1/models") { + bearerAuth(token.token) + } + + assertEquals(HttpStatusCode.OK, response.status) + val body = response.bodyAsText() + assertTrue(body.contains("\"format\":\"gguf\"")) + assertTrue(body.contains("\"quantization\":\"Q5_K_M\"")) + assertTrue(body.contains("\"family\":\"test\"")) + assertTrue(!body.contains("\"quantization\":\"q4_k_m\"")) + } +} + +private class IntegrityFailureRuntime(private val message: String) : ModelRuntime { + override suspend fun catalog(): List = descriptor() + override suspend fun installed(): List = descriptor() + override suspend fun load(modelId: String): Result = Result.failure(IllegalStateException(message)) + override suspend fun unload(modelId: String) = Unit + override fun loaded(): List = emptyList() + + private fun descriptor() = listOf( + ModelDescriptor( + id = "test-model", + name = "Test Model", + kind = ModelKind.LLM, + providerId = "test", + isLocal = true, + contextWindow = 2048, + sizeBytes = 1L, + digest = "digest", + ) + ) +} + +private class MetadataRuntime : ModelRuntime { + private val descriptor = ModelDescriptor( + id = "model-q4_k_m", + name = "Metadata Model", + kind = ModelKind.LLM, + providerId = "test", + isLocal = true, + contextWindow = 4096, + sizeBytes = 123L, + digest = "abc", + format = "gguf", + quantization = "Q5_K_M", + metadata = mapOf("family" to "test"), + ) + + override suspend fun catalog(): List = listOf(descriptor) + override suspend fun installed(): List = listOf(descriptor) + override suspend fun load(modelId: String): Result = Result.failure(IllegalStateException("unused")) + override suspend fun unload(modelId: String) = Unit + override fun loaded(): List = emptyList() +} diff --git a/engine/androidapp/src/jvmTest/kotlin/fi/italeino/aidos/engine/inference/InferenceRequestManagerTest.kt b/engine/androidapp/src/jvmTest/kotlin/fi/italeino/aidos/engine/inference/InferenceRequestManagerTest.kt index c09f7fad..92f95cd2 100644 --- a/engine/androidapp/src/jvmTest/kotlin/fi/italeino/aidos/engine/inference/InferenceRequestManagerTest.kt +++ b/engine/androidapp/src/jvmTest/kotlin/fi/italeino/aidos/engine/inference/InferenceRequestManagerTest.kt @@ -1,5 +1,6 @@ package fi.italeino.aidos.engine.inference +import dev.aidos.kernel.CancellableModelAdapter import dev.aidos.kernel.ModelAdapter import dev.aidos.kernel.ModelDescriptor import dev.aidos.kernel.ModelKind @@ -21,163 +22,232 @@ import kotlin.test.assertTrue import kotlin.time.Duration.Companion.milliseconds class InferenceRequestManagerTest { - @Test fun execute_rejectsWhenQueueIsSaturated() = runTest { val gate = CompletableDeferred() val runtime = FakeRuntime(BlockingAdapter(gate)) - val manager = InferenceRequestManager(runtime, maxConcurrentRequests = 1, maxQueuedRequests = 0) - + val manager = InferenceRequestManager(runtime, 1, 0) coroutineScope { - val first = async { - manager.execute("test-model") { adapter -> - adapter.invoke(dummyRequest()).getOrThrow() - } - } + val first = async { manager.execute("test-model") { it.invoke(dummyRequest()).getOrThrow() } } delay(50) - val second = manager.execute("test-model") { adapter -> - adapter.invoke(dummyRequest()).getOrThrow() - } - + val second = manager.execute("test-model") { it.invoke(dummyRequest()).getOrThrow() } assertTrue(second.isFailure) assertTrue(second.exceptionOrNull() is EngineBusyException) gate.complete(Unit) assertTrue(first.await().isSuccess) } - - val metrics = manager.snapshotMetrics() - assertEquals(1, metrics.totalRequests) - assertEquals(1, metrics.completedRequests) + assertEquals(1, manager.snapshotMetrics().totalRequests) + assertEquals(1, manager.snapshotMetrics().completedRequests) } @Test fun execute_serializesConcurrentCallsPerModel() = runTest { - val serializingAdapter = CountingAdapter() - val runtime = FakeRuntime(serializingAdapter) - val manager = InferenceRequestManager(runtime, maxConcurrentRequests = 2, maxQueuedRequests = 2) - + val adapter = CountingAdapter() + val manager = InferenceRequestManager(FakeRuntime(adapter), 2, 2) coroutineScope { - val first = async { - manager.execute("test-model") { adapter -> - adapter.invoke(dummyRequest()).getOrThrow() - } - } - val second = async { - manager.execute("test-model") { adapter -> - adapter.invoke(dummyRequest()).getOrThrow() - } - } - + val first = async { manager.execute("test-model") { it.invoke(dummyRequest()).getOrThrow() } } + val second = async { manager.execute("test-model") { it.invoke(dummyRequest()).getOrThrow() } } assertTrue(first.await().isSuccess) assertTrue(second.await().isSuccess) } + assertEquals(1, adapter.maxConcurrentInvokes) + } - assertEquals(1, serializingAdapter.maxConcurrentInvokes) - val metrics = manager.snapshotMetrics() - assertEquals(2, metrics.completedRequests) + @Test + fun openModel_forceLoadsWithoutInference() = runTest { + val adapter = CountingAdapter() + val runtime = FakeRuntime(adapter) + val manager = InferenceRequestManager(runtime, 1, 0) + + val result = manager.openModel("test-model") + + assertTrue(result.isSuccess) + assertEquals(1, runtime.loadCalls) + assertEquals(0, adapter.invokeCalls) + } + + @Test + fun closeModel_interruptsRunningInferenceAndUnloadsAfterCancellation() = runTest { + val adapter = CancellationAwareAdapter() + val runtime = FakeRuntime(adapter) + val manager = InferenceRequestManager(runtime, 1, 0) + coroutineScope { + val request = async { manager.execute("test-model") { it.invoke(dummyRequest()).getOrThrow() } } + adapter.started.await() + val close = async { manager.closeModel("test-model") } + adapter.cancelled.await() + assertTrue(!close.isCompleted, "close must wait for the request to leave the manager") + assertTrue(close.await().isSuccess) + assertTrue(request.await().isFailure) + assertEquals(1, runtime.unloadCalls) + assertEquals(1, adapter.cancelCalls) + } + } + + @Test + fun closeModel_cancelsQueuedRequestsForTargetModel() = runTest { + val adapter = CancellationAwareAdapter() + val runtime = FakeRuntime(adapter) + val manager = InferenceRequestManager(runtime, 1, 2) + coroutineScope { + val first = async { manager.execute("test-model") { it.invoke(dummyRequest()).getOrThrow() } } + adapter.started.await() + val queued = async { manager.execute("test-model") { it.invoke(dummyRequest()).getOrThrow() } } + delay(25) + assertTrue(manager.closeModel("test-model").isSuccess) + assertTrue(first.await().isFailure) + assertTrue(queued.await().isFailure) + assertEquals(1, runtime.unloadCalls) + } + } + + @Test + fun deleteModel_interruptsInferenceBeforeDeleting() = runTest { + val adapter = CancellationAwareAdapter() + val runtime = FakeRuntime(adapter) + val manager = InferenceRequestManager(runtime, 1, 0) + coroutineScope { + val request = async { manager.execute("test-model") { it.invoke(dummyRequest()).getOrThrow() } } + adapter.started.await() + val deletion = async { manager.deleteModel("test-model") } + adapter.cancelled.await() + assertTrue(deletion.await().isSuccess) + assertTrue(request.await().isFailure) + assertEquals(1, runtime.unloadCalls) + assertEquals(1, runtime.deleteCalls) + } + } + + @Test + fun deleteModel_blocksNewInferenceUntilDeletionFinishes() = runTest { + val deleteStarted = CompletableDeferred() + val deleteGate = CompletableDeferred() + val runtime = FakeRuntime(CountingAdapter(), deleteStarted, deleteGate) + val manager = InferenceRequestManager(runtime, 1, 0) + coroutineScope { + val deletion = async { manager.deleteModel("test-model") } + deleteStarted.await() + val inference = manager.execute("test-model") { it.invoke(dummyRequest()).getOrThrow() } + assertTrue(inference.isFailure) + assertTrue(inference.exceptionOrNull() is EngineModelBusyException) + deleteGate.complete(Unit) + assertTrue(deletion.await().isSuccess) + assertEquals(1, runtime.deleteCalls) + } } @Test fun shutdownAndDrain_waitsUntilRunningRequestsFinish() = runTest { val gate = CompletableDeferred() - val runtime = FakeRuntime(BlockingAdapter(gate)) - val manager = InferenceRequestManager(runtime, maxConcurrentRequests = 1, maxQueuedRequests = 1) - + val manager = InferenceRequestManager(FakeRuntime(BlockingAdapter(gate)), 1, 1) coroutineScope { - val request = async { - manager.execute("test-model") { adapter -> - adapter.invoke(dummyRequest()).getOrThrow() - } - } + val request = async { manager.execute("test-model") { it.invoke(dummyRequest()).getOrThrow() } } delay(50) - - val shutdown = async { manager.shutdownAndDrain(timeout = 300.milliseconds) } + val shutdown = async { manager.shutdownAndDrain(300.milliseconds) } delay(50) - assertTrue(!shutdown.isCompleted, "shutdown should wait while request is still running") + assertTrue(!shutdown.isCompleted) gate.complete(Unit) assertTrue(shutdown.await()) request.await() } } - private fun dummyRequest() = ModelRequest( - messages = emptyList(), - tools = emptyList(), - toolChoice = dev.aidos.kernel.ToolChoice.None, - maxOutputTokens = 16, - ) -} - -private class FakeRuntime(private val adapter: ModelAdapter) : ModelRuntime { - override suspend fun catalog(): List = listOf( - ModelDescriptor( - id = "test-model", - name = "Test", - kind = ModelKind.LLM, - providerId = "test", - isLocal = true, - contextWindow = 2048, - sizeBytes = 1234, - digest = null, - ) - ) - - override suspend fun installed(): List = catalog() - - override suspend fun load(modelId: String): Result = Result.success(adapter) + @Test + fun shutdownAndDrain_cancelsRunningRequestAndRecordsCancellation() = runTest { + val adapter = CancellationAwareAdapter() + val manager = InferenceRequestManager(FakeRuntime(adapter), 1, 0) + coroutineScope { + val request = async { manager.execute("test-model") { it.invoke(dummyRequest()).getOrThrow() } } + adapter.started.await() + assertTrue(manager.shutdownAndDrain(500.milliseconds)) + assertTrue(request.await().isFailure) + } + val metrics = manager.snapshotMetrics() + assertEquals(1, metrics.totalRequests) + assertEquals(1, metrics.cancelledRequests) + assertEquals(0, metrics.runningRequests) + } - override suspend fun unload(modelId: String) = Unit + private fun dummyRequest() = ModelRequest(emptyList(), emptyList(), dev.aidos.kernel.ToolChoice.None, 16) +} - override fun loaded(): List = listOf("test-model") +private open class FakeRuntime( + private val adapter: ModelAdapter, + private val deleteStarted: CompletableDeferred? = null, + private val deleteGate: CompletableDeferred? = null, +) : ModelRuntime { + var deleteCalls = 0 + private set + var unloadCalls = 0 + private set + var loadCalls = 0 + private set + override suspend fun catalog() = listOf(ModelDescriptor("test-model", "Test", ModelKind.LLM, "test", true, 2048, 1234, null)) + override suspend fun installed() = catalog() + override suspend fun load(modelId: String): Result { + loadCalls++ + return Result.success(adapter) + } + override suspend fun unload(modelId: String) { unloadCalls++ } + override suspend fun delete(modelId: String) { + deleteCalls++ + deleteStarted?.complete(Unit) + deleteGate?.await() + } + override fun loaded() = listOf("test-model") } private class BlockingAdapter(private val gate: CompletableDeferred) : ModelAdapter { - override val providerId: String = "test" - override val modelId: String = "test-model" - override val modelVersion: String = "1" - override val contextWindow: Int = 2048 - override val isLocal: Boolean = true - - override fun supportsNativeToolCalls(): Boolean = false - + override val providerId = "test" + override val modelId = "test-model" + override val modelVersion = "1" + override val contextWindow = 2048 + override val isLocal = true + override fun supportsNativeToolCalls() = false override suspend fun invoke(request: ModelRequest): Result { gate.await() return Result.success(response()) } + private fun response() = ModelResponse(listOf(TextOutput("ok")), StopReason.END_TURN, Usage(1, 1, 2), ModelRef(modelId, modelVersion)) +} - private fun response() = ModelResponse( - outputs = listOf(TextOutput("ok")), - stopReason = StopReason.END_TURN, - usage = Usage(1, 1, 2), - model = ModelRef(modelId, modelVersion), - ) +private class CancellationAwareAdapter : CancellableModelAdapter { + override val providerId = "test" + override val modelId = "test-model" + override val modelVersion = "1" + override val contextWindow = 2048 + override val isLocal = true + val started = CompletableDeferred() + val cancelled = CompletableDeferred() + var cancelCalls = 0 + private set + override fun supportsNativeToolCalls() = false + override suspend fun invoke(request: ModelRequest): Result { + started.complete(Unit) + cancelled.await() + return Result.failure(kotlinx.coroutines.CancellationException("native inference cancelled")) + } + override fun cancelCurrentInference() { cancelCalls++; cancelled.complete(Unit) } } private class CountingAdapter : ModelAdapter { - override val providerId: String = "test" - override val modelId: String = "test-model" - override val modelVersion: String = "1" - override val contextWindow: Int = 2048 - override val isLocal: Boolean = true - - private var activeInvokes: Int = 0 - var maxConcurrentInvokes: Int = 0 + override val providerId = "test" + override val modelId = "test-model" + override val modelVersion = "1" + override val contextWindow = 2048 + override val isLocal = true + private var activeInvokes = 0 + var maxConcurrentInvokes = 0 private set - - override fun supportsNativeToolCalls(): Boolean = false - + var invokeCalls = 0 + private set + override fun supportsNativeToolCalls() = false override suspend fun invoke(request: ModelRequest): Result { + invokeCalls++ activeInvokes++ - if (activeInvokes > maxConcurrentInvokes) maxConcurrentInvokes = activeInvokes + maxConcurrentInvokes = maxOf(maxConcurrentInvokes, activeInvokes) delay(50) activeInvokes-- - return Result.success( - ModelResponse( - outputs = listOf(TextOutput("ok")), - stopReason = StopReason.END_TURN, - usage = Usage(1, 1, 2), - model = ModelRef(modelId, modelVersion), - ) - ) + return Result.success(ModelResponse(listOf(TextOutput("ok")), StopReason.END_TURN, Usage(1, 1, 2), ModelRef(modelId, modelVersion))) } } diff --git a/engine/models/src/commonMain/kotlin/dev/aidos/models/DatabaseModelCatalogManager.kt b/engine/models/src/commonMain/kotlin/dev/aidos/models/DatabaseModelCatalogManager.kt index e9020a15..0a7e1613 100644 --- a/engine/models/src/commonMain/kotlin/dev/aidos/models/DatabaseModelCatalogManager.kt +++ b/engine/models/src/commonMain/kotlin/dev/aidos/models/DatabaseModelCatalogManager.kt @@ -3,18 +3,16 @@ package dev.aidos.models import app.cash.sqldelight.db.QueryResult import app.cash.sqldelight.db.SqlDriver import dev.aidos.kernel.ModelKind +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive import java.time.Instant /** * Database-backed implementation of ModelCatalogManager (RFC-0022). * - * Manages model_catalog and installed_models tables in storage using SqlDelight's SqlDriver. - * Lives in commonMain (not jvmMain, where it lived before being deleted in 7d2c9ea without a - * replacement -- restored here since EngineService references it and the app cannot build - * without it) so both Aidos Engine's Android app and any JVM host can use it. `java.time.Instant` - * is safe in commonMain here because this module's targets are jvm()+androidTarget() only, both - * of which have it (see dev.aidos.api.ProjectLocker's doc comment for the same reasoning about - * java.io.File). + * The catalog is authoritative: an installed row may only reference a known catalog entry, and + * when the catalog pins a SHA-256 digest the installed artifact must carry that same digest. */ class DatabaseModelCatalogManager( private val userDriver: SqlDriver, @@ -61,9 +59,7 @@ class DatabaseModelCatalogManager( QueryResult.Value(results) }, parameters = 0, - ) { - // No parameters - }.value + ) { }.value } override suspend fun getCatalog(modelId: String): Result = runCatching { @@ -83,14 +79,10 @@ class DatabaseModelCatalogManager( discoveredAt = c.getString(6)!!, ) ) - } else { - QueryResult.Value(null) - } + } else QueryResult.Value(null) }, parameters = 1, - ) { - bindString(0, modelId) - }.value + ) { bindString(0, modelId) }.value } override suspend fun listInstalled(): Result> = runCatching { @@ -120,9 +112,7 @@ class DatabaseModelCatalogManager( QueryResult.Value(results) }, parameters = 0, - ) { - // No parameters - }.value + ) { }.value } override suspend fun markInstalled( @@ -132,6 +122,17 @@ class DatabaseModelCatalogManager( sizeBytes: Long, quantization: String?, ): Result = runCatching { + val catalog = getCatalog(modelId).getOrThrow() + ?: throw IllegalArgumentException("Cannot install unknown model '$modelId'") + val expectedDigest = runCatching { + Json.parseToJsonElement(catalog.propertiesJson).jsonObject["sha256"]?.jsonPrimitive?.content + }.getOrNull() + if (expectedDigest != null && !expectedDigest.equals(digest, ignoreCase = true)) { + throw IllegalArgumentException( + "Catalog digest mismatch for '$modelId': expected $expectedDigest, got $digest" + ) + } + val now = Instant.now().toString() userDriver.execute( identifier = null, @@ -156,9 +157,7 @@ class DatabaseModelCatalogManager( identifier = null, sql = "DELETE FROM installed_models WHERE model_id = ?", parameters = 1, - ) { - bindString(0, modelId) - } + ) { bindString(0, modelId) } } override suspend fun updateInstalledMetadata( @@ -166,53 +165,36 @@ class DatabaseModelCatalogManager( userLabel: String?, propertiesJson: String?, ): Result = runCatching { - // Build dynamic UPDATE statement based on what's being updated when { - userLabel != null && propertiesJson != null -> { - // Update both fields - userDriver.execute( - identifier = null, - sql = "UPDATE installed_models SET user_label = ?, properties_json = ? WHERE model_id = ?", - parameters = 3, - ) { - bindString(0, userLabel) - bindString(1, propertiesJson) - bindString(2, modelId) - } + userLabel != null && propertiesJson != null -> userDriver.execute( + identifier = null, + sql = "UPDATE installed_models SET user_label = ?, properties_json = ? WHERE model_id = ?", + parameters = 3, + ) { + bindString(0, userLabel) + bindString(1, propertiesJson) + bindString(2, modelId) } - userLabel != null -> { - // Update only user_label - userDriver.execute( - identifier = null, - sql = "UPDATE installed_models SET user_label = ? WHERE model_id = ?", - parameters = 2, - ) { - bindString(0, userLabel) - bindString(1, modelId) - } + userLabel != null -> userDriver.execute( + identifier = null, + sql = "UPDATE installed_models SET user_label = ? WHERE model_id = ?", + parameters = 2, + ) { + bindString(0, userLabel) + bindString(1, modelId) } - propertiesJson != null -> { - // Update only properties_json - userDriver.execute( - identifier = null, - sql = "UPDATE installed_models SET properties_json = ? WHERE model_id = ?", - parameters = 2, - ) { - bindString(0, propertiesJson) - bindString(1, modelId) - } + propertiesJson != null -> userDriver.execute( + identifier = null, + sql = "UPDATE installed_models SET properties_json = ? WHERE model_id = ?", + parameters = 2, + ) { + bindString(0, propertiesJson) + bindString(1, modelId) } - // else: nothing to update, just return success } } companion object { - /** - * Creates model_catalog and installed_models if they don't exist yet. No migration - * runner exists for this module (unlike agent/storage's MigrationRunner) -- this is the - * schema's one and only version, called from the SqlDriver's SqlSchema.create() at first - * open. - */ fun createTables(driver: SqlDriver) { driver.execute( identifier = null, diff --git a/engine/models/src/jvmTest/kotlin/dev/aidos/models/DatabaseModelCatalogManagerTest.kt b/engine/models/src/jvmTest/kotlin/dev/aidos/models/DatabaseModelCatalogManagerTest.kt index 35b1a32f..f3c473a4 100644 --- a/engine/models/src/jvmTest/kotlin/dev/aidos/models/DatabaseModelCatalogManagerTest.kt +++ b/engine/models/src/jvmTest/kotlin/dev/aidos/models/DatabaseModelCatalogManagerTest.kt @@ -5,18 +5,11 @@ import dev.aidos.kernel.ModelKind import kotlinx.coroutines.runBlocking import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertFalse import kotlin.test.assertNull import kotlin.test.assertTrue -/** - * DatabaseModelCatalogManager (RFC-0022) against a real, in-memory SQLite driver. - * - * This class was deleted in 7d2c9ea with no replacement, leaving EngineService referencing a - * class that no longer existed anywhere -- a build break that only a real Android compile catches - * (this sandbox has no Android SDK; `gradle jvmTest` alone would not have caught it either, since - * it never constructed one against real storage). These tests exercise the schema - * (`createTables`) and the manager together, the way EngineService actually wires them. - */ +/** DatabaseModelCatalogManager against a real in-memory SQLite driver. */ class DatabaseModelCatalogManagerTest { private fun manager(): DatabaseModelCatalogManager { @@ -25,16 +18,21 @@ class DatabaseModelCatalogManagerTest { return DatabaseModelCatalogManager(driver) } - private fun entry(id: String) = CatalogEntry( - id = id, name = "Model $id", kind = ModelKind.LLM, provider = "huggingface", - remoteUrl = "https://example.test/$id", discoveredAt = "2026-08-25T00:00:00Z", + private fun entry(id: String, propertiesJson: String = "{}") = CatalogEntry( + id = id, + name = "Model $id", + kind = ModelKind.LLM, + provider = "huggingface", + remoteUrl = "https://example.test/$id", + propertiesJson = propertiesJson, + discoveredAt = "2026-08-25T00:00:00Z", ) @Test fun `createTables is idempotent`() = runBlocking { val driver = JdbcSqliteDriver(JdbcSqliteDriver.IN_MEMORY) DatabaseModelCatalogManager.createTables(driver) - DatabaseModelCatalogManager.createTables(driver) // must not throw "table already exists" + DatabaseModelCatalogManager.createTables(driver) } @Test @@ -54,9 +52,51 @@ class DatabaseModelCatalogManagerTest { assertNull(manager.getCatalog("no-such-model").getOrThrow()) } + @Test + fun `markInstalled requires a catalog entry`() = runBlocking { + val manager = manager() + + val result = manager.markInstalled( + "model-a", digest = "sha256:abc", path = "/models/a.gguf", sizeBytes = 4096L, + ) + + assertTrue(result.isFailure) + assertTrue(result.exceptionOrNull()?.message?.contains("unknown model") == true) + assertTrue(manager.listInstalled().getOrThrow().isEmpty()) + } + + @Test + fun `markInstalled accepts a digest pinned by catalog metadata`() = runBlocking { + val manager = manager() + manager.addToCatalog(entry("model-a", "{\"sha256\":\"sha256:abc\"}")).getOrThrow() + + manager.markInstalled( + "model-a", digest = "SHA256:ABC", path = "/models/a.gguf", sizeBytes = 4096L, + ).getOrThrow() + + val installed = manager.listInstalled().getOrThrow().single() + assertEquals("SHA256:ABC", installed.digest) + assertEquals("/models/a.gguf", installed.path) + } + + @Test + fun `markInstalled rejects a digest different from catalog`() = runBlocking { + val manager = manager() + manager.addToCatalog(entry("model-a", "{\"sha256\":\"sha256:abc\"}")).getOrThrow() + + val result = manager.markInstalled( + "model-a", digest = "sha256:wrong", path = "/models/a.gguf", sizeBytes = 4096L, + ) + + assertFalse(result.isSuccess) + assertTrue(result.exceptionOrNull()?.message?.contains("Catalog digest mismatch") == true) + assertTrue(manager.listInstalled().getOrThrow().isEmpty()) + } + @Test fun `markInstalled then listInstalled round-trips`() = runBlocking { val manager = manager() + manager.addToCatalog(entry("model-a")).getOrThrow() manager.markInstalled("model-a", digest = "sha256:abc", path = "/models/a.gguf", sizeBytes = 4096L, quantization = "Q4_K_M").getOrThrow() val installed = manager.listInstalled().getOrThrow() @@ -69,7 +109,8 @@ class DatabaseModelCatalogManagerTest { @Test fun `uninstall removes the installed row`() = runBlocking { val manager = manager() - manager.markInstalled("model-a", digest = "d", path = "/p", sizeBytes = 1L, quantization = null).getOrThrow() + manager.addToCatalog(entry("model-a")).getOrThrow() + manager.markInstalled("model-a", digest = "d", path = "/p", sizeBytes = 1L).getOrThrow() manager.uninstall("model-a").getOrThrow() assertTrue(manager.listInstalled().getOrThrow().isEmpty()) @@ -78,7 +119,8 @@ class DatabaseModelCatalogManagerTest { @Test fun `updateInstalledMetadata sets userLabel without touching propertiesJson`() = runBlocking { val manager = manager() - manager.markInstalled("model-a", digest = "d", path = "/p", sizeBytes = 1L, quantization = null).getOrThrow() + manager.addToCatalog(entry("model-a")).getOrThrow() + manager.markInstalled("model-a", digest = "d", path = "/p", sizeBytes = 1L).getOrThrow() manager.updateInstalledMetadata("model-a", userLabel = "My Model").getOrThrow() diff --git a/kernel/src/commonMain/kotlin/dev/aidos/kernel/Models.kt b/kernel/src/commonMain/kotlin/dev/aidos/kernel/Models.kt index 17a29f42..4c2d72a4 100644 --- a/kernel/src/commonMain/kotlin/dev/aidos/kernel/Models.kt +++ b/kernel/src/commonMain/kotlin/dev/aidos/kernel/Models.kt @@ -18,14 +18,6 @@ interface ModelAdapter { fun supportsNativeToolCalls(): Boolean suspend fun invoke(request: ModelRequest): Result - /** - * Token-by-token variant of [invoke] (RFC-0021, "Streaming"). The default falls back to - * [invoke] and emits its result as a single terminal event — correct for any adapter whose - * backend has no partial output to offer. An adapter backed by a token-emitting inference - * engine (e.g. llama.cpp) overrides this to yield real incremental [ModelStreamEvent.Delta]s - * as they're produced, rather than a caller having to wait for [invoke] to return and then - * chop the finished text into fake chunks. - */ suspend fun invokeStreaming(request: ModelRequest): Flow = flow { invoke(request).fold( onSuccess = { emit(ModelStreamEvent.Done(it)) }, @@ -36,19 +28,20 @@ interface ModelAdapter { val providerRetention: ProviderRetention? get() = null } -/** - * One event of a streamed [ModelAdapter.invokeStreaming] response. Named to parallel RFC-0052's - * `RuntimeEvent.AiResponseDelta` one layer down the stack: this is the adapter-to-router event, - * not the router-to-frontend event RFC-0052 already defines from it. - */ +/** Adapter capability for interrupting the currently running native inference. */ +interface CancellableModelAdapter : ModelAdapter { + /** Interrupt the native inference operation currently owned by this adapter, if any. */ + fun cancelCurrentInference() +} + +/** Adapter capability for models that expose native vector embeddings. */ +interface EmbeddingModelAdapter : ModelAdapter { + suspend fun embed(text: String): Result +} + sealed interface ModelStreamEvent { - /** A partial increment of assistant text as it is produced. Zero or more per stream. */ data class Delta(val text: String) : ModelStreamEvent - - /** Terminal: the complete response this stream produced, once generation finished. */ data class Done(val response: ModelResponse) : ModelStreamEvent - - /** Terminal: generation failed partway through (or before producing any output at all). */ data class Failed(val error: Throwable) : ModelStreamEvent } @@ -71,7 +64,6 @@ data class ModelRequest( val stopConditions: List = emptyList(), ) -/** Generalized RFC-0022 response. Outputs remain ordered and ModelOutput is intentionally open. */ data class ModelResponse( val outputs: List, val stopReason: StopReason?, @@ -124,14 +116,6 @@ sealed interface RoutingDecision { data object ForegroundRequired : RoutingDecision } -interface ModelRuntime { - suspend fun catalog(): List - suspend fun installed(): List - suspend fun load(modelId: String): Result - suspend fun unload(modelId: String) - fun loaded(): List -} - data class ModelDescriptor( val id: String, val name: String, @@ -141,6 +125,9 @@ data class ModelDescriptor( val contextWindow: Int, val sizeBytes: Long?, val digest: String?, + val format: String? = null, + val quantization: String? = null, + val metadata: Map = emptyMap(), ) @Serializable @@ -182,3 +169,12 @@ data class ScheduledJob( val missedOccurrences: Int = 0, val createdAt: Instant, ) + +interface ModelRuntime { + suspend fun catalog(): List + suspend fun installed(): List + suspend fun load(modelId: String): Result + suspend fun unload(modelId: String) + fun loaded(): List + suspend fun delete(modelId: String) +}