From 0a5666a098838797d0f03e2d2b415e7d9acb036b Mon Sep 17 00:00:00 2001 From: biersoeckli Date: Sun, 8 Nov 2026 12:24:14 +0000 Subject: [PATCH 01/11] feat: report app volume filesystem usage --- src/app/monitoring/app-volumes-monitoring.tsx | 46 ++---- .../app-volumes-monitoring.unit.spec.ts | 22 +++ .../apps/details/volumes/actions.ts | 2 +- .../apps/details/volumes/volumes-list.tsx | 14 +- src/server/adapter/volume-summary.adapter.ts | 29 ++++ src/server/services/monitoring.service.ts | 137 ++++++++---------- .../services/monitoring.service.unit.spec.ts | 118 +++++++-------- .../services/volume-filesystem-drivers.ts | 13 ++ .../app-volume-monitoring-usage.model.ts | 27 ++-- 9 files changed, 217 insertions(+), 191 deletions(-) create mode 100644 src/app/monitoring/app-volumes-monitoring.unit.spec.ts create mode 100644 src/server/adapter/volume-summary.adapter.ts create mode 100644 src/server/services/volume-filesystem-drivers.ts diff --git a/src/app/monitoring/app-volumes-monitoring.tsx b/src/app/monitoring/app-volumes-monitoring.tsx index 3466850d..6855ea38 100644 --- a/src/app/monitoring/app-volumes-monitoring.tsx +++ b/src/app/monitoring/app-volumes-monitoring.tsx @@ -18,37 +18,23 @@ import { Button } from '@/components/ui/button'; import { Progress } from "@/components/ui/progress" import { PathBuilderUtils } from '@/shared/utils/path-builder.utils'; -type AppVolumeMonitoringUsageExtendedModel = AppVolumeMonitoringUsageModel & { - usedPercentage: number; -}; - -function convertToExtendedModel(input?: AppVolumeMonitoringUsageModel[]): AppVolumeMonitoringUsageExtendedModel[] | undefined { - if (input) { - return input.map(item => ({ - ...item, - usedPercentage: Math.round(item.usedBytes / item.capacityBytes * 100) - })); - } - return undefined; -} - export default function AppVolumeMonitoring({ volumesUsage }: { volumesUsage?: AppVolumeMonitoringUsageModel[] }) { - const updatedVolumeUsage = useMemo(() => convertToExtendedModel(volumesUsage), [volumesUsage]); - - const { totalUsedBytes, totalCapacityBytes } = useMemo(() => { - if (!updatedVolumeUsage) { - return { totalUsedBytes: undefined, totalCapacityBytes: undefined }; - } + const updatedVolumeUsage = volumesUsage; + const totals = useMemo(() => { + const unique = new Map(); + for (const volume of volumesUsage ?? []) unique.set(`${volume.projectId}/${volume.pvcName}`, volume); + const values = [...unique.values()]; return { - totalUsedBytes: updatedVolumeUsage.reduce((acc, item) => acc + item.usedBytes, 0), - totalCapacityBytes: updatedVolumeUsage.reduce((acc, item) => acc + item.capacityBytes, 0), + used: values.reduce((sum, value) => sum + (value.usedBytes ?? 0), 0), + capacity: values.reduce((sum, value) => sum + (value.capacityBytes ?? 0), 0), + incomplete: values.some(value => value.status !== 'available'), }; - }, [updatedVolumeUsage]); + }, [volumesUsage]); if (!updatedVolumeUsage) { return @@ -68,10 +54,7 @@ export default function AppVolumeMonitoring({ - {updatedVolumeUsage.length} App Volumes {totalUsedBytes && totalCapacityBytes && <> - | Total used {KubeSizeConverter.convertBytesToReadableSize(totalUsedBytes)} | Total allocated {KubeSizeConverter.convertBytesToReadableSize(totalCapacityBytes)} - } - + {updatedVolumeUsage.length} App Volumes | Total used {KubeSizeConverter.convertBytesToReadableSize(totals.used)} / {KubeSizeConverter.convertBytesToReadableSize(totals.capacity)}{totals.incomplete ? ' (incomplete)' : ''} Project @@ -83,14 +66,15 @@ export default function AppVolumeMonitoring({ {updatedVolumeUsage.map((item) => ( - + {item.projectName} {item.appName} {item.mountPath} - = 90 ? 'red' : (item.usedPercentage >= 80 ? 'orange' : undefined)} /> -
{KubeSizeConverter.convertBytesToReadableSize(item.usedBytes)} / {KubeSizeConverter.convertBytesToReadableSize(item.capacityBytes)}
+ {item.status === 'available' && item.usedBytes !== undefined && item.capacityBytes !== undefined && item.availableBytes !== undefined ? <> + +
{KubeSizeConverter.convertBytesToReadableSize(item.usedBytes)} used / {KubeSizeConverter.convertBytesToReadableSize(item.capacityBytes)} | {KubeSizeConverter.convertBytesToReadableSize(item.availableBytes)} available
+ : Unavailable ({item.status})} diff --git a/src/app/monitoring/app-volumes-monitoring.unit.spec.ts b/src/app/monitoring/app-volumes-monitoring.unit.spec.ts new file mode 100644 index 00000000..764d93a1 --- /dev/null +++ b/src/app/monitoring/app-volumes-monitoring.unit.spec.ts @@ -0,0 +1,22 @@ +import React from 'react'; +import { cleanup, render, screen } from '@testing-library/react'; +import AppVolumeMonitoring from './app-volumes-monitoring'; +import { AppVolumeMonitoringUsageModel } from '@/shared/model/app-volume-monitoring-usage.model'; + +afterEach(cleanup); + +it('counts shared volumes once and marks incomplete totals', () => { + const base: AppVolumeMonitoringUsageModel = { + projectId: 'p', projectName: 'Project', appName: 'App', appId: 'a', mountPath: '/data', + pvcName: 'pvc-1', status: 'available', usedBytes: 1024, capacityBytes: 4096, + availableBytes: 3072, isBaseVolume: true, + }; + render(React.createElement(AppVolumeMonitoring, { volumesUsage: [ + base, + { ...base, appId: 'b', appName: 'Other', isBaseVolume: false }, + { ...base, appId: 'c', pvcName: 'pvc-2', status: 'unavailable', usedBytes: undefined, capacityBytes: undefined, availableBytes: undefined }, + ] })); + expect(screen.getByText(/Total used.*incomplete/).textContent).toContain('1 KB'); + expect(screen.getByText(/Total used.*incomplete/).textContent).toContain('4 KB'); + expect(screen.getByText('Unavailable (unavailable)')).toBeTruthy(); +}); diff --git a/src/app/project/[projectId]/apps/details/volumes/actions.ts b/src/app/project/[projectId]/apps/details/volumes/actions.ts index 0ea634dd..1c1ccdf3 100644 --- a/src/app/project/[projectId]/apps/details/volumes/actions.ts +++ b/src/app/project/[projectId]/apps/details/volumes/actions.ts @@ -105,7 +105,7 @@ export const getPvcUsage = async (appId: string, projectId: string) => simpleAction(async () => { await isAuthorizedReadForApp(appId); return monitoringService.getPvcUsageFromApp(appId, projectId); - }) as Promise>; + }) as Promise>; export const getShareableVolumes = async (appId: string) => simpleAction(async () => { diff --git a/src/app/project/[projectId]/apps/details/volumes/volumes-list.tsx b/src/app/project/[projectId]/apps/details/volumes/volumes-list.tsx index 8772f630..2d4c4711 100644 --- a/src/app/project/[projectId]/apps/details/volumes/volumes-list.tsx +++ b/src/app/project/[projectId]/apps/details/volumes/volumes-list.tsx @@ -28,6 +28,8 @@ type AppVolumeWithCapacity = (AppVolume & { usedBytes?: number; capacityBytes?: number; usedPercentage?: number; + availableBytes?: number; + usageStatus?: string; }); export default function StorageList({ app, readonly, storageClasses }: { @@ -49,8 +51,10 @@ export default function StorageList({ app, readonly, storageClasses }: { const volume = response.data.find(x => x.pvcName === KubeObjectNameUtils.toPvcName(item.sharedVolumeId || item.id)); if (volume) { item.usedBytes = volume.usedBytes; - item.capacityBytes = KubeSizeConverter.fromMegabytesToBytes(item.size); - item.usedPercentage = Math.round(volume.usedBytes / item.capacityBytes * 100); + item.capacityBytes = volume.capacityBytes; + item.availableBytes = volume.availableBytes; + item.usageStatus = volume.status; + item.usedPercentage = volume.usedBytes !== undefined && volume.capacityBytes ? Math.round(volume.usedBytes / volume.capacityBytes * 100) : undefined; } } setVolumesWithStorage(mappedVolumeData); @@ -171,13 +175,13 @@ export default function StorageList({ app, readonly, storageClasses }: { {volume.containerMountPath} {volume.size} MB - {volume.usedPercentage && <> + {volume.usedPercentage !== undefined && volume.usedBytes !== undefined && volume.capacityBytes !== undefined && volume.availableBytes !== undefined ? <> = 90 ? 'red' : (volume.usedPercentage >= 80 ? 'orange' : undefined)} />
- {KubeSizeConverter.convertBytesToReadableSize(volume.usedBytes!)} used ({volume.usedPercentage}%) + {KubeSizeConverter.convertBytesToReadableSize(volume.usedBytes)} used / {KubeSizeConverter.convertBytesToReadableSize(volume.capacityBytes)} ({volume.usedPercentage}%) | {KubeSizeConverter.convertBytesToReadableSize(volume.availableBytes)} available
- } + : Unavailable{volume.usageStatus ? ` (${volume.usageStatus})` : ""}} {volume.storageClassName?.replace('-', ' ')} {volume.accessMode} diff --git a/src/server/adapter/volume-summary.adapter.ts b/src/server/adapter/volume-summary.adapter.ts new file mode 100644 index 00000000..34082601 --- /dev/null +++ b/src/server/adapter/volume-summary.adapter.ts @@ -0,0 +1,29 @@ +import k3s from './kubernetes-api.adapter'; + +export interface VolumeFilesystemSample { + pvcRef?: { namespace?: string; name?: string }; + time?: string; + usedBytes?: number; + availableBytes?: number; + capacityBytes?: number; +} + +interface NodeSummary { + pods?: { volume?: VolumeFilesystemSample[] }[]; +} + +class VolumeSummaryAdapter { + async getSamples(): Promise { + const nodes = await k3s.core.listNode(); + const summaries = await Promise.allSettled(nodes.items.map(async node => { + if (!node.metadata?.name) return []; + const raw = await k3s.core.connectGetNodeProxyWithPath({ name: node.metadata.name, path: 'stats/summary' }); + const summary: NodeSummary = typeof raw === 'string' ? JSON.parse(raw) : raw; + return summary.pods?.flatMap(pod => pod.volume ?? []) ?? []; + })); + return summaries.flatMap(result => result.status === 'fulfilled' ? result.value : []); + } +} + +const volumeSummaryAdapter = new VolumeSummaryAdapter(); +export default volumeSummaryAdapter; diff --git a/src/server/services/monitoring.service.ts b/src/server/services/monitoring.service.ts index ce795dd8..f4ac3da8 100644 --- a/src/server/services/monitoring.service.ts +++ b/src/server/services/monitoring.service.ts @@ -5,7 +5,9 @@ import clusterService from "./cluster.service"; import { PodsResourceInfoModel } from "@/shared/model/pods-resource-info.model"; import { KubeSizeConverter } from "../../shared/utils/kubernetes-size-converter.utils"; import { AppVolumeMonitoringUsageModel } from "@/shared/model/app-volume-monitoring-usage.model"; -import longhornApiAdapter from "../adapter/longhorn-api.adapter"; +import volumeSummaryAdapter from "../adapter/volume-summary.adapter"; +import { supportsVolumeFilesystemMetrics } from "./volume-filesystem-drivers"; +import { VolumeFilesystemUsage } from "@/shared/model/app-volume-monitoring-usage.model"; import dataAccess from "../adapter/db.client"; import pvcService from "./pvc.service"; import { KubeObjectNameUtils } from "../utils/kube-object-name.utils"; @@ -14,63 +16,65 @@ import { AppMonitoringUsageModel } from "@/shared/model/app-monitoring-usage.mod class MonitorService { - async getAllAppVolumesUsage() { - const [longhornData, appVolumes, pvcs] = await Promise.all([ - longhornApiAdapter.getAllLonghornVolumes(), - dataAccess.client.appVolume.findMany({ - include: { - app: { - include: { - project: true - } - } - }, - orderBy: { - appId: 'asc' - } - }), - pvcService.getAllPvc() + private async getVolumeFilesystemUsage(volumes: { id: string; sharedVolumeId: string | null; projectId: string }[]): Promise> { + const [pvcs, persistentVolumes, samples] = await Promise.all([ + pvcService.getAllPvc().catch(() => []), + k3s.core.listPersistentVolume().then(result => result.items).catch(() => []), + volumeSummaryAdapter.getSamples().catch(() => []), ]); - - const appVolumesWithUsage: AppVolumeMonitoringUsageModel[] = []; - const volumeMap = new Map(appVolumes.map(volume => [volume.id, volume])); - const pvcByName = new Map(pvcs.map(pvc => [pvc.metadata?.name, pvc])); - const longhornVolumeByName = new Map(longhornData.map(volume => [volume.name, volume])); - - for (const appVolume of appVolumes) { - const sharedVolumeId = (appVolume as { sharedVolumeId?: string | null }).sharedVolumeId; - const baseVolumeId = sharedVolumeId ?? appVolume.id; - const baseVolume = volumeMap.get(baseVolumeId); - const pvc = pvcByName.get(KubeObjectNameUtils.toPvcName(baseVolumeId)); - if (!pvc) { + const pvcByKey = new Map(pvcs.map(pvc => [`${pvc.metadata?.namespace}/${pvc.metadata?.name}`, pvc])); + const pvByName = new Map(persistentVolumes.map(pv => [pv.metadata?.name, pv])); + const result = new Map(); + const now = Date.now(); + for (const volume of volumes) { + const pvcName = KubeObjectNameUtils.toPvcName(volume.sharedVolumeId || volume.id); + const key = `${volume.projectId}/${pvcName}`; + if (result.has(key)) continue; + const pvc = pvcByKey.get(key); + const pv = pvc?.spec?.volumeName ? pvByName.get(pvc.spec.volumeName) : undefined; + if (!pv) { + result.set(key, { pvcName, status: 'unavailable' }); continue; } - const volumeName = pvc.spec?.volumeName; - const longhornVolume = volumeName ? longhornVolumeByName.get(volumeName) : undefined; - if (!longhornVolume) { + if (!supportsVolumeFilesystemMetrics(pv)) { + result.set(key, { pvcName, status: 'unsupported' }); continue; } - - appVolumesWithUsage.push({ - projectId: appVolume.app.projectId, - projectName: appVolume.app.project.name, - appName: appVolume.app.name, - appId: appVolume.appId, - mountPath: appVolume.containerMountPath, - usedBytes: longhornVolume.actualSizeBytes, - capacityBytes: KubeSizeConverter.fromMegabytesToBytes(baseVolume?.size ?? appVolume.size), - isBaseVolume: !sharedVolumeId - }); + const candidates = samples.filter(sample => sample.pvcRef?.namespace === volume.projectId && sample.pvcRef.name === pvcName) + .filter(sample => { + const values = [sample.usedBytes, sample.availableBytes, sample.capacityBytes]; + return values.every(value => typeof value === 'number' && Number.isFinite(value) && value >= 0) + && !!sample.capacityBytes && Number.isFinite(Date.parse(sample.time ?? '')); + }).sort ( (a, b) => Date.parse(b.time!) - Date.parse(a.time!)); + const sample = candidates[0]; + const age = sample ? now - Date.parse(sample.time!) : Infinity; + if (!sample || age > 5 * 60 * 1000 || age < -60 * 1000) { + result.set(key, { pvcName, status: sample ? 'stale' : 'unavailable' }); + continue; + } + result.set(key, { pvcName, status: 'available', usedBytes: sample.usedBytes, availableBytes: sample.availableBytes, capacityBytes: sample.capacityBytes, measuredAt: sample.time }); } + return result; + } - // sort appVolumesWithUsage first by projectName (asc) then by appName - appVolumesWithUsage.sort((a, b) => { - if (a.projectName === b.projectName) { - return a.appName.localeCompare(b.appName); - } - return a.projectName.localeCompare(b.projectName); + async getAllAppVolumesUsage(): Promise { + const appVolumes = await dataAccess.client.appVolume.findMany({ + include: { app: { include: { project: true } } }, + orderBy: { appId: 'asc' }, }); - return appVolumesWithUsage; + const usages = await this.getVolumeFilesystemUsage(appVolumes.map(volume => ({ ...volume, projectId: volume.app.projectId }))); + return appVolumes.map(volume => { + const pvcName = KubeObjectNameUtils.toPvcName(volume.sharedVolumeId || volume.id); + return { + ...usages.get(`${volume.app.projectId}/${pvcName}`)!, + projectId: volume.app.projectId, + projectName: volume.app.project.name, + appName: volume.app.name, + appId: volume.appId, + mountPath: volume.containerMountPath, + isBaseVolume: !volume.sharedVolumeId, + }; + }).sort((a, b) => a.projectName.localeCompare(b.projectName) || a.appName.localeCompare(b.appName)); } async getMonitoringForAllApps() { @@ -185,34 +189,13 @@ class MonitorService { return totalResourcesNodes; } - async getPvcUsageFromApp(appId: string, projectId: string): Promise> { - const appVolumes = await dataAccess.client.appVolume.findMany({ - where: { - appId - }, - select: { - id: true, - sharedVolumeId: true - } + async getPvcUsageFromApp(appId: string, projectId: string): Promise { + const volumes = await dataAccess.client.appVolume.findMany({ + where: { appId }, + select: { id: true, sharedVolumeId: true }, }); - if (appVolumes.length === 0) { - return []; - } - const baseVolumeIds = Array.from(new Set(appVolumes.map(volume => (volume as { sharedVolumeId?: string | null }).sharedVolumeId || volume.id))); - const pvcNames = new Set(baseVolumeIds.map(id => KubeObjectNameUtils.toPvcName(id))); - const pvcFromProject = await k3s.core.listNamespacedPersistentVolumeClaim({ namespace: projectId }); - const pvcUsageData: Array<{ pvcName: string, usedBytes: number }> = []; - - for (const pvc of pvcFromProject.items) { - const pvcName = pvc.metadata?.name; - const volumeName = pvc.spec?.volumeName; - - if (pvcName && volumeName && pvcNames.has(pvcName as `pvc-${string}`)) { - const usedBytes = await longhornApiAdapter.getLonghornVolume(volumeName); - pvcUsageData.push({ pvcName, usedBytes }); - } - } - return pvcUsageData; + const usages = await this.getVolumeFilesystemUsage(volumes.map(volume => ({ ...volume, projectId }))); + return Array.from(usages.values()); } } diff --git a/src/server/services/monitoring.service.unit.spec.ts b/src/server/services/monitoring.service.unit.spec.ts index 697afb6e..9af55b3d 100644 --- a/src/server/services/monitoring.service.unit.spec.ts +++ b/src/server/services/monitoring.service.unit.spec.ts @@ -14,7 +14,7 @@ vi.mock('@kubernetes/client-node', async (importOriginal) => { vi.mock('@/server/adapter/kubernetes-api.adapter', () => ({ default: { - core: {}, + core: { listNode: vi.fn(), listPersistentVolume: vi.fn(), connectGetNodeProxyWithPath: vi.fn() }, getKubeConfig: vi.fn(() => ({})), }, })); @@ -29,13 +29,6 @@ vi.mock('@/server/adapter/db.client', () => ({ }, })); -vi.mock('@/server/adapter/longhorn-api.adapter', () => ({ - default: { - getAllLonghornVolumes: vi.fn(), - getLonghornVolume: vi.fn(), - }, -})); - vi.mock('@/server/services/project.service', () => ({ default: { getAll: vi.fn(), @@ -60,8 +53,8 @@ vi.mock('@/server/services/standalone-services/standalone-pod.service', () => ({ }, })); +import k3s from '@/server/adapter/kubernetes-api.adapter'; import dataAccess from '@/server/adapter/db.client'; -import longhornApiAdapter from '@/server/adapter/longhorn-api.adapter'; import projectService from '@/server/services/project.service'; import clusterService from '@/server/services/cluster.service'; import pvcService from '@/server/services/pvc.service'; @@ -70,7 +63,6 @@ import monitoringService from './monitoring.service'; const mockedTopPods = vi.mocked(k8sMocks.topPods); const mockedFindMany = vi.mocked(dataAccess.client.appVolume.findMany); -const mockedGetAllLonghornVolumes = vi.mocked(longhornApiAdapter.getAllLonghornVolumes); const mockedProjectGetAll = vi.mocked(projectService.getAll); const mockedGetNodeInfo = vi.mocked(clusterService.getNodeInfo); const mockedGetAllPvc = vi.mocked(pvcService.getAllPvc); @@ -116,10 +108,6 @@ function appVolume(id: string, appId: string, size: number, sharedVolumeId: stri }; } -function pvc(name: string, volumeName: string) { - return { metadata: { name }, spec: { volumeName } }; -} - describe('monitoringService.getMonitoringForAllApps', () => { beforeEach(() => { vi.clearAllMocks(); @@ -166,69 +154,63 @@ describe('monitoringService.getMonitoringForAllApps', () => { }); describe('monitoringService.getAllAppVolumesUsage', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it('resolves the correct pvc and longhorn volume for each app volume', async () => { - mockedGetAllLonghornVolumes.mockResolvedValue([ - { name: 'lh-1', actualSizeBytes: 111, sizeBytes: 1000 }, - { name: 'lh-2', actualSizeBytes: 222, sizeBytes: 2000 }, - ]); - mockedFindMany.mockResolvedValue([ - appVolume('vol-1', 'app-1', 1024, null), - appVolume('vol-2', 'app-2', 2048, null), - ] as never); - mockedGetAllPvc.mockResolvedValue([ - pvc('pvc-vol-1', 'lh-1'), - pvc('pvc-vol-2', 'lh-2'), - ] as never); + it('reports fresh Longhorn filesystem usage for the configured volume', async () => { + mockedFindMany.mockResolvedValue([appVolume('vol-1', 'app-1', 1024, null)] as never); + mockedGetAllPvc.mockResolvedValue([{ metadata: { name: 'pvc-vol-1', namespace: 'proj-1' }, spec: { volumeName: 'pv-1' } }] as never); + vi.mocked(k3s.core.listPersistentVolume).mockResolvedValue({ items: [{ metadata: { name: 'pv-1' }, spec: { csi: { driver: 'driver.longhorn.io' } } }] } as never); + vi.mocked(k3s.core.listNode).mockResolvedValue({ items: [{ metadata: { name: 'node-1' } }] } as never); + vi.mocked(k3s.core.connectGetNodeProxyWithPath).mockResolvedValue(JSON.stringify({ pods: [{ podRef: { namespace: 'proj-1' }, volume: [{ pvcRef: { namespace: 'proj-1', name: 'pvc-vol-1' }, time: new Date().toISOString(), usedBytes: 111, availableBytes: 889, capacityBytes: 1000 }] }] })); const result = await monitoringService.getAllAppVolumesUsage(); - - expect(result).toHaveLength(2); - expect(result.find((volume) => volume.appId === 'app-1')).toMatchObject({ - usedBytes: 111, - capacityBytes: 1024 * 1024 * 1024, - isBaseVolume: true, - }); - expect(result.find((volume) => volume.appId === 'app-2')).toMatchObject({ - usedBytes: 222, - capacityBytes: 2048 * 1024 * 1024, - isBaseVolume: true, - }); + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ usedBytes: 111, availableBytes: 889, capacityBytes: 1000, status: 'available' }); }); - - it('derives a shared volume from its base volume and longhorn counterpart', async () => { - mockedGetAllLonghornVolumes.mockResolvedValue([ - { name: 'lh-1', actualSizeBytes: 111, sizeBytes: 1000 }, - ]); + it('keeps shared and unavailable volumes while using fresh samples across nodes', async () => { mockedFindMany.mockResolvedValue([ - appVolume('vol-1', 'app-1', 1024, null), - appVolume('vol-3', 'app-3', 9999, 'vol-1'), + appVolume('vol-1', 'app-1', 1, null), + appVolume('vol-2', 'app-2', 1, 'vol-1'), + appVolume('vol-3', 'app-3', 1, null), + appVolume('vol-4', 'app-4', 1, null), + appVolume('vol-5', 'app-5', 1, null), ] as never); mockedGetAllPvc.mockResolvedValue([ - pvc('pvc-vol-1', 'lh-1'), + { metadata: { namespace: 'proj-1', name: 'pvc-vol-1' }, spec: { volumeName: 'pv-1' } }, + { metadata: { namespace: 'proj-1', name: 'pvc-vol-3' }, spec: { volumeName: 'pv-3' } }, + { metadata: { namespace: 'proj-1', name: 'pvc-vol-4' }, spec: { volumeName: 'pv-4' } }, ] as never); - - const result = await monitoringService.getAllAppVolumesUsage(); - - expect(result.find((volume) => volume.appId === 'app-3')).toMatchObject({ - usedBytes: 111, - capacityBytes: 1024 * 1024 * 1024, - isBaseVolume: false, + vi.mocked(k3s.core.listPersistentVolume).mockResolvedValue({ items: [ + { metadata: { name: 'pv-1' }, spec: { csi: { driver: 'driver.longhorn.io' } } }, + { metadata: { name: 'pv-3' }, spec: { csi: { driver: 'driver.longhorn.io' } } }, + { metadata: { name: 'pv-4' }, spec: { csi: { driver: 'rancher.io/local-path' } } }, + ] } as never); + vi.mocked(k3s.core.listNode).mockResolvedValue({ items: [{ metadata: { name: 'broken' } }, { metadata: { name: 'healthy' } }] } as never); + vi.mocked(k3s.core.connectGetNodeProxyWithPath).mockImplementation(async ({ name }) => { + if (name === 'broken') throw new Error('node unavailable'); + return JSON.stringify({ pods: [{ volume: [ + { pvcRef: { namespace: 'proj-1', name: 'pvc-vol-1' }, time: new Date().toISOString(), usedBytes: 0, availableBytes: 1000, capacityBytes: 1000 }, + { pvcRef: { namespace: 'proj-1', name: 'pvc-vol-3' }, time: new Date(Date.now() - 600000).toISOString(), usedBytes: 500, availableBytes: 500, capacityBytes: 1000 }, + { pvcRef: { namespace: 'proj-1', name: 'pvc-vol-4' }, time: new Date().toISOString(), usedBytes: 500, availableBytes: 500, capacityBytes: 1000 }, + ] }] }); }); - }); - - it('skips volumes without a matching pvc or longhorn volume', async () => { - mockedGetAllLonghornVolumes.mockResolvedValue([]); - mockedFindMany.mockResolvedValue([ - appVolume('vol-1', 'app-1', 1024, null), - ] as never); - mockedGetAllPvc.mockResolvedValue([] as never); - const result = await monitoringService.getAllAppVolumesUsage(); + expect(result).toHaveLength(5); + expect(result.find(v => v.appId === 'app-1')).toMatchObject({ status: 'available', usedBytes: 0 }); + expect(result.find(v => v.appId === 'app-2')).toMatchObject({ status: 'available', usedBytes: 0, pvcName: 'pvc-vol-1' }); + expect(result.find(v => v.appId === 'app-3')).toMatchObject({ status: 'stale' }); + expect(result.find(v => v.appId === 'app-4')).toMatchObject({ status: 'unsupported' }); + expect(result.find(v => v.appId === 'app-5')).toMatchObject({ status: 'unavailable' }); + }); - expect(result).toHaveLength(0); + it('treats missing fields and zero capacity as unavailable', async () => { + mockedFindMany.mockResolvedValue([appVolume('vol-1', 'app-1', 1, null)] as never); + mockedGetAllPvc.mockResolvedValue([{ metadata: { namespace: 'proj-1', name: 'pvc-vol-1' }, spec: { volumeName: 'pv-1' } }] as never); + vi.mocked(k3s.core.listPersistentVolume).mockResolvedValue({ items: [{ metadata: { name: 'pv-1' }, spec: { csi: { driver: 'driver.longhorn.io' } } }] } as never); + vi.mocked(k3s.core.listNode).mockResolvedValue({ items: [{ metadata: { name: 'node-1' } }] } as never); + vi.mocked(k3s.core.connectGetNodeProxyWithPath).mockResolvedValue(JSON.stringify({ pods: [{ volume: [ + { pvcRef: { namespace: 'proj-1', name: 'pvc-vol-1' }, time: new Date().toISOString(), usedBytes: 0, capacityBytes: 100 }, + { pvcRef: { namespace: 'proj-1', name: 'pvc-vol-1' }, time: new Date().toISOString(), usedBytes: 0, availableBytes: 0, capacityBytes: 0 }, + ] }] })); + expect((await monitoringService.getAllAppVolumesUsage())[0]).toMatchObject({ status: 'unavailable' }); }); + }); diff --git a/src/server/services/volume-filesystem-drivers.ts b/src/server/services/volume-filesystem-drivers.ts new file mode 100644 index 00000000..ab290fb9 --- /dev/null +++ b/src/server/services/volume-filesystem-drivers.ts @@ -0,0 +1,13 @@ +import { V1PersistentVolume } from '@kubernetes/client-node'; + +interface VolumeFilesystemDriver { + supports(volume: V1PersistentVolume): boolean; +} + +const drivers: VolumeFilesystemDriver[] = [ + { supports: volume => volume.spec?.csi?.driver === 'driver.longhorn.io' }, +]; + +export function supportsVolumeFilesystemMetrics(volume: V1PersistentVolume): boolean { + return drivers.some(driver => driver.supports(volume)); +} diff --git a/src/shared/model/app-volume-monitoring-usage.model.ts b/src/shared/model/app-volume-monitoring-usage.model.ts index 418036df..738a24f7 100644 --- a/src/shared/model/app-volume-monitoring-usage.model.ts +++ b/src/shared/model/app-volume-monitoring-usage.model.ts @@ -1,10 +1,19 @@ -export interface AppVolumeMonitoringUsageModel { - projectId: string, - projectName: string, - appName: string, - appId: string, - mountPath: string, - usedBytes: number, - capacityBytes: number, - isBaseVolume: boolean +export type VolumeUsageStatus = 'available' | 'unavailable' | 'stale' | 'unsupported'; + +export interface VolumeFilesystemUsage { + pvcName: string; + status: VolumeUsageStatus; + usedBytes?: number; + availableBytes?: number; + capacityBytes?: number; + measuredAt?: string; +} + +export interface AppVolumeMonitoringUsageModel extends VolumeFilesystemUsage { + projectId: string; + projectName: string; + appName: string; + appId: string; + mountPath: string; + isBaseVolume: boolean; } From 181af6cebfe02bfb1e682448e90c85114523817f Mon Sep 17 00:00:00 2001 From: biersoeckli Date: Sun, 8 Nov 2026 12:26:43 +0000 Subject: [PATCH 02/11] fix: address filesystem monitoring review --- src/app/monitoring/app-volumes-monitoring.tsx | 21 ++++++-- .../app-volumes-monitoring.unit.spec.ts | 1 + src/server/adapter/kubernetes-api.adapter.ts | 33 +++++++++++++ src/server/adapter/volume-summary.adapter.ts | 7 ++- src/server/services/monitoring.service.ts | 25 +++++++--- .../services/monitoring.service.unit.spec.ts | 48 ++++++++++++++++--- 6 files changed, 116 insertions(+), 19 deletions(-) diff --git a/src/app/monitoring/app-volumes-monitoring.tsx b/src/app/monitoring/app-volumes-monitoring.tsx index 6855ea38..4c0ba4b5 100644 --- a/src/app/monitoring/app-volumes-monitoring.tsx +++ b/src/app/monitoring/app-volumes-monitoring.tsx @@ -71,10 +71,9 @@ export default function AppVolumeMonitoring({ {item.appName} {item.mountPath} - {item.status === 'available' && item.usedBytes !== undefined && item.capacityBytes !== undefined && item.availableBytes !== undefined ? <> - -
{KubeSizeConverter.convertBytesToReadableSize(item.usedBytes)} used / {KubeSizeConverter.convertBytesToReadableSize(item.capacityBytes)} | {KubeSizeConverter.convertBytesToReadableSize(item.availableBytes)} available
- : Unavailable ({item.status})} + {item.status === 'available' && item.usedBytes !== undefined && item.capacityBytes !== undefined && item.capacityBytes > 0 && item.availableBytes !== undefined ? ( + + ) : Unavailable ({item.status})} @@ -91,3 +90,17 @@ export default function AppVolumeMonitoring({ ); } + +function VolumeUsage({ usedBytes, capacityBytes, availableBytes }: { + usedBytes: number; + capacityBytes: number; + availableBytes: number; +}) { + const percentage = Math.round(usedBytes / capacityBytes * 100); + return <> + = 90 ? 'red' : percentage >= 80 ? 'orange' : undefined} /> +
+ {KubeSizeConverter.convertBytesToReadableSize(usedBytes)} used ({percentage}%) / {KubeSizeConverter.convertBytesToReadableSize(capacityBytes)} | {KubeSizeConverter.convertBytesToReadableSize(availableBytes)} available +
+ ; +} diff --git a/src/app/monitoring/app-volumes-monitoring.unit.spec.ts b/src/app/monitoring/app-volumes-monitoring.unit.spec.ts index 764d93a1..514d4882 100644 --- a/src/app/monitoring/app-volumes-monitoring.unit.spec.ts +++ b/src/app/monitoring/app-volumes-monitoring.unit.spec.ts @@ -19,4 +19,5 @@ it('counts shared volumes once and marks incomplete totals', () => { expect(screen.getByText(/Total used.*incomplete/).textContent).toContain('1 KB'); expect(screen.getByText(/Total used.*incomplete/).textContent).toContain('4 KB'); expect(screen.getByText('Unavailable (unavailable)')).toBeTruthy(); + expect(screen.getAllByText(/25%/)).toHaveLength(2); }); diff --git a/src/server/adapter/kubernetes-api.adapter.ts b/src/server/adapter/kubernetes-api.adapter.ts index 46f4465c..5252c46a 100644 --- a/src/server/adapter/kubernetes-api.adapter.ts +++ b/src/server/adapter/kubernetes-api.adapter.ts @@ -1,6 +1,8 @@ import { ServiceException } from '@/shared/model/service.exception.model'; import * as k8s from '@kubernetes/client-node'; import { ConfigurationOptions, PatchStrategy } from '@kubernetes/client-node'; +import * as http from 'node:http'; +import * as https from 'node:https'; export function kubernetesPatchOptions(contentType: PatchStrategy): ConfigurationOptions { return { @@ -100,6 +102,37 @@ export class K3sApiAdapter { return new k8s.Metrics(this.getKubeConfig()); } + /** Read the fixed kubelet summary endpoint through the Kubernetes API server. */ + async getNodeSummary(nodeName: string): Promise { + const config = this.getKubeConfig(); + const cluster = config.getCurrentCluster(); + if (!cluster) throw new Error('No Kubernetes cluster configured'); + const url = new URL(cluster.server); + url.pathname = `${url.pathname.replace(/\/$/, '')}/api/v1/nodes/${encodeURIComponent(nodeName)}/proxy/stats/summary`; + const options: https.RequestOptions = { method: 'GET', headers: { Accept: 'application/json' } }; + await config.applyToHTTPSOptions(options); + const transport = url.protocol === 'https:' ? https : http; + return new Promise((resolve, reject) => { + const request = transport.request(url, options, response => { + if (response.statusCode !== 200) { + response.resume(); + reject(new Error(`Node summary returned HTTP ${response.statusCode}`)); + return; + } + let body = ''; + response.setEncoding('utf8'); + response.on('data', chunk => { body += chunk; }); + response.on('error', reject); + response.on('end', () => { + try { resolve(JSON.parse(body)); } catch (error) { reject(error); } + }); + }); + request.setTimeout(10000, () => request.destroy(new Error('Node summary timed out'))); + request.on('error', reject); + request.end(); + }); + } + /** * Applies a single Kubernetes resource to the cluster * @param kc KubeConfig instance diff --git a/src/server/adapter/volume-summary.adapter.ts b/src/server/adapter/volume-summary.adapter.ts index 34082601..47f730f1 100644 --- a/src/server/adapter/volume-summary.adapter.ts +++ b/src/server/adapter/volume-summary.adapter.ts @@ -13,12 +13,15 @@ interface NodeSummary { } class VolumeSummaryAdapter { + async getPersistentVolumes() { + return (await k3s.core.listPersistentVolume()).items; + } + async getSamples(): Promise { const nodes = await k3s.core.listNode(); const summaries = await Promise.allSettled(nodes.items.map(async node => { if (!node.metadata?.name) return []; - const raw = await k3s.core.connectGetNodeProxyWithPath({ name: node.metadata.name, path: 'stats/summary' }); - const summary: NodeSummary = typeof raw === 'string' ? JSON.parse(raw) : raw; + const summary = await k3s.getNodeSummary(node.metadata.name) as NodeSummary; return summary.pods?.flatMap(pod => pod.volume ?? []) ?? []; })); return summaries.flatMap(result => result.status === 'fulfilled' ? result.value : []); diff --git a/src/server/services/monitoring.service.ts b/src/server/services/monitoring.service.ts index f4ac3da8..e21c694c 100644 --- a/src/server/services/monitoring.service.ts +++ b/src/server/services/monitoring.service.ts @@ -16,10 +16,12 @@ import { AppMonitoringUsageModel } from "@/shared/model/app-monitoring-usage.mod class MonitorService { - private async getVolumeFilesystemUsage(volumes: { id: string; sharedVolumeId: string | null; projectId: string }[]): Promise> { + private async getVolumeFilesystemUsage( + volumes: { id: string; sharedVolumeId: string | null; projectId: string }[], + ): Promise> { const [pvcs, persistentVolumes, samples] = await Promise.all([ pvcService.getAllPvc().catch(() => []), - k3s.core.listPersistentVolume().then(result => result.items).catch(() => []), + volumeSummaryAdapter.getPersistentVolumes().catch(() => []), volumeSummaryAdapter.getSamples().catch(() => []), ]); const pvcByKey = new Map(pvcs.map(pvc => [`${pvc.metadata?.namespace}/${pvc.metadata?.name}`, pvc])); @@ -40,19 +42,28 @@ class MonitorService { result.set(key, { pvcName, status: 'unsupported' }); continue; } - const candidates = samples.filter(sample => sample.pvcRef?.namespace === volume.projectId && sample.pvcRef.name === pvcName) + const candidates = samples + .filter(sample => sample.pvcRef?.namespace === volume.projectId && sample.pvcRef.name === pvcName) .filter(sample => { const values = [sample.usedBytes, sample.availableBytes, sample.capacityBytes]; return values.every(value => typeof value === 'number' && Number.isFinite(value) && value >= 0) && !!sample.capacityBytes && Number.isFinite(Date.parse(sample.time ?? '')); - }).sort ( (a, b) => Date.parse(b.time!) - Date.parse(a.time!)); + }) + .sort((a, b) => Date.parse(b.time!) - Date.parse(a.time!)); const sample = candidates[0]; const age = sample ? now - Date.parse(sample.time!) : Infinity; if (!sample || age > 5 * 60 * 1000 || age < -60 * 1000) { result.set(key, { pvcName, status: sample ? 'stale' : 'unavailable' }); continue; } - result.set(key, { pvcName, status: 'available', usedBytes: sample.usedBytes, availableBytes: sample.availableBytes, capacityBytes: sample.capacityBytes, measuredAt: sample.time }); + result.set(key, { + pvcName, + status: 'available', + usedBytes: sample.usedBytes, + availableBytes: sample.availableBytes, + capacityBytes: sample.capacityBytes, + measuredAt: sample.time, + }); } return result; } @@ -62,7 +73,9 @@ class MonitorService { include: { app: { include: { project: true } } }, orderBy: { appId: 'asc' }, }); - const usages = await this.getVolumeFilesystemUsage(appVolumes.map(volume => ({ ...volume, projectId: volume.app.projectId }))); + const usages = await this.getVolumeFilesystemUsage( + appVolumes.map(volume => ({ ...volume, projectId: volume.app.projectId })), + ); return appVolumes.map(volume => { const pvcName = KubeObjectNameUtils.toPvcName(volume.sharedVolumeId || volume.id); return { diff --git a/src/server/services/monitoring.service.unit.spec.ts b/src/server/services/monitoring.service.unit.spec.ts index 9af55b3d..b625e9eb 100644 --- a/src/server/services/monitoring.service.unit.spec.ts +++ b/src/server/services/monitoring.service.unit.spec.ts @@ -14,7 +14,8 @@ vi.mock('@kubernetes/client-node', async (importOriginal) => { vi.mock('@/server/adapter/kubernetes-api.adapter', () => ({ default: { - core: { listNode: vi.fn(), listPersistentVolume: vi.fn(), connectGetNodeProxyWithPath: vi.fn() }, + core: { listNode: vi.fn(), listPersistentVolume: vi.fn() }, + getNodeSummary: vi.fn(), getKubeConfig: vi.fn(() => ({})), }, })); @@ -159,7 +160,7 @@ describe('monitoringService.getAllAppVolumesUsage', () => { mockedGetAllPvc.mockResolvedValue([{ metadata: { name: 'pvc-vol-1', namespace: 'proj-1' }, spec: { volumeName: 'pv-1' } }] as never); vi.mocked(k3s.core.listPersistentVolume).mockResolvedValue({ items: [{ metadata: { name: 'pv-1' }, spec: { csi: { driver: 'driver.longhorn.io' } } }] } as never); vi.mocked(k3s.core.listNode).mockResolvedValue({ items: [{ metadata: { name: 'node-1' } }] } as never); - vi.mocked(k3s.core.connectGetNodeProxyWithPath).mockResolvedValue(JSON.stringify({ pods: [{ podRef: { namespace: 'proj-1' }, volume: [{ pvcRef: { namespace: 'proj-1', name: 'pvc-vol-1' }, time: new Date().toISOString(), usedBytes: 111, availableBytes: 889, capacityBytes: 1000 }] }] })); + vi.mocked(k3s.getNodeSummary).mockResolvedValue({ pods: [{ podRef: { namespace: 'proj-1' }, volume: [{ pvcRef: { namespace: 'proj-1', name: 'pvc-vol-1' }, time: new Date().toISOString(), usedBytes: 111, availableBytes: 889, capacityBytes: 1000 }] }] }); const result = await monitoringService.getAllAppVolumesUsage(); expect(result).toHaveLength(1); @@ -184,13 +185,13 @@ describe('monitoringService.getAllAppVolumesUsage', () => { { metadata: { name: 'pv-4' }, spec: { csi: { driver: 'rancher.io/local-path' } } }, ] } as never); vi.mocked(k3s.core.listNode).mockResolvedValue({ items: [{ metadata: { name: 'broken' } }, { metadata: { name: 'healthy' } }] } as never); - vi.mocked(k3s.core.connectGetNodeProxyWithPath).mockImplementation(async ({ name }) => { + vi.mocked(k3s.getNodeSummary).mockImplementation(async (name) => { if (name === 'broken') throw new Error('node unavailable'); - return JSON.stringify({ pods: [{ volume: [ + return { pods: [{ volume: [ { pvcRef: { namespace: 'proj-1', name: 'pvc-vol-1' }, time: new Date().toISOString(), usedBytes: 0, availableBytes: 1000, capacityBytes: 1000 }, { pvcRef: { namespace: 'proj-1', name: 'pvc-vol-3' }, time: new Date(Date.now() - 600000).toISOString(), usedBytes: 500, availableBytes: 500, capacityBytes: 1000 }, { pvcRef: { namespace: 'proj-1', name: 'pvc-vol-4' }, time: new Date().toISOString(), usedBytes: 500, availableBytes: 500, capacityBytes: 1000 }, - ] }] }); + ] }] }; }); const result = await monitoringService.getAllAppVolumesUsage(); expect(result).toHaveLength(5); @@ -206,11 +207,44 @@ describe('monitoringService.getAllAppVolumesUsage', () => { mockedGetAllPvc.mockResolvedValue([{ metadata: { namespace: 'proj-1', name: 'pvc-vol-1' }, spec: { volumeName: 'pv-1' } }] as never); vi.mocked(k3s.core.listPersistentVolume).mockResolvedValue({ items: [{ metadata: { name: 'pv-1' }, spec: { csi: { driver: 'driver.longhorn.io' } } }] } as never); vi.mocked(k3s.core.listNode).mockResolvedValue({ items: [{ metadata: { name: 'node-1' } }] } as never); - vi.mocked(k3s.core.connectGetNodeProxyWithPath).mockResolvedValue(JSON.stringify({ pods: [{ volume: [ + vi.mocked(k3s.getNodeSummary).mockResolvedValue({ pods: [{ volume: [ { pvcRef: { namespace: 'proj-1', name: 'pvc-vol-1' }, time: new Date().toISOString(), usedBytes: 0, capacityBytes: 100 }, { pvcRef: { namespace: 'proj-1', name: 'pvc-vol-1' }, time: new Date().toISOString(), usedBytes: 0, availableBytes: 0, capacityBytes: 0 }, - ] }] })); + ] }] }); expect((await monitoringService.getAllAppVolumesUsage())[0]).toMatchObject({ status: 'unavailable' }); }); + it('reports unknown drivers and unmounted bound volumes without exposing host usage', async () => { + mockedFindMany.mockResolvedValue([ + appVolume('unknown', 'app-1', 1, null), + appVolume('unmounted', 'app-2', 1, null), + ] as never); + mockedGetAllPvc.mockResolvedValue([ + { metadata: { namespace: 'proj-1', name: 'pvc-unknown' }, spec: { volumeName: 'pv-unknown' } }, + { metadata: { namespace: 'proj-1', name: 'pvc-unmounted' }, spec: { volumeName: 'pv-unmounted' } }, + ] as never); + vi.mocked(k3s.core.listPersistentVolume).mockResolvedValue({ items: [ + { metadata: { name: 'pv-unknown' }, spec: { csi: { driver: 'example.org/unknown' } } }, + { metadata: { name: 'pv-unmounted' }, spec: { csi: { driver: 'driver.longhorn.io' } } }, + ] } as never); + vi.mocked(k3s.core.listNode).mockResolvedValue({ items: [] } as never); + const result = await monitoringService.getAllAppVolumesUsage(); + expect(result.find(v => v.appId === 'app-1')).toMatchObject({ status: 'unsupported' }); + expect(result.find(v => v.appId === 'app-2')).toMatchObject({ status: 'unavailable' }); + }); + + it('returns the underlying PVC measurement for an app shared volume', async () => { + mockedFindMany.mockResolvedValue([{ id: 'shared', sharedVolumeId: 'base' }] as never); + mockedGetAllPvc.mockResolvedValue([{ metadata: { namespace: 'proj-1', name: 'pvc-base' }, spec: { volumeName: 'pv-base' } }] as never); + vi.mocked(k3s.core.listPersistentVolume).mockResolvedValue({ items: [{ metadata: { name: 'pv-base' }, spec: { csi: { driver: 'driver.longhorn.io' } } }] } as never); + vi.mocked(k3s.core.listNode).mockResolvedValue({ items: [{ metadata: { name: 'node-1' } }] } as never); + vi.mocked(k3s.getNodeSummary).mockResolvedValue({ pods: [{ volume: [{ + pvcRef: { namespace: 'proj-1', name: 'pvc-base' }, time: new Date().toISOString(), + usedBytes: 90, availableBytes: 10, capacityBytes: 100, + }] }] }); + expect(await monitoringService.getPvcUsageFromApp('app-1', 'proj-1')).toEqual([ + expect.objectContaining({ pvcName: 'pvc-base', status: 'available', usedBytes: 90, availableBytes: 10, capacityBytes: 100 }), + ]); + }); + }); From 58d86db9a9fad4aa45eb77a99fd33f638ffbff59 Mon Sep 17 00:00:00 2001 From: biersoeckli Date: Sun, 8 Nov 2026 12:37:52 +0000 Subject: [PATCH 03/11] feat: implement filesystem usage metrics for App Volumes and update related components --- .../issues/01-storage-filesystem-usage.md | 54 +++++ GLOSSARY.md | 4 + .../app-volumes-monitoring.unit.spec.ts | 4 +- src/app/monitoring/disk-chart.tsx | 221 ++---------------- src/app/monitoring/monitoring-nodes.tsx | 88 +++---- .../volumes/agent-volume-edit-overlay.tsx | 2 +- .../details/volumes/agent-volumes-card.tsx | 4 +- .../volumes/shared-storage-edit-overlay.tsx | 4 +- .../details/volumes/storage-edit-overlay.tsx | 2 +- .../apps/details/volumes/volumes-list.tsx | 2 +- .../source-wizard/types.ts | 2 +- src/frontend/utils/format.utils.ts | 2 +- src/server/adapter/longhorn-api.adapter.ts | 52 ----- src/server/services/cluster.service.ts | 27 ++- .../services/cluster.service.unit.spec.ts | 43 ++++ src/shared/model/node-resource.model.ts | 3 +- .../utils/kubernetes-size-converter.utils.ts | 4 +- ...bernetes-size-converter.utils.unit.spec.ts | 8 +- 18 files changed, 180 insertions(+), 346 deletions(-) create mode 100644 .scratch/storage-filesystem-usage/issues/01-storage-filesystem-usage.md create mode 100644 src/server/services/cluster.service.unit.spec.ts diff --git a/.scratch/storage-filesystem-usage/issues/01-storage-filesystem-usage.md b/.scratch/storage-filesystem-usage/issues/01-storage-filesystem-usage.md new file mode 100644 index 00000000..64a33b0b --- /dev/null +++ b/.scratch/storage-filesystem-usage/issues/01-storage-filesystem-usage.md @@ -0,0 +1,54 @@ +Status: done + +# StorageClass-independent App Volume filesystem usage + +## Problem Statement + +As a QuickStack user, I want to see how full an App Volume's filesystem is. Existing screens use Longhorn actual size, which includes snapshot-related storage and does not describe filesystem usage. It also ties the feature to Longhorn. + +## Solution + +Read App Volume filesystem usage through the Kubernetes API and Kubelet Summary API. Show reliable filesystem usage in existing App Volume screens and the Monitoring overview, independent of the Longhorn API. Clearly show unavailable usage where a trustworthy measurement cannot be obtained. + +## User Stories + +1. As an App owner, I want to see my App Volume's used filesystem space, so that I can judge remaining space. +2. As an App owner, I want to see filesystem capacity and available space, so that the usage has context. +3. As an App owner, I want a percentage based on filesystem usage and capacity, so that I can recognize a nearly full volume. +4. As an App owner, I want the existing App Volume UI to use the new measurements, so that it no longer confuses snapshot storage with filesystem usage. +5. As an operator, I want the Monitoring overview to show App Volume usage, so that I can find full volumes across projects. +6. As an operator, I want a shared App Volume shown at each relevant App mount, so that its users can see its usage. +7. As an operator, I want shared volumes counted once in totals, so that totals do not double-count usage. +8. As a user, I want missing, stale, unsupported, and inaccessible measurements identified, so that they are not mistaken for zero usage. +9. As a user, I want valid usage from supported volumes even when other volumes cannot be measured, so that partial failures do not hide useful information. +10. As a maintainer, I want a driver-specific extension point, so that additional storage drivers can be supported after their metrics are verified. +11. As an operator, I want local-path and unverified drivers marked unavailable, so that host filesystem usage is not presented as exclusive PVC usage. +12. As an operator, I want volume measurements to work across cluster nodes, so that App placement does not hide metrics. + +## Implementation Decisions + +- Replace Longhorn-derived filesystem usage for all existing consumers with a shared monitoring-service interface backed by node Summary API data accessed through the Kubernetes API. Do not require direct Longhorn API or pod access for these measurements. Other Longhorn features are unaffected. +- Map Summary volume data by namespace and PVC reference, discover nodes and associate values with configured App Volumes, and resolve shared App Volumes to the underlying PVC. Deduplicate that identity for totals, including across pods and nodes. +- Introduce a common storage-driver interface with driver-specific implementations. Determine the effective driver from the bound persistent volume and its provisioner/driver metadata, not merely the StorageClass name. Support verified Longhorn filesystem semantics initially. Treat local-path, unknown and unverified drivers as unavailable; make it possible to add verified drivers later. +- Use reported filesystem used, available and capacity values with their measurement time. Do not substitute configured PVC capacity, Longhorn actual size, host-wide values, zero, or stale values as if they were a fresh filesystem measurement. Define and enforce a bounded freshness policy; communicate unavailable or stale data and preserve available results on partial failure. +- Include configured App Volumes even when unbound, unmounted, unsupported, or missing metrics. Handle zero or missing capacity safely. Sum unique volumes and identify incomplete totals rather than presenting them as complete. +- Account for the existing installation's broad node proxy access; avoid introducing broader access. Node proxy is privileged, so do not expose a generic proxy endpoint to users. +- Existing App Volume and Monitoring authorization applies. No Agent Volume monitoring is added. + +## Testing Decisions + +- Test observable behavior at the existing monitoring-service boundary using representative Kubernetes responses and database records; test UI behavior where necessary. Follow existing monitoring service unit-test and component-test patterns. +- Cover Longhorn metrics, multi-node lookup, app and shared-volume mapping, deduplication, unmounted/unbound volumes, local-path and unknown drivers, stale and missing fields, unavailable nodes, partial results and safe capacity calculations. +- Test externally visible results and error states, not internal helper call sequences. Validate against a real supported k3s cluster where available. + +## Out of Scope + +Historical metrics, Agent Volumes, supporting unverified storage drivers, Longhorn snapshot or replica storage accounting, alerting and export. + +## Further Notes + +The current cluster returned Longhorn filesystem metrics through the Summary API. A local-path example reported the host filesystem, and not all PVCs appeared in the response. The history issue builds on this feature. + +## Comments + +Implemented on `integration/storage-filesystem-usage` (commits `0a5666a0`, `181af6ce`). Targeted tests and lint pass; real-cluster validation remains outstanding. diff --git a/GLOSSARY.md b/GLOSSARY.md index 5b9f92cd..d9131eb4 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -202,6 +202,10 @@ _Avoid_: disk, storage, PVC when referring to the domain concept A **Volume** attached to an **App**, optionally shareable with other Apps in the same project. _Avoid_: app storage, app disk +**App Volume Filesystem Usage**: +The occupied and available space and the capacity of the filesystem of an **App Volume**. It excludes storage consumed by snapshots outside that filesystem. +_Avoid_: Longhorn actual size, physical replica usage + **Agent Volume**: A **Volume** attached to an **Agent**, without backup scheduling. An Agent Volume is not shareable with other Agents or Apps. Its **Volume Type** is a **Shared Agent Volume**, a **Per-Sandbox Agent Volume**, or a **Per-Custom-Tag Agent Volume**. _Avoid_: agent storage, agent disk diff --git a/src/app/monitoring/app-volumes-monitoring.unit.spec.ts b/src/app/monitoring/app-volumes-monitoring.unit.spec.ts index 514d4882..654aef0d 100644 --- a/src/app/monitoring/app-volumes-monitoring.unit.spec.ts +++ b/src/app/monitoring/app-volumes-monitoring.unit.spec.ts @@ -16,8 +16,8 @@ it('counts shared volumes once and marks incomplete totals', () => { { ...base, appId: 'b', appName: 'Other', isBaseVolume: false }, { ...base, appId: 'c', pvcName: 'pvc-2', status: 'unavailable', usedBytes: undefined, capacityBytes: undefined, availableBytes: undefined }, ] })); - expect(screen.getByText(/Total used.*incomplete/).textContent).toContain('1 KB'); - expect(screen.getByText(/Total used.*incomplete/).textContent).toContain('4 KB'); + expect(screen.getByText(/Total used.*incomplete/).textContent).toContain('1 KiB'); + expect(screen.getByText(/Total used.*incomplete/).textContent).toContain('4 KiB'); expect(screen.getByText('Unavailable (unavailable)')).toBeTruthy(); expect(screen.getAllByText(/25%/)).toHaveLength(2); }); diff --git a/src/app/monitoring/disk-chart.tsx b/src/app/monitoring/disk-chart.tsx index 25937aac..928d915e 100644 --- a/src/app/monitoring/disk-chart.tsx +++ b/src/app/monitoring/disk-chart.tsx @@ -1,217 +1,24 @@ 'use client'; -import { - Label, - PolarRadiusAxis, - RadialBar, - RadialBarChart, -} from 'recharts'; - -import { ChartConfig, ChartContainer, ChartTooltip, ChartTooltipContent } from '@/components/ui/chart'; import { NodeResourceModel } from '@/shared/model/node-resource.model'; import { KubeSizeConverter } from '@/shared/utils/kubernetes-size-converter.utils'; +import { Progress } from '@/components/ui/progress'; -export default function ChartDiskRessources({ - nodeRessource, -}: { - nodeRessource: NodeResourceModel; -}) { - - const diskUsed = nodeRessource.diskUsageAbsolut ?? 0; - const diskReserved = nodeRessource.diskUsageReserved ?? 0; - const diskCapacity = nodeRessource.diskUsageCapacity ?? 0; - const diskSchedulable = nodeRessource.diskSpaceSchedulable ?? Math.max(0, diskCapacity - diskUsed - diskReserved); - const diskUsedAndReserved = diskUsed + diskReserved; - const diskUsagePercent = diskCapacity > 0 ? (diskUsedAndReserved / diskCapacity) * 100 : 0; - - const chartData = [{ - diskUsed, - diskReserved, - diskSchedulable - }]; +export default function ChartDiskRessources({ nodeRessource }: { nodeRessource: NodeResourceModel }) { + const { diskUsageAbsolut: used, diskUsageAvailable: available, diskUsageCapacity: capacity } = nodeRessource; - const chartConfig = { - diskUsed: { - label: "Used", - color: "var(--chart-1)", - }, - diskReserved: { - label: "Reserved (free but not usable)", - color: "var(--chart-2)", - }, - diskSchedulable: { - label: "Schedulable", - color: "var(--muted)", - }, - } satisfies ChartConfig + if (used === undefined || available === undefined || capacity === undefined || capacity <= 0) { + return

Storage metrics unavailable

; + } return ( - - - { - // Convert the value from bytes to gigabytes - const formattedValue = KubeSizeConverter.convertBytesToReadableSize(value as number); - const config = typeof name === 'string' ? chartConfig[name as keyof typeof chartConfig] : undefined; - // Optionally, you can customize the label (name) here if needed - return
-
-
{config?.label ?? name}:
-
{formattedValue}
-
- }} />} - /> - - - - - - - +
+
+ {(used / capacity * 100).toFixed(0)}% used + {KubeSizeConverter.convertBytesToReadableSize(used)} / {KubeSizeConverter.convertBytesToReadableSize(capacity)} +
+ +

Available: {KubeSizeConverter.convertBytesToReadableSize(available)}

+
); } - - -/* - - - - - - - - - - - - - - - - -*/ diff --git a/src/app/monitoring/monitoring-nodes.tsx b/src/app/monitoring/monitoring-nodes.tsx index f63022ad..6e2f90c4 100644 --- a/src/app/monitoring/monitoring-nodes.tsx +++ b/src/app/monitoring/monitoring-nodes.tsx @@ -38,7 +38,6 @@ export default function ResourcesNodes({ }) { const getDiskUsageAbsolut = (node: NodeResourceModel) => node.diskUsageAbsolut ?? 0; - const getDiskUsageReserved = (node: NodeResourceModel) => node.diskUsageReserved ?? 0; const getDiskUsageCapacity = (node: NodeResourceModel) => node.diskUsageCapacity ?? 0; const toPercent = (used: number, capacity: number) => (capacity > 0 ? (used / capacity) * 100 : 0); @@ -53,7 +52,7 @@ export default function ResourcesNodes({ if (!updatedNodeRessources) return { cpuUsage: 0, cpuCapacity: 1, ramUsage: 0, ramCapacity: 1, - diskUsageAbsolut: 0, diskUsageReserved: 0, diskCapacity: 1 + diskUsageAbsolut: 0, diskCapacity: 1 }; return updatedNodeRessources.reduce((acc, node) => ({ @@ -62,12 +61,11 @@ export default function ResourcesNodes({ ramUsage: acc.ramUsage + node.ramUsage, ramCapacity: acc.ramCapacity + node.ramCapacity, diskUsageAbsolut: acc.diskUsageAbsolut + getDiskUsageAbsolut(node), - diskUsageReserved: acc.diskUsageReserved + getDiskUsageReserved(node), diskCapacity: acc.diskCapacity + getDiskUsageCapacity(node), }), { cpuUsage: 0, cpuCapacity: 0, ramUsage: 0, ramCapacity: 0, - diskUsageAbsolut: 0, diskUsageReserved: 0, diskCapacity: 0 + diskUsageAbsolut: 0, diskCapacity: 0 }); }, [updatedNodeRessources]); @@ -83,22 +81,7 @@ export default function ResourcesNodes({ color: "var(--chart-1)", }, free: { - label: "Free", - color: "var(--muted)", - }, - } satisfies ChartConfig; - - const storagePieChartConfig = { - used: { - label: "Used", - color: "var(--chart-1)", - }, - reserved: { - label: "Reserved", - color: "var(--chart-2)", - }, - free: { - label: "Free", + label: "Remaining", color: "var(--muted)", }, } satisfies ChartConfig; @@ -111,14 +94,6 @@ export default function ResourcesNodes({ ]; }; - const getStorageChartData = (used: number, reserved: number, capacity: number) => { - return [ - { status: 'used', value: used, fill: "var(--chart-1)" }, - { status: 'reserved', value: reserved, fill: "var(--chart-2)" }, - { status: 'free', value: Math.max(0, capacity - used - reserved), fill: 'var(--color-free)' }, - ]; - }; - if (!updatedNodeRessources) { return } @@ -192,38 +167,29 @@ export default function ResourcesNodes({ Cluster Storage - Total Disk Usage + Node filesystem usage - - - { - if (value === clusterStats.diskUsageAbsolut) { - return KubeSizeConverter.convertBytesToReadableSize(clusterStats.diskUsageAbsolut) + ' (Used)'; - } - if (value === clusterStats.diskUsageReserved) { - return KubeSizeConverter.convertBytesToReadableSize(clusterStats.diskUsageReserved) + ' (Free but unusable)'; - } - return KubeSizeConverter.convertBytesToReadableSize(value as number) + ' (Free)'; - }} />} /> - - - - + {updatedNodeRessources.some(node => node.diskUsageCapacity === undefined) && ( +

Incomplete: some node filesystem metrics are unavailable.

+ )} + {clusterStats.diskCapacity > 0 ? ( + + + KubeSizeConverter.convertBytesToReadableSize(value as number)} />} /> + + + + + ) :

Storage metrics unavailable

} @@ -270,10 +236,10 @@ export default function ResourcesNodes({
{(() => { const diskUsed = getDiskUsageAbsolut(node); - const diskReserved = getDiskUsageReserved(node); const diskCapacity = getDiskUsageCapacity(node); - const diskUsedAndReserved = diskUsed + diskReserved; + const diskUsedAndReserved = diskUsed; const diskPercent = toPercent(diskUsedAndReserved, diskCapacity); + if (node.diskUsageCapacity === undefined) return Unavailable; return ( <>
@@ -472,7 +438,7 @@ function NodeDetailsSheet({ node }: { node: NodeResourceModel }) { y={(viewBox.cy || 0) + 30} className="fill-muted-foreground" > - {(node.ramUsage / (1024 * 1024 * 1024)).toFixed(2)} / {KubeSizeConverter.convertBytesToReadableSize(node.ramCapacity)} + {KubeSizeConverter.convertBytesToReadableSize(node.ramUsage)} / {KubeSizeConverter.convertBytesToReadableSize(node.ramCapacity)} ); diff --git a/src/app/project/[projectId]/agents/details/volumes/agent-volume-edit-overlay.tsx b/src/app/project/[projectId]/agents/details/volumes/agent-volume-edit-overlay.tsx index 83d6d52a..c348bf3a 100644 --- a/src/app/project/[projectId]/agents/details/volumes/agent-volume-edit-overlay.tsx +++ b/src/app/project/[projectId]/agents/details/volumes/agent-volume-edit-overlay.tsx @@ -111,7 +111,7 @@ export default function AgentVolumeEditOverlay({ name="size" render={({ field }) => ( - Size (MB) + Size (MiB) diff --git a/src/app/project/[projectId]/agents/details/volumes/agent-volumes-card.tsx b/src/app/project/[projectId]/agents/details/volumes/agent-volumes-card.tsx index af5137b7..1d67fd97 100644 --- a/src/app/project/[projectId]/agents/details/volumes/agent-volumes-card.tsx +++ b/src/app/project/[projectId]/agents/details/volumes/agent-volumes-card.tsx @@ -13,8 +13,8 @@ import { deleteAgentVolume } from "../configuration/actions"; import { Constants } from "@/shared/utils/constants"; function formatSize(mb: number): string { - if (mb >= 1024) return `${(mb / 1024).toFixed(1)} GB`; - return `${mb} MB`; + if (mb >= 1024) return `${(mb / 1024).toFixed(1)} GiB`; + return `${mb} MiB`; } function volumeTypeLabel(volumeType: string): string { diff --git a/src/app/project/[projectId]/apps/details/volumes/shared-storage-edit-overlay.tsx b/src/app/project/[projectId]/apps/details/volumes/shared-storage-edit-overlay.tsx index 22203239..ccdab001 100644 --- a/src/app/project/[projectId]/apps/details/volumes/shared-storage-edit-overlay.tsx +++ b/src/app/project/[projectId]/apps/details/volumes/shared-storage-edit-overlay.tsx @@ -133,7 +133,7 @@ export default function SharedStorageEditDialog({ app }: { label="Select Shared Volume" values={shareableVolumes.map(v => [ v.id, - `${v.app.name} - ${v.containerMountPath} (${v.size}MB)` + `${v.app.name} - ${v.containerMountPath} (${v.size}MiB)` ])} placeholder="Select volume to share..." /> @@ -153,7 +153,7 @@ export default function SharedStorageEditDialog({ app }: { />
-

Size: {form.watch("size")} MB (inherited from shared volume)

+

Size: {form.watch("size")} MiB (inherited from shared volume)

Storage Class: {form.watch("storageClassName")} (inherited from shared volume)

diff --git a/src/app/project/[projectId]/apps/details/volumes/storage-edit-overlay.tsx b/src/app/project/[projectId]/apps/details/volumes/storage-edit-overlay.tsx index 28900164..3da657f0 100644 --- a/src/app/project/[projectId]/apps/details/volumes/storage-edit-overlay.tsx +++ b/src/app/project/[projectId]/apps/details/volumes/storage-edit-overlay.tsx @@ -140,7 +140,7 @@ export default function StorageEditDialog({ volume, app, storageClasses }: { name="size" render={({ field }) => ( - Size in MB + Size in MiB diff --git a/src/app/project/[projectId]/apps/details/volumes/volumes-list.tsx b/src/app/project/[projectId]/apps/details/volumes/volumes-list.tsx index 2d4c4711..049a6b72 100644 --- a/src/app/project/[projectId]/apps/details/volumes/volumes-list.tsx +++ b/src/app/project/[projectId]/apps/details/volumes/volumes-list.tsx @@ -173,7 +173,7 @@ export default function StorageList({ app, readonly, storageClasses }: { {volumesWithStorage.map(volume => ( {volume.containerMountPath} - {volume.size} MB + {volume.size} MiB {volume.usedPercentage !== undefined && volume.usedBytes !== undefined && volume.capacityBytes !== undefined && volume.availableBytes !== undefined ? <> = { export const buildMethodLabels: Record = appBuildMethodLabels; -export const defaultDockerfilePath = '../../[projectId]/app-components/app-detail/general/app-source-wizard/Dockerfile'; +export const defaultDockerfilePath = './Dockerfile'; diff --git a/src/frontend/utils/format.utils.ts b/src/frontend/utils/format.utils.ts index fbeb4a95..0d19c4ae 100644 --- a/src/frontend/utils/format.utils.ts +++ b/src/frontend/utils/format.utils.ts @@ -54,7 +54,7 @@ export function formatTime(date: DateInput): string { export function formatBytes(bytes: number) { if (bytes === 0) return '0 B'; const k = 1024; - const sizes = ['B', 'KB', 'MB', 'GB']; + const sizes = ['B', 'KiB', 'MiB', 'GiB']; const i = Math.floor(Math.log(bytes) / Math.log(k)); return Math.round(bytes / Math.pow(k, i) * 100) / 100 + ' ' + sizes[i]; }; diff --git a/src/server/adapter/longhorn-api.adapter.ts b/src/server/adapter/longhorn-api.adapter.ts index 247e4221..736815d3 100644 --- a/src/server/adapter/longhorn-api.adapter.ts +++ b/src/server/adapter/longhorn-api.adapter.ts @@ -70,58 +70,6 @@ class LonghornApiAdapter { } - async getNodeStorageInfo(nodeName: String) { - const response = await fetch(`${this.longhornBaseUrl}/v1/nodes/${nodeName}`, { - cache: 'no-cache', - method: 'GET', - headers: { - - 'Accept': 'application/json', - 'Content-Type': 'application/json' - } - }); - - if (!response.ok) { - throw new Error(`HTTP-Error: ${response.status}`); - } - - const data = await response.json() as { - disks: { - [key: string]: { - storageMaximum: number, - storageAvailable: number, - storageReserved: number, - storageScheduled: number - } - } - }; - - if (!data.disks || typeof data.disks !== "object") { - throw new Error("Invalid node data: 'disks' property is missing or not an object."); - } - - let totalStorageMaximum = 0; - let totalStorageAvailable = 0; - let totalStorageReserved = 0; - Object.values(data.disks).forEach(disk => { - totalStorageMaximum += disk.storageMaximum; - totalStorageAvailable += disk.storageAvailable; - totalStorageReserved += disk.storageReserved; - }); - - // The available Storage is the total storage minus the reserved storage (which is not available for scheduling --> 30% of disk space) - const totalSchedulableStorage = totalStorageAvailable - totalStorageReserved; - - return { - totalStorageMaximum, - totalStorageAvailable, - totalSchedulableStorage, - totalStorageReserved - }; - - } - - async backupPvc(pvcName: string) { const snapshot = await this.createSnapshot(pvcName); await this.createBackup(pvcName, snapshot.id); diff --git a/src/server/services/cluster.service.ts b/src/server/services/cluster.service.ts index 49e5ca14..0de75ab6 100644 --- a/src/server/services/cluster.service.ts +++ b/src/server/services/cluster.service.ts @@ -4,9 +4,7 @@ import { NodeInfoModel } from "@/shared/model/node-info.model"; import { NodeResourceModel } from "@/shared/model/node-resource.model"; import { Tags } from "../utils/cache-tag-generator.utils"; import { revalidateTag, unstable_cache } from "next/cache"; -import longhornApiAdapter from "../adapter/longhorn-api.adapter"; import { KubeSizeConverter } from "../../shared/utils/kubernetes-size-converter.utils"; -import { CatchUtils } from "@/shared/utils/catch.utils"; import { StorageClassUtils } from "@/shared/utils/storage-class.utils"; class ClusterService { @@ -146,7 +144,23 @@ class ClusterService { nodeMetrics.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime()); const latestUsageItem = nodeMetrics[0]; - const diskInfo = await CatchUtils.resultOrUndefined(() => longhornApiAdapter.getNodeStorageInfo(node.Node.metadata?.name!)); + let filesystem: { usedBytes?: number; availableBytes?: number; capacityBytes?: number; time?: string } | undefined; + try { + const summary = await k3s.getNodeSummary(node.Node.metadata?.name!) as { + node?: { fs?: typeof filesystem }; + }; + const fs = summary.node?.fs; + const timestamp = Date.parse(fs?.time ?? ''); + if (fs && Number.isFinite(timestamp) && timestamp <= Date.now() && Date.now() - timestamp <= 5 * 60_000 && + typeof fs.usedBytes === 'number' && Number.isFinite(fs.usedBytes) && fs.usedBytes >= 0 && + typeof fs.availableBytes === 'number' && Number.isFinite(fs.availableBytes) && fs.availableBytes >= 0 && + typeof fs.capacityBytes === 'number' && Number.isFinite(fs.capacityBytes) && fs.capacityBytes > 0 && + fs.usedBytes <= fs.capacityBytes && fs.availableBytes <= fs.capacityBytes) { + filesystem = fs; + } + } catch { + // A node without a summary must not hide the other nodes' metrics. + } return { name: node.Node.metadata?.name!, @@ -154,10 +168,9 @@ class ClusterService { cpuCapacity: Number(node.CPU?.Capacity!), ramUsage: latestUsageItem.ramUsage, ramCapacity: Number(node.Memory?.Capacity!), - diskUsageAbsolut: diskInfo ? diskInfo.totalStorageMaximum - diskInfo.totalStorageAvailable : undefined, - diskUsageReserved: diskInfo?.totalStorageReserved, - diskUsageCapacity: diskInfo?.totalStorageMaximum, - diskSpaceSchedulable: diskInfo?.totalSchedulableStorage + diskUsageAbsolut: filesystem?.usedBytes, + diskUsageAvailable: filesystem?.availableBytes, + diskUsageCapacity: filesystem?.capacityBytes } })); } diff --git a/src/server/services/cluster.service.unit.spec.ts b/src/server/services/cluster.service.unit.spec.ts new file mode 100644 index 00000000..6239b82c --- /dev/null +++ b/src/server/services/cluster.service.unit.spec.ts @@ -0,0 +1,43 @@ + + +const mocks = vi.hoisted(() => ({ topNodes: vi.fn(), getNodeMetrics: vi.fn(), getNodeSummary: vi.fn() })); +vi.mock('@kubernetes/client-node', async (importOriginal) => ({ + ...await importOriginal(), + topNodes: mocks.topNodes, +})); +vi.mock('@/server/adapter/kubernetes-api.adapter', () => ({ + default: { core: {}, metrics: { getNodeMetrics: mocks.getNodeMetrics }, getNodeSummary: mocks.getNodeSummary }, + kubernetesPatchOptions: vi.fn(), +})); +import clusterService from './cluster.service'; + +describe('cluster node storage', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.topNodes.mockResolvedValue(['node-a', 'node-b'].map(name => ({ + Node: { metadata: { name } }, CPU: { Capacity: '2' }, Memory: { Capacity: '1000' }, + }))); + mocks.getNodeMetrics.mockResolvedValue({ items: ['node-a', 'node-b'].map(name => ({ + metadata: { name }, timestamp: new Date().toISOString(), usage: { cpu: '1000000n', memory: '100Mi' }, + })) }); + }); + + it('returns filesystem metrics and preserves other nodes when a summary fails', async () => { + mocks.getNodeSummary.mockImplementation(async (name: string) => { + if (name === 'node-b') throw new Error('unavailable'); + return { node: { fs: { time: new Date().toISOString(), usedBytes: 300, availableBytes: 600, capacityBytes: 1000 } } }; + }); + const nodes = await clusterService.getNodeResourceUsage(); + expect(nodes[0]).toMatchObject({ name: 'node-a', diskUsageAbsolut: 300, diskUsageAvailable: 600, diskUsageCapacity: 1000 }); + expect(nodes[1].diskUsageCapacity).toBeUndefined(); + }); + + it('does not present stale or incomplete filesystem measurements', async () => { + mocks.getNodeSummary.mockImplementation(async (name: string) => ({ node: { fs: name === 'node-a' + ? { time: '2000-01-01T00:00:00Z', usedBytes: 10, availableBytes: 90, capacityBytes: 100 } + : { time: new Date().toISOString(), usedBytes: 10, capacityBytes: 100 }, + } })); + const nodes = await clusterService.getNodeResourceUsage(); + expect(nodes.map(node => node.diskUsageCapacity)).toEqual([undefined, undefined]); + }); +}); diff --git a/src/shared/model/node-resource.model.ts b/src/shared/model/node-resource.model.ts index 11974619..c7fbc327 100644 --- a/src/shared/model/node-resource.model.ts +++ b/src/shared/model/node-resource.model.ts @@ -8,8 +8,7 @@ export const nodeResourceZodModel = z.object({ ramCapacity: z.number(), diskUsageAbsolut: z.number().optional(), diskUsageCapacity: z.number().optional(), - diskUsageReserved: z.number().optional(), - diskSpaceSchedulable: z.number().optional(), + diskUsageAvailable: z.number().optional(), }) export type NodeResourceModel = z.infer; \ No newline at end of file diff --git a/src/shared/utils/kubernetes-size-converter.utils.ts b/src/shared/utils/kubernetes-size-converter.utils.ts index 716c0235..d65b0160 100644 --- a/src/shared/utils/kubernetes-size-converter.utils.ts +++ b/src/shared/utils/kubernetes-size-converter.utils.ts @@ -11,14 +11,14 @@ export class KubeSizeConverter { /** * Converts a size in bytes to a human-readable format. - * eg. 1024 -> 1 KB, 1024 * 1024 -> 1 MB, etc. + * eg. 1024 -> 1 KiB, 1024 * 1024 -> 1 MiB, etc. */ static convertBytesToReadableSize(bytes: number, fractionDigits = 2, hideSize = false): string { if (isNaN(bytes) || bytes === 0) { return '0 B'; } - const sizes = ['B', 'KB', 'MB', 'GB', 'TB']; + const sizes = ['B', 'KiB', 'MiB', 'GiB', 'TiB']; const i = Math.floor(Math.log(bytes) / Math.log(1024)); return parseFloat((bytes / Math.pow(1024, i)).toFixed(fractionDigits)) + (hideSize ? '' : (' ' + sizes[i])); diff --git a/src/shared/utils/kubernetes-size-converter.utils.unit.spec.ts b/src/shared/utils/kubernetes-size-converter.utils.unit.spec.ts index 549186bd..5443f976 100644 --- a/src/shared/utils/kubernetes-size-converter.utils.unit.spec.ts +++ b/src/shared/utils/kubernetes-size-converter.utils.unit.spec.ts @@ -4,10 +4,10 @@ describe(KubeSizeConverter.name, () => { describe('convertBytesToReadableSize', () => { it('should convert bytes to human-readable format', () => { - expect(KubeSizeConverter.convertBytesToReadableSize(1024)).toBe('1 KB'); - expect(KubeSizeConverter.convertBytesToReadableSize(1024 * 1024)).toBe('1 MB'); - expect(KubeSizeConverter.convertBytesToReadableSize(1024 * 1024 * 1024)).toBe('1 GB'); - expect(KubeSizeConverter.convertBytesToReadableSize(1500)).toBe('1.46 KB'); + expect(KubeSizeConverter.convertBytesToReadableSize(1024)).toBe('1 KiB'); + expect(KubeSizeConverter.convertBytesToReadableSize(1024 * 1024)).toBe('1 MiB'); + expect(KubeSizeConverter.convertBytesToReadableSize(1024 * 1024 * 1024)).toBe('1 GiB'); + expect(KubeSizeConverter.convertBytesToReadableSize(1500)).toBe('1.46 KiB'); }); it('should handle zero bytes', () => { From 41d6178d11957db465652c29b8d906b5ef1f5c88 Mon Sep 17 00:00:00 2001 From: biersoeckli Date: Sun, 8 Nov 2026 12:53:42 +0000 Subject: [PATCH 04/11] feat: add app monitoring history --- .../migration.sql | 33 ++++++++ prisma/schema.prisma | 43 ++++++++++ ...app-monitoring-history.integration.spec.ts | 71 ++++++++++++++++ src/app/monitoring/actions.ts | 11 ++- .../monitoring/history-actions.unit.spec.ts | 16 ++++ src/app/monitoring/monitoring-client.tsx | 2 + .../apps/details/deployments/actions.ts | 9 ++ .../apps/details/runtime/monitoring-app.tsx | 9 +- .../operations/monitoring-history-actions.ts | 17 ++++ .../monitoring-history-settings.tsx | 53 ++++++++++++ src/app/settings/platform/operations/page.tsx | 2 + .../custom/app-monitoring-history.tsx | 83 +++++++++++++++++++ .../app-monitoring-history.unit.spec.ts | 31 +++++++ src/server.ts | 2 + .../adapter/app-history-metrics.adapter.ts | 15 ++++ src/server/adapter/app-history.adapter.ts | 53 ++++++++++++ src/server/adapter/volume-summary.adapter.ts | 4 + .../app-monitoring-history-query.service.ts | 71 ++++++++++++++++ ...itoring-history-query.service.unit.spec.ts | 25 ++++++ src/server/services/monitoring.service.ts | 60 +------------- .../app-history-metrics.service.ts | 65 +++++++++++++++ .../app-history-metrics.service.unit.spec.ts | 42 ++++++++++ .../app-monitoring-history.service.ts | 55 ++++++++++++ ...pp-monitoring-history.service.unit.spec.ts | 52 ++++++++++++ .../app-volume-filesystem.service.ts | 64 ++++++++++++++ .../model/app-monitoring-history.model.ts | 20 +++++ src/shared/model/generated-zod/app.ts | 4 +- .../generated-zod/appmonitoringsample.ts | 25 ++++++ src/shared/model/generated-zod/appvolume.ts | 6 +- .../appvolumemonitoringmembership.ts | 23 +++++ .../appvolumemonitoringsample.ts | 29 +++++++ src/shared/model/generated-zod/index.ts | 3 + src/shared/utils/constants.ts | 1 + 33 files changed, 937 insertions(+), 62 deletions(-) create mode 100644 prisma/migrations/20261108130000_app_monitoring_history/migration.sql create mode 100644 src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts create mode 100644 src/app/monitoring/history-actions.unit.spec.ts create mode 100644 src/app/settings/platform/operations/monitoring-history-actions.ts create mode 100644 src/app/settings/platform/operations/monitoring-history-settings.tsx create mode 100644 src/components/custom/app-monitoring-history.tsx create mode 100644 src/components/custom/app-monitoring-history.unit.spec.ts create mode 100644 src/server/adapter/app-history-metrics.adapter.ts create mode 100644 src/server/adapter/app-history.adapter.ts create mode 100644 src/server/services/app-monitoring-history-query.service.ts create mode 100644 src/server/services/app-monitoring-history-query.service.unit.spec.ts create mode 100644 src/server/services/standalone-services/app-history-metrics.service.ts create mode 100644 src/server/services/standalone-services/app-history-metrics.service.unit.spec.ts create mode 100644 src/server/services/standalone-services/app-monitoring-history.service.ts create mode 100644 src/server/services/standalone-services/app-monitoring-history.service.unit.spec.ts create mode 100644 src/server/services/standalone-services/app-volume-filesystem.service.ts create mode 100644 src/shared/model/app-monitoring-history.model.ts create mode 100644 src/shared/model/generated-zod/appmonitoringsample.ts create mode 100644 src/shared/model/generated-zod/appvolumemonitoringmembership.ts create mode 100644 src/shared/model/generated-zod/appvolumemonitoringsample.ts diff --git a/prisma/migrations/20261108130000_app_monitoring_history/migration.sql b/prisma/migrations/20261108130000_app_monitoring_history/migration.sql new file mode 100644 index 00000000..7aefa3d5 --- /dev/null +++ b/prisma/migrations/20261108130000_app_monitoring_history/migration.sql @@ -0,0 +1,33 @@ +CREATE TABLE "AppMonitoringSample" ( + "id" TEXT NOT NULL PRIMARY KEY, + "sampledAt" DATETIME NOT NULL, + "appId" TEXT NOT NULL, + "cpuCores" REAL, + "ramBytes" REAL, + "measuredAt" DATETIME, + CONSTRAINT "AppMonitoringSample_appId_fkey" FOREIGN KEY ("appId") REFERENCES "App" ("id") ON DELETE CASCADE ON UPDATE CASCADE +); +CREATE INDEX "AppMonitoringSample_appId_sampledAt_idx" ON "AppMonitoringSample"("appId", "sampledAt"); +CREATE INDEX "AppMonitoringSample_sampledAt_idx" ON "AppMonitoringSample"("sampledAt"); +CREATE TABLE "AppVolumeMonitoringSample" ( + "id" TEXT NOT NULL PRIMARY KEY, + "sampledAt" DATETIME NOT NULL, + "volumeId" TEXT NOT NULL, + "status" TEXT NOT NULL, + "usedBytes" REAL, + "availableBytes" REAL, + "capacityBytes" REAL, + "measuredAt" DATETIME, + CONSTRAINT "AppVolumeMonitoringSample_volumeId_fkey" FOREIGN KEY ("volumeId") REFERENCES "AppVolume" ("id") ON DELETE CASCADE ON UPDATE CASCADE +); +CREATE INDEX "AppVolumeMonitoringSample_volumeId_sampledAt_idx" ON "AppVolumeMonitoringSample"("volumeId", "sampledAt"); +CREATE INDEX "AppVolumeMonitoringSample_sampledAt_idx" ON "AppVolumeMonitoringSample"("sampledAt"); + +CREATE TABLE "AppVolumeMonitoringMembership" ( + "sampleId" TEXT NOT NULL, + "volumeId" TEXT NOT NULL, + PRIMARY KEY ("sampleId", "volumeId"), + CONSTRAINT "AppVolumeMonitoringMembership_sampleId_fkey" FOREIGN KEY ("sampleId") REFERENCES "AppVolumeMonitoringSample" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT "AppVolumeMonitoringMembership_volumeId_fkey" FOREIGN KEY ("volumeId") REFERENCES "AppVolume" ("id") ON DELETE CASCADE ON UPDATE CASCADE +); +CREATE INDEX "AppVolumeMonitoringMembership_volumeId_idx" ON "AppVolumeMonitoringMembership"("volumeId"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 19724a9f..9452d17b 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -465,6 +465,7 @@ model App { appPorts AppPort[] appNodePorts AppNodePort[] appVolumes AppVolume[] + monitoringSamples AppMonitoringSample[] appFileMounts AppFileMount[] appBasicAuths AppBasicAuth[] appGitSshKey AppGitSshKey? @@ -570,6 +571,8 @@ model AppVolume { appId String app App @relation(fields: [appId], references: [id], onDelete: Cascade) volumeBackups VolumeBackup[] + monitoringSamples AppVolumeMonitoringSample[] + monitoringMemberships AppVolumeMonitoringMembership[] sharedVolume AppVolume? @relation("SharedVolume", fields: [sharedVolumeId], references: [id], onDelete: Cascade) sharedVolumes AppVolume[] @relation("SharedVolume") @@ -643,3 +646,43 @@ model AppBasicAuth { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt } + +// One row per known App and Volume at each collection time. Null values are gaps. +model AppMonitoringSample { + id String @id @default(uuid()) + sampledAt DateTime + appId String + app App @relation(fields: [appId], references: [id], onDelete: Cascade) + cpuCores Float? + ramBytes Float? + measuredAt DateTime? + + @@index([appId, sampledAt]) + @@index([sampledAt]) +} + +model AppVolumeMonitoringSample { + id String @id @default(uuid()) + sampledAt DateTime + volumeId String + volume AppVolume @relation(fields: [volumeId], references: [id], onDelete: Cascade) + status String + usedBytes Float? + availableBytes Float? + capacityBytes Float? + measuredAt DateTime? + memberships AppVolumeMonitoringMembership[] + + @@index([volumeId, sampledAt]) + @@index([sampledAt]) +} + +model AppVolumeMonitoringMembership { + sampleId String + sample AppVolumeMonitoringSample @relation(fields: [sampleId], references: [id], onDelete: Cascade) + volumeId String + volume AppVolume @relation(fields: [volumeId], references: [id], onDelete: Cascade) + + @@id([sampleId, volumeId]) + @@index([volumeId]) +} diff --git a/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts b/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts new file mode 100644 index 00000000..bb665eb0 --- /dev/null +++ b/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts @@ -0,0 +1,71 @@ +// @vitest-environment node +import { createPrismaTestContext } from '@/__tests__/prisma-test.utils'; +import adapter from '@/server/adapter/app-history.adapter'; +import history from '@/server/services/app-monitoring-history-query.service'; + +describe('App monitoring history persistence', () => { + const ctx = createPrismaTestContext('app-monitoring-history'); + + it('retains the boundary sample and removes history when its App is deleted', async () => { + const db = ctx.getDataAccess().client; + await db.project.create({ data: { id: 'project', name: 'Project' } }); + await db.app.create({ data: { id: 'app', name: 'App', projectId: 'project' } }); + const cutoff = new Date(Date.now() - 3600_000); + await adapter.save([ + { appId: 'app', sampledAt: new Date(cutoff.getTime() - 1), cpuCores: 1, ramBytes: 2 }, + { appId: 'app', sampledAt: cutoff, cpuCores: 3, ramBytes: 4 }, + ], [], [], cutoff); + expect((await history.getHistory(['app'])).points).toMatchObject([{ cpuCores: 3, ramBytes: 4 }]); + await db.app.delete({ where: { id: 'app' } }); + expect((await history.getHistory(['app'])).points).toEqual([]); + }); + + it('applies a reduced retention period to existing history', async () => { + const db = ctx.getDataAccess().client; + await db.project.create({ data: { id: 'project', name: 'Project' } }); + await db.app.create({ data: { id: 'app', name: 'App', projectId: 'project' } }); + const now = Date.now(); + await adapter.save([{ appId: 'app', sampledAt: new Date(now - 2 * 3600_000), cpuCores: 1, ramBytes: 2 }], [], [], new Date(now - 24 * 3600_000)); + await adapter.setRetentionHours(1); + expect((await history.getHistory(['app'])).points).toEqual([]); + await adapter.save([], [], [], new Date(now - (await adapter.getRetentionHours()) * 3600_000)); + expect(await db.appMonitoringSample.count()).toBe(0); + }); + + it('stores shared Volume usage once and exposes it only through accessible mounts', async () => { + const db = ctx.getDataAccess().client; + await db.project.create({ data: { id: 'project', name: 'Project' } }); + await db.app.createMany({ data: [ + { id: 'one', name: 'One', projectId: 'project' }, + { id: 'two', name: 'Two', projectId: 'project' }, + ] }); + await db.appVolume.create({ data: { id: 'base', appId: 'one', containerMountPath: '/data', size: 1 } }); + await db.appVolume.create({ data: { id: 'shared', appId: 'two', containerMountPath: '/share', size: 1, sharedVolumeId: 'base' } }); + const sampledAt = new Date(); + await adapter.save([], [{ id: 'sample', volumeId: 'base', sampledAt, status: 'available', usedBytes: 42, measuredAt: sampledAt }], [ + { sampleId: 'sample', volumeId: 'base' }, { sampleId: 'sample', volumeId: 'shared' }, + ], new Date(sampledAt.getTime() - 3600_000)); + expect(await db.appVolumeMonitoringSample.count()).toBe(1); + expect((await history.getHistory(['two'])).volumes.map(volume => volume.label)).toEqual(['/share']); + expect((await history.getHistory(['one', 'two'])).points[0]).toMatchObject({ storageBytes: 42, availableVolumes: 1 }); + await db.appVolume.delete({ where: { id: 'base' } }); + expect((await history.getHistory(['one', 'two'])).points).toEqual([]); + }); + + + it('reads a day of five-minute history for 100 Apps', async () => { + const db = ctx.getDataAccess().client; + await db.project.create({ data: { id: 'project', name: 'Project' } }); + const appIds = Array.from({ length: 100 }, (_, index) => `app-${index}`); + await db.app.createMany({ data: appIds.map(id => ({ id, name: id, projectId: 'project' })) }); + const now = Date.now(); + const samples = Array.from({ length: 288 }, (_, index) => appIds.map(appId => ({ + appId, sampledAt: new Date(now - index * 5 * 60_000), cpuCores: 0.25, ramBytes: 1024, + }))).flat(); + await adapter.save(samples, [], [], new Date(now - 24 * 3600_000)); + const result = await history.getHistory(appIds); + expect(result.points).toHaveLength(288); + expect(result.points[0]).toMatchObject({ cpuCores: 25, ramBytes: 102400 }); + }, 30000); + +}); diff --git a/src/app/monitoring/actions.ts b/src/app/monitoring/actions.ts index 47b5ba91..c412a364 100644 --- a/src/app/monitoring/actions.ts +++ b/src/app/monitoring/actions.ts @@ -1,5 +1,7 @@ 'use server' +import appHistoryAdapter from "@/server/adapter/app-history.adapter"; +import appMonitoringHistoryQueryService from "@/server/services/app-monitoring-history-query.service"; import monitoringService from "@/server/services/monitoring.service"; import clusterService from "@/server/services/cluster.service"; import { getAuthUserSession, simpleAction } from "@/server/utils/action-wrapper.utils"; @@ -29,4 +31,11 @@ export const getMonitoringForAllApps = async () => let updatedNodeRessources = await monitoringService.getMonitoringForAllApps(); updatedNodeRessources = updatedNodeRessources?.filter((app) => UserGroupUtils.sessionHasReadAccessForApp(session, app.appId)); return updatedNodeRessources; - }) as Promise>; \ No newline at end of file + }) as Promise>; +export const getAppMonitoringHistory = async () => + simpleAction(async () => { + const session = await getAuthUserSession(); + const { apps } = await appHistoryAdapter.getPopulation(); + const appIds = apps.filter(app => UserGroupUtils.sessionHasReadAccessForApp(session, app.id)).map(app => app.id); + return appMonitoringHistoryQueryService.getHistory(appIds); + }); diff --git a/src/app/monitoring/history-actions.unit.spec.ts b/src/app/monitoring/history-actions.unit.spec.ts new file mode 100644 index 00000000..10b53dea --- /dev/null +++ b/src/app/monitoring/history-actions.unit.spec.ts @@ -0,0 +1,16 @@ +vi.mock('@/server/utils/action-wrapper.utils', () => ({ + getAuthUserSession: vi.fn().mockResolvedValue({ userId: 'user' }), + simpleAction: async (callback: () => Promise) => ({ status: 'success', data: await callback() }), +})); +vi.mock('@/server/adapter/app-history.adapter', () => ({ default: { getPopulation: vi.fn().mockResolvedValue({ apps: [{ id: 'allowed' }, { id: 'denied' }], volumes: [] }) } })); +vi.mock('@/server/services/app-monitoring-history-query.service', () => ({ default: { getHistory: vi.fn().mockResolvedValue({}) } })); +vi.mock('@/shared/utils/role.utils', () => ({ UserGroupUtils: { sessionHasReadAccessForApp: (_session: unknown, appId: string) => appId === 'allowed' } })); +vi.mock('@/server/services/monitoring.service', () => ({ default: {} })); +vi.mock('@/server/services/cluster.service', () => ({ default: {} })); +import history from '@/server/services/app-monitoring-history-query.service'; +import { getAppMonitoringHistory } from './actions'; + +it('queries history only for Apps the user may read', async () => { + await getAppMonitoringHistory(); + expect(history.getHistory).toHaveBeenCalledWith(['allowed']); +}); diff --git a/src/app/monitoring/monitoring-client.tsx b/src/app/monitoring/monitoring-client.tsx index a804b9a0..59499fe4 100644 --- a/src/app/monitoring/monitoring-client.tsx +++ b/src/app/monitoring/monitoring-client.tsx @@ -10,6 +10,7 @@ import { NodeResourceModel } from '@/shared/model/node-resource.model'; import { getMonitoringForAllApps, getNodeResourceUsage, getVolumeMonitoringUsage } from './actions'; import AppRessourceMonitoring from './app-monitoring'; import AppVolumeMonitoring from './app-volumes-monitoring'; +import AppMonitoringHistoryView from '@/components/custom/app-monitoring-history'; import ResourcesNodes from './monitoring-nodes'; const MONITORING_POLL_INTERVAL_MS = 10000; @@ -54,6 +55,7 @@ export default function MonitoringClient({ return (
+
diff --git a/src/app/project/[projectId]/apps/details/deployments/actions.ts b/src/app/project/[projectId]/apps/details/deployments/actions.ts index 6d5957b8..1d98ab2f 100644 --- a/src/app/project/[projectId]/apps/details/deployments/actions.ts +++ b/src/app/project/[projectId]/apps/details/deployments/actions.ts @@ -1,5 +1,7 @@ 'use server' +import appMonitoringHistoryQueryService from '@/server/services/app-monitoring-history-query.service'; + import { SuccessActionResult } from "@/shared/model/server-action-error-return.model"; import appService from "@/server/services/app.service"; import buildService from "@/server/services/build.service"; @@ -47,3 +49,10 @@ export const createNewWebhookUrl = async (appId: string) => await isAuthorizedWriteForWorkload(appId); return await appService.regenerateWebhookId(appId); }); + + +export const getAppMonitoringHistory = async (appId: string) => + simpleAction(async () => { + await isAuthorizedReadForApp(appId); + return appMonitoringHistoryQueryService.getHistory([appId]); + }); diff --git a/src/app/project/[projectId]/apps/details/runtime/monitoring-app.tsx b/src/app/project/[projectId]/apps/details/runtime/monitoring-app.tsx index 52daa810..fa118152 100644 --- a/src/app/project/[projectId]/apps/details/runtime/monitoring-app.tsx +++ b/src/app/project/[projectId]/apps/details/runtime/monitoring-app.tsx @@ -1,3 +1,4 @@ +import AppMonitoringHistoryView from "@/components/custom/app-monitoring-history"; import { CardContent } from "@/components/ui/card"; import { AppExtendedModel } from "@/shared/model/app-extended.model"; import { useCallback, useState } from "react"; @@ -17,7 +18,7 @@ export default function MonitoringTab({ }) { const [selectedPod, setSelectedPod] = useState(undefined); - const [, setError] = useState(undefined); + const [error, setError] = useState(undefined); const updateValues = useCallback(async () => { setError(undefined); @@ -40,7 +41,8 @@ export default function MonitoringTab({ return <> - {!selectedPod ? : + {error &&

Unable to load current App metrics: {error}

} + {!selectedPod && !error ? : selectedPod ?
@@ -63,8 +65,9 @@ export default function MonitoringTab({ {KubeSizeConverter.convertBytesToReadableSize(selectedPod?.ramAbsolutBytes)} -
+ : null } + ; } diff --git a/src/app/settings/platform/operations/monitoring-history-actions.ts b/src/app/settings/platform/operations/monitoring-history-actions.ts new file mode 100644 index 00000000..fb053b7c --- /dev/null +++ b/src/app/settings/platform/operations/monitoring-history-actions.ts @@ -0,0 +1,17 @@ +'use server'; + +import appHistoryAdapter from '@/server/adapter/app-history.adapter'; +import { getAdminUserSession, simpleAction } from '@/server/utils/action-wrapper.utils'; +import { z } from 'zod'; + +const retentionSchema = z.number().int().min(1).max(168); + +export const getMonitoringHistoryRetention = async () => simpleAction(async () => { + await getAdminUserSession(); + return appHistoryAdapter.getRetentionHours(); +}); + +export const setMonitoringHistoryRetention = async (hours: number) => simpleAction(async () => { + await getAdminUserSession(); + await appHistoryAdapter.setRetentionHours(retentionSchema.parse(hours)); +}); diff --git a/src/app/settings/platform/operations/monitoring-history-settings.tsx b/src/app/settings/platform/operations/monitoring-history-settings.tsx new file mode 100644 index 00000000..2be84d1c --- /dev/null +++ b/src/app/settings/platform/operations/monitoring-history-settings.tsx @@ -0,0 +1,53 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { z } from 'zod'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { useForm } from 'react-hook-form'; +import { Button } from '@/components/ui/button'; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'; +import { Form, FormControl, FormField, FormItem, FormLabel, FormMessage } from '@/components/ui/form'; +import { Input } from '@/components/ui/input'; +import { getMonitoringHistoryRetention, setMonitoringHistoryRetention } from './monitoring-history-actions'; + +const schema = z.object({ hours: z.coerce.number().int().min(1).max(168) }); + +export default function MonitoringHistorySettings() { + const [loaded, setLoaded] = useState(false); + const [message, setMessage] = useState(''); + const form = useForm, unknown, z.output>({ + resolver: zodResolver(schema), + defaultValues: { hours: '' }, + }); + useEffect(() => { + void getMonitoringHistoryRetention().then(result => { + if (result.status === 'success' && result.data !== undefined) { + form.reset({ hours: result.data }); + setLoaded(true); + } else setMessage('Unable to load retention settings.'); + }).catch(() => setMessage('Unable to load retention settings.')); + }, [form]); + return + App monitoring historyKeep measurements for 1 to 168 hours. Default: 24 hours. + +
+ { + try { + const result = await setMonitoringHistoryRetention(hours); + setMessage(result.status === 'success' ? 'Retention saved. Older samples are removed during the next collection.' : result.message ?? 'Unable to save retention.'); + } catch { + setMessage('Unable to save retention.'); + } + })}> + + Retention (hours) + + + } /> + + {message &&

{message}

} + + +
+ ; +} diff --git a/src/app/settings/platform/operations/page.tsx b/src/app/settings/platform/operations/page.tsx index e730d6c5..59bbe62a 100644 --- a/src/app/settings/platform/operations/page.tsx +++ b/src/app/settings/platform/operations/page.tsx @@ -1,3 +1,4 @@ +import MonitoringHistorySettings from "./monitoring-history-settings" import { Constants } from "@/shared/utils/constants" import podService from "@/server/services/pod.service" import QuickStackMaintenanceSettings from "./qs-maintenance-settings" @@ -10,6 +11,7 @@ export default async function MaintenanceSettingsPage() { return
+
} diff --git a/src/components/custom/app-monitoring-history.tsx b/src/components/custom/app-monitoring-history.tsx new file mode 100644 index 00000000..c0d69a0f --- /dev/null +++ b/src/components/custom/app-monitoring-history.tsx @@ -0,0 +1,83 @@ +'use client'; + +import { useCallback, useState } from 'react'; +import { CartesianGrid, Line, LineChart, ResponsiveContainer, Tooltip, XAxis, YAxis } from 'recharts'; +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { usePolling } from '@/frontend/hooks/use-polling'; +import { AppMonitoringHistory } from '@/shared/model/app-monitoring-history.model'; +import { getAppMonitoringHistory as getAllHistory } from '@/app/monitoring/actions'; +import { getAppMonitoringHistory as getOneHistory } from '@/app/project/[projectId]/apps/details/deployments/actions'; + +function HistoryChart({ title, data, dataKey, unit }: { title: string; data: object[]; dataKey: string; unit: string }) { + return
+

{title} ({unit})

+
+ + + + new Date(value).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })} /> + + { + if (!active || !payload?.length) return null; + const item = payload[0]; + const sample = item.payload as { availableVolumes?: number; unavailableVolumes?: number }; + return
+

{new Date(Number(label)).toLocaleString()}

+

{item.value} {unit}

+ {dataKey === 'storageBytes' &&

{sample.availableVolumes} available, {sample.unavailableVolumes} unavailable{sample.unavailableVolumes ? ' (incomplete total)' : ''}

} +
; + }} /> + + + +
+
; +} + +function freshness(time: number | null) { + if (!time) return 'Measurement unavailable'; + return `Latest successful measurement: ${new Date(time).toLocaleString()}${Date.now() - time > 10 * 60_000 ? ' (stale)' : ''}`; +} + +export default function AppMonitoringHistoryView({ appId }: { appId?: string }) { + const [history, setHistory] = useState(); + const [error, setError] = useState(false); + const refresh = useCallback(async () => { + try { + const response = appId ? await getOneHistory(appId) : await getAllHistory(); + if (response.status !== 'success' || !response.data) throw new Error(response.message ?? 'Unable to load history'); + setHistory(response.data); + setError(false); + } catch { + setError(true); + } + }, [appId]); + usePolling(refresh, { intervalMs: 60_000, runImmediately: true }); + const latest = history?.points[history.points.length - 1]; + return + {appId ? 'App history' : 'App totals history'} + + {error &&

Unable to load monitoring history.

} + {!history && !error &&

Loading monitoring history…

} + {history && history.points.length === 0 &&

No measurements yet. History is collected every five minutes.

} + {history && history.points.length > 0 && <> + {history.points.length === 1 &&

One sample collected. More samples will show a trend.

} +

CPU is an average over the Kubernetes metrics source window; short peaks may be missed.

+ +

{freshness(history.latestCpuMeasurement)}

+ +

{freshness(history.latestRamMeasurement)}

+

{freshness(history.latestVolumeMeasurement)}

+ {!appId && <> + + {latest &&

Latest sample: {latest.availableVolumes} available and {latest.unavailableVolumes} unavailable volumes{latest.unavailableVolumes ? ' (incomplete total)' : ''}.

} + } + {appId && history.volumes.length === 0 &&

No App Volume measurements.

} + {appId && history.volumes.map(volume =>
+ +

{freshness(volume.points.reduce((latest, point) => point.usedBytes !== null && point.measuredAt ? Math.max(latest ?? 0, point.measuredAt) : latest, null))}

+
)} + } + + ; +} diff --git a/src/components/custom/app-monitoring-history.unit.spec.ts b/src/components/custom/app-monitoring-history.unit.spec.ts new file mode 100644 index 00000000..dc806277 --- /dev/null +++ b/src/components/custom/app-monitoring-history.unit.spec.ts @@ -0,0 +1,31 @@ +import React from 'react'; +import { cleanup, render, screen } from '@testing-library/react'; +import AppMonitoringHistoryView from './app-monitoring-history'; +import { getAppMonitoringHistory } from '@/app/monitoring/actions'; + +vi.mock('@/app/monitoring/actions', () => ({ getAppMonitoringHistory: vi.fn() })); +vi.mock('@/app/project/[projectId]/apps/details/deployments/actions', () => ({ getAppMonitoringHistory: vi.fn() })); +vi.mock('recharts', () => ({ + CartesianGrid: () => null, Line: () => null, LineChart: () => null, + ResponsiveContainer: () => null, Tooltip: () => null, XAxis: () => null, YAxis: () => null, +})); +afterEach(() => { cleanup(); vi.clearAllMocks(); }); + +it('shows an empty state before the first sample', async () => { + vi.mocked(getAppMonitoringHistory).mockResolvedValue({ status: 'success', data: { points: [], volumes: [], latestCpuMeasurement: null, latestRamMeasurement: null, latestVolumeMeasurement: null } } as never); + render(React.createElement(AppMonitoringHistoryView)); + expect(await screen.findByText(/No measurements yet/)).toBeTruthy(); +}); + +it('labels partial App Volume totals with availability counts', async () => { + vi.mocked(getAppMonitoringHistory).mockResolvedValue({ status: 'success', data: { points: [{ time: Date.now(), cpuCores: null, ramBytes: null, storageBytes: 10, availableVolumes: 1, unavailableVolumes: 2 }], volumes: [], latestCpuMeasurement: null, latestRamMeasurement: null, latestVolumeMeasurement: null } } as never); + render(React.createElement(AppMonitoringHistoryView)); + expect(await screen.findByText(/1 available and 2 unavailable volumes \(incomplete total\)/)).toBeTruthy(); + expect(screen.getByText(/App CPU total/)).toBeTruthy(); +}); + +it('shows an error when history cannot be loaded', async () => { + vi.mocked(getAppMonitoringHistory).mockRejectedValue(new Error('unavailable')); + render(React.createElement(AppMonitoringHistoryView)); + expect((await screen.findByRole('alert')).textContent).toContain('Unable to load monitoring history.'); +}); diff --git a/src/server.ts b/src/server.ts index e6bae396..6157dbb2 100644 --- a/src/server.ts +++ b/src/server.ts @@ -10,6 +10,7 @@ import { FancyConsoleUtils } from './shared/utils/fancy-console.utils' import { Constants } from './shared/utils/constants' import backupService from './server/services/standalone-services/backup.service' import maintenanceService from './server/services/standalone-services/maintenance.service' +import appMonitoringHistoryService from './server/services/standalone-services/app-monitoring-history.service' import passwordChangeService from './server/services/standalone-services/password-change.service' declare global { @@ -63,6 +64,7 @@ async function initializeNextJs() { await backupService.registerAllBackups(); maintenanceService.configureMaintenanceCronJobs(); + appMonitoringHistoryService.initialize(); const app = next({ dev }); const handle = app.getRequestHandler(); diff --git a/src/server/adapter/app-history-metrics.adapter.ts b/src/server/adapter/app-history-metrics.adapter.ts new file mode 100644 index 00000000..bd6e8a8d --- /dev/null +++ b/src/server/adapter/app-history-metrics.adapter.ts @@ -0,0 +1,15 @@ +import * as k8s from '@kubernetes/client-node'; +import k3s from './kubernetes-api.adapter'; + +class AppHistoryMetricsAdapter { + async getPods() { + return (await k3s.core.listPodForAllNamespaces()).items; + } + + async getMetrics() { + return (await new k8s.Metrics(k3s.getKubeConfig()).getPodMetrics()).items; + } +} + +const appHistoryMetricsAdapter = new AppHistoryMetricsAdapter(); +export default appHistoryMetricsAdapter; diff --git a/src/server/adapter/app-history.adapter.ts b/src/server/adapter/app-history.adapter.ts new file mode 100644 index 00000000..1404b9ac --- /dev/null +++ b/src/server/adapter/app-history.adapter.ts @@ -0,0 +1,53 @@ +import { Prisma } from '@prisma/client'; +import dataAccess from './db.client'; +import { Constants } from '@/shared/utils/constants'; + +class AppHistoryAdapter { + async getPopulation() { + const [apps, volumes] = await Promise.all([ + dataAccess.client.app.findMany({ select: { id: true, projectId: true } }), + dataAccess.client.appVolume.findMany({ include: { app: { select: { projectId: true } } } }), + ]); + return { apps, volumes }; + } + + async getRetentionHours() { + const value = (await dataAccess.client.parameter.findUnique({ where: { name: Constants.APP_MONITORING_HISTORY_RETENTION_HOURS } }))?.value; + const hours = Number(value); + return Number.isInteger(hours) && hours >= 1 && hours <= 168 ? hours : 24; + } + + async setRetentionHours(hours: number) { + await dataAccess.client.parameter.upsert({ + where: { name: Constants.APP_MONITORING_HISTORY_RETENTION_HOURS }, + create: { name: Constants.APP_MONITORING_HISTORY_RETENTION_HOURS, value: String(hours) }, + update: { value: String(hours) }, + }); + } + + async save(apps: Prisma.AppMonitoringSampleCreateManyInput[], volumes: Prisma.AppVolumeMonitoringSampleCreateManyInput[], memberships: Prisma.AppVolumeMonitoringMembershipCreateManyInput[], cutoff: Date) { + await dataAccess.client.$transaction(async tx => { + for (let i = 0; i < apps.length; i += 100) await tx.appMonitoringSample.createMany({ data: apps.slice(i, i + 100) }); + for (let i = 0; i < volumes.length; i += 100) await tx.appVolumeMonitoringSample.createMany({ data: volumes.slice(i, i + 100) }); + for (let i = 0; i < memberships.length; i += 100) await tx.appVolumeMonitoringMembership.createMany({ data: memberships.slice(i, i + 100) }); + await tx.appMonitoringSample.deleteMany({ where: { sampledAt: { lt: cutoff } } }); + await tx.appVolumeMonitoringSample.deleteMany({ where: { sampledAt: { lt: cutoff } } }); + }); + } + + async getHistory(appIds: string[], cutoff: Date) { + return Promise.all([ + dataAccess.client.appMonitoringSample.findMany({ where: { appId: { in: appIds }, sampledAt: { gte: cutoff } }, orderBy: { sampledAt: 'asc' } }), + dataAccess.client.appVolumeMonitoringMembership.findMany({ + where: { volume: { appId: { in: appIds } }, sample: { sampledAt: { gte: cutoff } } }, + include: { + sample: true, + volume: { select: { appId: true, sharedVolumeId: true, containerMountPath: true, app: { select: { projectId: true } } } }, + }, + orderBy: { sample: { sampledAt: 'asc' } }, + }).then(memberships => memberships.map(item => ({ ...item.sample, volumeId: item.volumeId, volume: item.volume }))), + ]); + } +} +const appHistoryAdapter = new AppHistoryAdapter(); +export default appHistoryAdapter; diff --git a/src/server/adapter/volume-summary.adapter.ts b/src/server/adapter/volume-summary.adapter.ts index 47f730f1..e1227b5f 100644 --- a/src/server/adapter/volume-summary.adapter.ts +++ b/src/server/adapter/volume-summary.adapter.ts @@ -13,6 +13,10 @@ interface NodeSummary { } class VolumeSummaryAdapter { + async getPersistentVolumeClaims() { + return (await k3s.core.listPersistentVolumeClaimForAllNamespaces()).items; + } + async getPersistentVolumes() { return (await k3s.core.listPersistentVolume()).items; } diff --git a/src/server/services/app-monitoring-history-query.service.ts b/src/server/services/app-monitoring-history-query.service.ts new file mode 100644 index 00000000..b197d57e --- /dev/null +++ b/src/server/services/app-monitoring-history-query.service.ts @@ -0,0 +1,71 @@ +import appHistoryAdapter from '@/server/adapter/app-history.adapter'; +import { AppHistoryPoint, AppMonitoringHistory, AppVolumeHistory } from '@/shared/model/app-monitoring-history.model'; + +class AppMonitoringHistoryQueryService { + async getHistory(appIds: string[]): Promise { + const retention = await appHistoryAdapter.getRetentionHours(); + const [apps, volumes] = await appHistoryAdapter.getHistory(appIds, new Date(Date.now() - retention * 3600_000)); + const appsByTime = new Map(); + const volumesByTime = new Map(); + for (const item of apps) { + const time = item.sampledAt.getTime(); + const group = appsByTime.get(time) ?? []; + group.push(item); + appsByTime.set(time, group); + } + for (const item of volumes) { + const time = item.sampledAt.getTime(); + const group = volumesByTime.get(time) ?? []; + group.push(item); + volumesByTime.set(time, group); + } + const times = new Set([...appsByTime.keys(), ...volumesByTime.keys()]); + const points: AppHistoryPoint[] = []; + let latestCpuMeasurement: number | null = null; + let latestRamMeasurement: number | null = null; + let latestVolumeMeasurement: number | null = null; + const volumeSeries = new Map(); + for (const time of [...times].sort((a, b) => a - b)) { + if (points.length && time - points[points.length - 1].time > 7 * 60_000) { + points.push({ time: points[points.length - 1].time + 5 * 60_000, cpuCores: null, ramBytes: null, storageBytes: null, availableVolumes: 0, unavailableVolumes: 0 }); + } + const currentApps = appsByTime.get(time) ?? []; + const currentVolumes = volumesByTime.get(time) ?? []; + const uniqueVolumes = new Map(); + for (const item of currentVolumes) { + const key = `${item.volume.app.projectId}/${item.volume.sharedVolumeId ?? item.volumeId}`; + if (!uniqueVolumes.has(key) || item.status === 'available') uniqueVolumes.set(key, item); + let series = volumeSeries.get(item.volumeId); + if (!series) { + series = { id: item.volumeId, label: item.volume.containerMountPath, points: [] }; + volumeSeries.set(item.volumeId, series); + } + if (series.points.length && time - series.points[series.points.length - 1].time > 7 * 60_000) { + series.points.push({ time: series.points[series.points.length - 1].time + 5 * 60_000, usedBytes: null, measuredAt: null }); + } + series.points.push({ time, usedBytes: item.status === 'available' ? item.usedBytes : null, measuredAt: item.measuredAt?.getTime() ?? null }); + } + if (currentApps.length && currentApps.every(item => item.cpuCores !== null && item.measuredAt)) { + latestCpuMeasurement = Math.max(latestCpuMeasurement ?? 0, Math.min(...currentApps.map(item => item.measuredAt!.getTime()))); + } + if (currentApps.length && currentApps.every(item => item.ramBytes !== null && item.measuredAt)) { + latestRamMeasurement = Math.max(latestRamMeasurement ?? 0, Math.min(...currentApps.map(item => item.measuredAt!.getTime()))); + } + const available = [...uniqueVolumes.values()].filter(item => item.status === 'available' && item.usedBytes !== null); + if (available.length && available.every(item => item.measuredAt)) { + latestVolumeMeasurement = Math.max(latestVolumeMeasurement ?? 0, Math.min(...available.map(item => item.measuredAt!.getTime()))); + } + points.push({ + time, + cpuCores: currentApps.length && currentApps.every(item => item.cpuCores !== null) ? currentApps.reduce((sum, item) => sum + item.cpuCores!, 0) : null, + ramBytes: currentApps.length && currentApps.every(item => item.ramBytes !== null) ? currentApps.reduce((sum, item) => sum + item.ramBytes!, 0) : null, + storageBytes: available.length ? available.reduce((sum, item) => sum + item.usedBytes!, 0) : null, + availableVolumes: available.length, + unavailableVolumes: uniqueVolumes.size - available.length, + }); + } + return { points, volumes: [...volumeSeries.values()], latestCpuMeasurement, latestRamMeasurement, latestVolumeMeasurement }; + } +} +const appMonitoringHistoryQueryService = new AppMonitoringHistoryQueryService(); +export default appMonitoringHistoryQueryService; diff --git a/src/server/services/app-monitoring-history-query.service.unit.spec.ts b/src/server/services/app-monitoring-history-query.service.unit.spec.ts new file mode 100644 index 00000000..dafccf2e --- /dev/null +++ b/src/server/services/app-monitoring-history-query.service.unit.spec.ts @@ -0,0 +1,25 @@ +vi.mock('@/server/adapter/app-history.adapter', () => ({ + default: { getRetentionHours: vi.fn().mockResolvedValue(24), getHistory: vi.fn() }, +})); +import adapter from '@/server/adapter/app-history.adapter'; +import historyService from './app-monitoring-history-query.service'; + +it('shows incomplete App totals as gaps and counts shared volumes once', async () => { + const now = new Date(); + const app = (appId: string, cpuCores: number | null) => ({ id: appId, appId, sampledAt: now, cpuCores, ramBytes: cpuCores === null ? null : 100, measuredAt: now }); + const volume = (id: string, sharedVolumeId: string | null, status: string, usedBytes: number | null) => ({ id, volumeId: id, sampledAt: now, status, usedBytes, availableBytes: null, capacityBytes: null, measuredAt: now, volume: { appId: id, sharedVolumeId, containerMountPath: '/data', app: { projectId: 'project' } } }); + vi.mocked(adapter.getHistory).mockResolvedValue([ + [app('one', 1), app('two', null)], + [volume('base', null, 'available', 20), volume('shared', 'base', 'available', 20), volume('missing', null, 'unavailable', null)], + ]); + const history = await historyService.getHistory(['one', 'two']); + expect(history.points[0]).toMatchObject({ cpuCores: null, ramBytes: null, storageBytes: 20, availableVolumes: 1, unavailableVolumes: 1 }); +}); + +it('inserts a gap across collection downtime', async () => { + const time = Date.now(); + const sample = (offset: number) => ({ id: String(offset), appId: 'app', sampledAt: new Date(time + offset), cpuCores: 1, ramBytes: 100, measuredAt: new Date(time + offset) }); + vi.mocked(adapter.getHistory).mockResolvedValue([[sample(0), sample(15 * 60_000)], []]); + const result = await historyService.getHistory(['app']); + expect(result.points.map(point => point.cpuCores)).toEqual([1, null, 1]); +}); diff --git a/src/server/services/monitoring.service.ts b/src/server/services/monitoring.service.ts index e21c694c..ada3e542 100644 --- a/src/server/services/monitoring.service.ts +++ b/src/server/services/monitoring.service.ts @@ -5,10 +5,9 @@ import clusterService from "./cluster.service"; import { PodsResourceInfoModel } from "@/shared/model/pods-resource-info.model"; import { KubeSizeConverter } from "../../shared/utils/kubernetes-size-converter.utils"; import { AppVolumeMonitoringUsageModel } from "@/shared/model/app-volume-monitoring-usage.model"; -import volumeSummaryAdapter from "../adapter/volume-summary.adapter"; -import { supportsVolumeFilesystemMetrics } from "./volume-filesystem-drivers"; import { VolumeFilesystemUsage } from "@/shared/model/app-volume-monitoring-usage.model"; import dataAccess from "../adapter/db.client"; +import appVolumeFilesystemService from "./standalone-services/app-volume-filesystem.service"; import pvcService from "./pvc.service"; import { KubeObjectNameUtils } from "../utils/kube-object-name.utils"; import projectService from "./project.service"; @@ -16,65 +15,14 @@ import { AppMonitoringUsageModel } from "@/shared/model/app-monitoring-usage.mod class MonitorService { - private async getVolumeFilesystemUsage( - volumes: { id: string; sharedVolumeId: string | null; projectId: string }[], - ): Promise> { - const [pvcs, persistentVolumes, samples] = await Promise.all([ - pvcService.getAllPvc().catch(() => []), - volumeSummaryAdapter.getPersistentVolumes().catch(() => []), - volumeSummaryAdapter.getSamples().catch(() => []), - ]); - const pvcByKey = new Map(pvcs.map(pvc => [`${pvc.metadata?.namespace}/${pvc.metadata?.name}`, pvc])); - const pvByName = new Map(persistentVolumes.map(pv => [pv.metadata?.name, pv])); - const result = new Map(); - const now = Date.now(); - for (const volume of volumes) { - const pvcName = KubeObjectNameUtils.toPvcName(volume.sharedVolumeId || volume.id); - const key = `${volume.projectId}/${pvcName}`; - if (result.has(key)) continue; - const pvc = pvcByKey.get(key); - const pv = pvc?.spec?.volumeName ? pvByName.get(pvc.spec.volumeName) : undefined; - if (!pv) { - result.set(key, { pvcName, status: 'unavailable' }); - continue; - } - if (!supportsVolumeFilesystemMetrics(pv)) { - result.set(key, { pvcName, status: 'unsupported' }); - continue; - } - const candidates = samples - .filter(sample => sample.pvcRef?.namespace === volume.projectId && sample.pvcRef.name === pvcName) - .filter(sample => { - const values = [sample.usedBytes, sample.availableBytes, sample.capacityBytes]; - return values.every(value => typeof value === 'number' && Number.isFinite(value) && value >= 0) - && !!sample.capacityBytes && Number.isFinite(Date.parse(sample.time ?? '')); - }) - .sort((a, b) => Date.parse(b.time!) - Date.parse(a.time!)); - const sample = candidates[0]; - const age = sample ? now - Date.parse(sample.time!) : Infinity; - if (!sample || age > 5 * 60 * 1000 || age < -60 * 1000) { - result.set(key, { pvcName, status: sample ? 'stale' : 'unavailable' }); - continue; - } - result.set(key, { - pvcName, - status: 'available', - usedBytes: sample.usedBytes, - availableBytes: sample.availableBytes, - capacityBytes: sample.capacityBytes, - measuredAt: sample.time, - }); - } - return result; - } - async getAllAppVolumesUsage(): Promise { const appVolumes = await dataAccess.client.appVolume.findMany({ include: { app: { include: { project: true } } }, orderBy: { appId: 'asc' }, }); - const usages = await this.getVolumeFilesystemUsage( + const usages = await appVolumeFilesystemService.getVolumeFilesystemUsage( appVolumes.map(volume => ({ ...volume, projectId: volume.app.projectId })), + () => pvcService.getAllPvc(), ); return appVolumes.map(volume => { const pvcName = KubeObjectNameUtils.toPvcName(volume.sharedVolumeId || volume.id); @@ -207,7 +155,7 @@ class MonitorService { where: { appId }, select: { id: true, sharedVolumeId: true }, }); - const usages = await this.getVolumeFilesystemUsage(volumes.map(volume => ({ ...volume, projectId }))); + const usages = await appVolumeFilesystemService.getVolumeFilesystemUsage(volumes.map(volume => ({ ...volume, projectId })), () => pvcService.getAllPvc()); return Array.from(usages.values()); } } diff --git a/src/server/services/standalone-services/app-history-metrics.service.ts b/src/server/services/standalone-services/app-history-metrics.service.ts new file mode 100644 index 00000000..7400cee1 --- /dev/null +++ b/src/server/services/standalone-services/app-history-metrics.service.ts @@ -0,0 +1,65 @@ +import { quantityToScalar } from '@kubernetes/client-node/dist/util.js'; +import metricsAdapter from '@/server/adapter/app-history-metrics.adapter'; + +type App = { id: string; projectId: string }; +type Measurement = { cpuCores: number | null; ramBytes: number | null; measuredAt: Date | null }; + +class AppHistoryMetricsService { + async getAppMetrics(apps: App[]): Promise> { + const [pods, metrics] = await Promise.all([metricsAdapter.getPods(), metricsAdapter.getMetrics()]); + const metricsByPod = new Map(metrics.map(metric => [`${metric.metadata.namespace}/${metric.metadata.name}`, metric])); + const podsByApp = new Map(); + for (const pod of pods) { + const appId = pod.metadata?.labels?.app; + const namespace = pod.metadata?.namespace; + if (!appId || !namespace || ['Succeeded', 'Failed'].includes(pod.status?.phase ?? '')) continue; + const key = `${namespace}/${appId}`; + const group = podsByApp.get(key) ?? []; + group.push(pod); + podsByApp.set(key, group); + } + const result = new Map(); + for (const app of apps) { + const appPods = podsByApp.get(`${app.projectId}/${app.id}`) ?? []; + let cpu = 0; + let ram = 0; + let cpuComplete = appPods.length > 0; + let ramComplete = appPods.length > 0; + let oldest = Infinity; + for (const pod of appPods) { + const metric = metricsByPod.get(`${app.projectId}/${pod.metadata?.name}`); + const timestamp = Date.parse(metric?.timestamp ?? ''); + if (!metric || !Number.isFinite(timestamp) || Date.now() - timestamp > 5 * 60_000 || timestamp - Date.now() > 60_000) { + cpuComplete = false; + ramComplete = false; + continue; + } + oldest = Math.min(oldest, timestamp); + if (!pod.spec?.containers?.length) { + cpuComplete = false; + ramComplete = false; + continue; + } + for (const container of pod.spec.containers) { + const usage = metric.containers.find(item => item.name === container.name)?.usage; + for (const kind of ['cpu', 'memory'] as const) { + let value = NaN; + try { + if (usage?.[kind]) value = Number(quantityToScalar(usage[kind])); + } catch { /* Invalid quantities are gaps. */ } + if (!Number.isFinite(value) || value < 0) { + if (kind === 'cpu') cpuComplete = false; + else ramComplete = false; + } else if (kind === 'cpu') cpu += value; + else ram += value; + } + } + } + result.set(app.id, { cpuCores: cpuComplete ? cpu : null, ramBytes: ramComplete ? ram : null, measuredAt: (cpuComplete || ramComplete) && Number.isFinite(oldest) ? new Date(oldest) : null }); + } + return result; + } +} + +const appHistoryMetricsService = new AppHistoryMetricsService(); +export default appHistoryMetricsService; diff --git a/src/server/services/standalone-services/app-history-metrics.service.unit.spec.ts b/src/server/services/standalone-services/app-history-metrics.service.unit.spec.ts new file mode 100644 index 00000000..35f867ac --- /dev/null +++ b/src/server/services/standalone-services/app-history-metrics.service.unit.spec.ts @@ -0,0 +1,42 @@ +import appHistoryMetricsService from './app-history-metrics.service'; + +vi.mock('@/server/adapter/app-history-metrics.adapter', () => ({ + default: { getPods: vi.fn(), getMetrics: vi.fn() }, +})); + +import metricsAdapter from '@/server/adapter/app-history-metrics.adapter'; + +it('marks an App incomplete when a running container has no metrics', async () => { + vi.mocked(metricsAdapter.getPods).mockResolvedValue([ + { metadata: { name: 'pod', namespace: 'project', labels: { app: 'app' } }, status: { phase: 'Running' }, spec: { containers: [{ name: 'one' }, { name: 'two' }] } }, + ] as never); + vi.mocked(metricsAdapter.getMetrics).mockResolvedValue([ + { metadata: { name: 'pod', namespace: 'project' }, timestamp: new Date().toISOString(), containers: [{ name: 'one', usage: { cpu: '250m', memory: '1Mi' } }] }, + ] as never); + + const result = await appHistoryMetricsService.getAppMetrics([{ id: 'app', projectId: 'project' }]); + expect(result.get('app')).toMatchObject({ cpuCores: null, ramBytes: null }); +}); + +it('matches pod metrics by namespace and sums complete containers', async () => { + vi.mocked(metricsAdapter.getPods).mockResolvedValue([ + { metadata: { name: 'pod', namespace: 'project', labels: { app: 'app' } }, status: { phase: 'Running' }, spec: { containers: [{ name: 'one' }] } }, + ] as never); + vi.mocked(metricsAdapter.getMetrics).mockResolvedValue([ + { metadata: { name: 'pod', namespace: 'other' }, timestamp: new Date().toISOString(), containers: [{ name: 'one', usage: { cpu: '9', memory: '9Mi' } }] }, + { metadata: { name: 'pod', namespace: 'project' }, timestamp: new Date().toISOString(), containers: [{ name: 'one', usage: { cpu: '250m', memory: '1Mi' } }] }, + ] as never); + const result = await appHistoryMetricsService.getAppMetrics([{ id: 'app', projectId: 'project' }]); + expect(result.get('app')).toMatchObject({ cpuCores: 0.25, ramBytes: 1048576 }); +}); + +it('preserves RAM when CPU measurement is missing', async () => { + vi.mocked(metricsAdapter.getPods).mockResolvedValue([ + { metadata: { name: 'pod', namespace: 'project', labels: { app: 'app' } }, status: { phase: 'Running' }, spec: { containers: [{ name: 'one' }] } }, + ] as never); + vi.mocked(metricsAdapter.getMetrics).mockResolvedValue([ + { metadata: { name: 'pod', namespace: 'project' }, timestamp: new Date().toISOString(), containers: [{ name: 'one', usage: { memory: '1Mi' } }] }, + ] as never); + const result = await appHistoryMetricsService.getAppMetrics([{ id: 'app', projectId: 'project' }]); + expect(result.get('app')).toMatchObject({ cpuCores: null, ramBytes: 1048576 }); +}); diff --git a/src/server/services/standalone-services/app-monitoring-history.service.ts b/src/server/services/standalone-services/app-monitoring-history.service.ts new file mode 100644 index 00000000..38163804 --- /dev/null +++ b/src/server/services/standalone-services/app-monitoring-history.service.ts @@ -0,0 +1,55 @@ +import { randomUUID } from 'crypto'; +import appHistoryAdapter from '@/server/adapter/app-history.adapter'; +import appHistoryMetricsService from './app-history-metrics.service'; +import appVolumeFilesystemService from './app-volume-filesystem.service'; +import scheduleService from './schedule.service'; +import { KubeObjectNameUtils } from '@/server/utils/kube-object-name.utils'; + +class AppMonitoringHistoryService { + private running = false; + + initialize() { + scheduleService.scheduleJob('app-monitoring-history', '*/5 * * * *', () => { void this.collect(); }); + void this.collect(); + } + + async collect() { + if (this.running) return; + this.running = true; + try { + const sampledAt = new Date(); + const { apps, volumes } = await appHistoryAdapter.getPopulation(); + const [metrics, filesystem, retention] = await Promise.all([ + appHistoryMetricsService.getAppMetrics(apps).catch(error => { console.error('App metrics collection failed', error); return new Map(); }), + appVolumeFilesystemService.getVolumeFilesystemUsage(volumes.map(volume => ({ ...volume, projectId: volume.app.projectId }))) + .catch(error => { console.error('Volume metrics collection failed', error); return new Map(); }), + appHistoryAdapter.getRetentionHours(), + ]); + const samples = new Map(); + const memberships: { sampleId: string; volumeId: string }[] = []; + for (const volume of volumes) { + const underlyingId = volume.sharedVolumeId || volume.id; + let sample = samples.get(underlyingId); + if (!sample) { + const pvcName = KubeObjectNameUtils.toPvcName(underlyingId); + const usage = filesystem.get(`${volume.app.projectId}/${pvcName}`); + sample = { id: randomUUID(), volumeId: underlyingId, sampledAt, status: usage?.status ?? 'unavailable', usedBytes: usage?.usedBytes ?? null, availableBytes: usage?.availableBytes ?? null, capacityBytes: usage?.capacityBytes ?? null, measuredAt: usage?.measuredAt ? new Date(usage.measuredAt) : null }; + samples.set(underlyingId, sample); + } + memberships.push({ sampleId: sample.id, volumeId: volume.id }); + } + await appHistoryAdapter.save( + apps.map(app => ({ appId: app.id, sampledAt, cpuCores: metrics.get(app.id)?.cpuCores ?? null, ramBytes: metrics.get(app.id)?.ramBytes ?? null, measuredAt: metrics.get(app.id)?.measuredAt ?? null })), + [...samples.values()], + memberships, + new Date(sampledAt.getTime() - retention * 60 * 60 * 1000), + ); + } catch (error) { + console.error('App monitoring history collection failed', error); + } finally { + this.running = false; + } + } +} +const appMonitoringHistoryService = new AppMonitoringHistoryService(); +export default appMonitoringHistoryService; diff --git a/src/server/services/standalone-services/app-monitoring-history.service.unit.spec.ts b/src/server/services/standalone-services/app-monitoring-history.service.unit.spec.ts new file mode 100644 index 00000000..ac9a54b5 --- /dev/null +++ b/src/server/services/standalone-services/app-monitoring-history.service.unit.spec.ts @@ -0,0 +1,52 @@ +vi.mock('@/server/adapter/app-history.adapter', () => ({ default: { getPopulation: vi.fn(), getRetentionHours: vi.fn(), save: vi.fn() } })); +vi.mock('./app-history-metrics.service', () => ({ default: { getAppMetrics: vi.fn() } })); +vi.mock('./app-volume-filesystem.service', () => ({ default: { getVolumeFilesystemUsage: vi.fn() } })); +vi.mock('./schedule.service', () => ({ default: { scheduleJob: vi.fn() } })); +import history from './app-monitoring-history.service'; +import adapter from '@/server/adapter/app-history.adapter'; +import metrics from './app-history-metrics.service'; +import filesystem from './app-volume-filesystem.service'; + +it('records a gap for failed App metrics while preserving successful volume data', async () => { + vi.mocked(adapter.getPopulation).mockResolvedValue({ apps: [{ id: 'app', projectId: 'p' }], volumes: [{ id: 'v', appId: 'app', sharedVolumeId: null, app: { projectId: 'p' } }] } as never); + vi.mocked(adapter.getRetentionHours).mockResolvedValue(24); + vi.mocked(metrics.getAppMetrics).mockRejectedValue(new Error('metrics unavailable')); + vi.mocked(filesystem.getVolumeFilesystemUsage).mockResolvedValue(new Map([['p/pvc-v', { pvcName: 'pvc-v', status: 'available', usedBytes: 4 }]])); + await history.collect(); + expect(adapter.save).toHaveBeenCalledWith( + [expect.objectContaining({ appId: 'app', cpuCores: null, ramBytes: null })], + [expect.objectContaining({ volumeId: 'v', status: 'available', usedBytes: 4 })], + [expect.objectContaining({ volumeId: 'v' })], + expect.any(Date), + ); +}); + +it('does not start a second collection while the first is running', async () => { + vi.clearAllMocks(); + let release!: (value: { apps: never[]; volumes: never[] }) => void; + vi.mocked(adapter.getPopulation).mockReturnValue(new Promise(resolve => { release = resolve; })); + vi.mocked(adapter.getRetentionHours).mockResolvedValue(24); + vi.mocked(metrics.getAppMetrics).mockResolvedValue(new Map()); + vi.mocked(filesystem.getVolumeFilesystemUsage).mockResolvedValue(new Map()); + const first = history.collect(); + await history.collect(); + expect(adapter.getPopulation).toHaveBeenCalledTimes(1); + release({ apps: [], volumes: [] }); + await first; +}); + + +it('stores one filesystem measurement for a shared Volume with a link for each mount', async () => { + vi.clearAllMocks(); + vi.mocked(adapter.getPopulation).mockResolvedValue({ apps: [], volumes: [ + { id: 'v', appId: 'app', sharedVolumeId: null, app: { projectId: 'p' } }, + { id: 'shared', appId: 'other', sharedVolumeId: 'v', app: { projectId: 'p' } }, + ] } as never); + vi.mocked(adapter.getRetentionHours).mockResolvedValue(24); + vi.mocked(metrics.getAppMetrics).mockResolvedValue(new Map()); + vi.mocked(filesystem.getVolumeFilesystemUsage).mockResolvedValue(new Map([['p/pvc-v', { pvcName: 'pvc-v', status: 'available', usedBytes: 4 }]])); + await history.collect(); + const [, samples, memberships] = vi.mocked(adapter.save).mock.calls[0]; + expect(samples).toHaveLength(1); + expect(memberships).toHaveLength(2); +}); diff --git a/src/server/services/standalone-services/app-volume-filesystem.service.ts b/src/server/services/standalone-services/app-volume-filesystem.service.ts new file mode 100644 index 00000000..9018d357 --- /dev/null +++ b/src/server/services/standalone-services/app-volume-filesystem.service.ts @@ -0,0 +1,64 @@ +import { V1PersistentVolumeClaim } from '@kubernetes/client-node'; +import volumeSummaryAdapter from '@/server/adapter/volume-summary.adapter'; +import { supportsVolumeFilesystemMetrics } from '../volume-filesystem-drivers'; +import { VolumeFilesystemUsage } from '@/shared/model/app-volume-monitoring-usage.model'; +import { KubeObjectNameUtils } from '@/server/utils/kube-object-name.utils'; + +class AppVolumeFilesystemService { + async getVolumeFilesystemUsage( + volumes: { id: string; sharedVolumeId: string | null; projectId: string }[], + getClaims: () => Promise = () => volumeSummaryAdapter.getPersistentVolumeClaims(), + ): Promise> { + const [pvcs, persistentVolumes, samples] = await Promise.all([ + getClaims().catch(() => []), + volumeSummaryAdapter.getPersistentVolumes().catch(() => []), + volumeSummaryAdapter.getSamples().catch(() => []), + ]); + const pvcByKey = new Map(pvcs.map(pvc => [`${pvc.metadata?.namespace}/${pvc.metadata?.name}`, pvc])); + const pvByName = new Map(persistentVolumes.map(pv => [pv.metadata?.name, pv])); + const result = new Map(); + const now = Date.now(); + for (const volume of volumes) { + const pvcName = KubeObjectNameUtils.toPvcName(volume.sharedVolumeId || volume.id); + const key = `${volume.projectId}/${pvcName}`; + if (result.has(key)) continue; + const pvc = pvcByKey.get(key); + const pv = pvc?.spec?.volumeName ? pvByName.get(pvc.spec.volumeName) : undefined; + if (!pv) { + result.set(key, { pvcName, status: 'unavailable' }); + continue; + } + if (!supportsVolumeFilesystemMetrics(pv)) { + result.set(key, { pvcName, status: 'unsupported' }); + continue; + } + const candidates = samples + .filter(sample => sample.pvcRef?.namespace === volume.projectId && sample.pvcRef.name === pvcName) + .filter(sample => { + const values = [sample.usedBytes, sample.availableBytes, sample.capacityBytes]; + return values.every(value => typeof value === 'number' && Number.isFinite(value) && value >= 0) + && !!sample.capacityBytes && Number.isFinite(Date.parse(sample.time ?? '')); + }) + .sort((a, b) => Date.parse(b.time!) - Date.parse(a.time!)); + const sample = candidates[0]; + const age = sample ? now - Date.parse(sample.time!) : Infinity; + if (!sample || age > 5 * 60 * 1000 || age < -60 * 1000) { + result.set(key, { pvcName, status: sample ? 'stale' : 'unavailable' }); + continue; + } + result.set(key, { + pvcName, + status: 'available', + usedBytes: sample.usedBytes, + availableBytes: sample.availableBytes, + capacityBytes: sample.capacityBytes, + measuredAt: sample.time, + }); + } + return result; + } + +} + +const appVolumeFilesystemService = new AppVolumeFilesystemService(); +export default appVolumeFilesystemService; diff --git a/src/shared/model/app-monitoring-history.model.ts b/src/shared/model/app-monitoring-history.model.ts new file mode 100644 index 00000000..2fb02b18 --- /dev/null +++ b/src/shared/model/app-monitoring-history.model.ts @@ -0,0 +1,20 @@ +export interface AppHistoryPoint { + time: number; + cpuCores: number | null; + ramBytes: number | null; + storageBytes: number | null; + availableVolumes: number; + unavailableVolumes: number; +} +export interface AppVolumeHistory { + id: string; + label: string; + points: { time: number; usedBytes: number | null; measuredAt: number | null }[]; +} +export interface AppMonitoringHistory { + points: AppHistoryPoint[]; + volumes: AppVolumeHistory[]; + latestCpuMeasurement: number | null; + latestRamMeasurement: number | null; + latestVolumeMeasurement: number | null; +} diff --git a/src/shared/model/generated-zod/app.ts b/src/shared/model/generated-zod/app.ts index 07769ec4..ba408b16 100644 --- a/src/shared/model/generated-zod/app.ts +++ b/src/shared/model/generated-zod/app.ts @@ -1,6 +1,6 @@ import * as z from "zod" -import { CompleteProject, RelatedProjectModel, CompleteAppDomain, RelatedAppDomainModel, CompleteAppPort, RelatedAppPortModel, CompleteAppNodePort, RelatedAppNodePortModel, CompleteAppVolume, RelatedAppVolumeModel, CompleteAppFileMount, RelatedAppFileMountModel, CompleteAppBasicAuth, RelatedAppBasicAuthModel, CompleteAppGitSshKey, RelatedAppGitSshKeyModel, CompleteAppNetworkPolicy, RelatedAppNetworkPolicyModel, CompleteRoleAppPermission, RelatedRoleAppPermissionModel, CompleteAgentNetworkPolicyRule, RelatedAgentNetworkPolicyRuleModel, CompleteAppNetworkPolicyRule, RelatedAppNetworkPolicyRuleModel } from "./index" +import { CompleteProject, RelatedProjectModel, CompleteAppDomain, RelatedAppDomainModel, CompleteAppPort, RelatedAppPortModel, CompleteAppNodePort, RelatedAppNodePortModel, CompleteAppVolume, RelatedAppVolumeModel, CompleteAppMonitoringSample, RelatedAppMonitoringSampleModel, CompleteAppFileMount, RelatedAppFileMountModel, CompleteAppBasicAuth, RelatedAppBasicAuthModel, CompleteAppGitSshKey, RelatedAppGitSshKeyModel, CompleteAppNetworkPolicy, RelatedAppNetworkPolicyModel, CompleteRoleAppPermission, RelatedRoleAppPermissionModel, CompleteAgentNetworkPolicyRule, RelatedAgentNetworkPolicyRuleModel, CompleteAppNetworkPolicyRule, RelatedAppNetworkPolicyRuleModel } from "./index" export const AppModel = z.object({ id: z.string(), @@ -60,6 +60,7 @@ export interface CompleteApp extends z.infer { appPorts: CompleteAppPort[] appNodePorts: CompleteAppNodePort[] appVolumes: CompleteAppVolume[] + monitoringSamples: CompleteAppMonitoringSample[] appFileMounts: CompleteAppFileMount[] appBasicAuths: CompleteAppBasicAuth[] appGitSshKey?: CompleteAppGitSshKey | null @@ -80,6 +81,7 @@ export const RelatedAppModel: z.ZodSchema = z.lazy(() => AppModel.e appPorts: RelatedAppPortModel.array(), appNodePorts: RelatedAppNodePortModel.array(), appVolumes: RelatedAppVolumeModel.array(), + monitoringSamples: RelatedAppMonitoringSampleModel.array(), appFileMounts: RelatedAppFileMountModel.array(), appBasicAuths: RelatedAppBasicAuthModel.array(), appGitSshKey: RelatedAppGitSshKeyModel.nullish(), diff --git a/src/shared/model/generated-zod/appmonitoringsample.ts b/src/shared/model/generated-zod/appmonitoringsample.ts new file mode 100644 index 00000000..8fc9c140 --- /dev/null +++ b/src/shared/model/generated-zod/appmonitoringsample.ts @@ -0,0 +1,25 @@ +import * as z from "zod" + +import { CompleteApp, RelatedAppModel } from "./index" + +export const AppMonitoringSampleModel = z.object({ + id: z.string(), + sampledAt: z.date(), + appId: z.string(), + cpuCores: z.number().nullish(), + ramBytes: z.number().nullish(), + measuredAt: z.date().nullish(), +}) + +export interface CompleteAppMonitoringSample extends z.infer { + app: CompleteApp +} + +/** + * RelatedAppMonitoringSampleModel contains all relations on your model in addition to the scalars + * + * NOTE: Lazy required in case of potential circular dependencies within schema + */ +export const RelatedAppMonitoringSampleModel: z.ZodSchema = z.lazy(() => AppMonitoringSampleModel.extend({ + app: RelatedAppModel, +})) diff --git a/src/shared/model/generated-zod/appvolume.ts b/src/shared/model/generated-zod/appvolume.ts index 8255d96c..c0b401ca 100644 --- a/src/shared/model/generated-zod/appvolume.ts +++ b/src/shared/model/generated-zod/appvolume.ts @@ -1,6 +1,6 @@ import * as z from "zod" -import { CompleteApp, RelatedAppModel, CompleteVolumeBackup, RelatedVolumeBackupModel } from "./index" +import { CompleteApp, RelatedAppModel, CompleteVolumeBackup, RelatedVolumeBackupModel, CompleteAppVolumeMonitoringSample, RelatedAppVolumeMonitoringSampleModel, CompleteAppVolumeMonitoringMembership, RelatedAppVolumeMonitoringMembershipModel } from "./index" export const AppVolumeModel = z.object({ id: z.string(), @@ -18,6 +18,8 @@ export const AppVolumeModel = z.object({ export interface CompleteAppVolume extends z.infer { app: CompleteApp volumeBackups: CompleteVolumeBackup[] + monitoringSamples: CompleteAppVolumeMonitoringSample[] + monitoringMemberships: CompleteAppVolumeMonitoringMembership[] sharedVolume?: CompleteAppVolume | null sharedVolumes: CompleteAppVolume[] } @@ -30,6 +32,8 @@ export interface CompleteAppVolume extends z.infer { export const RelatedAppVolumeModel: z.ZodSchema = z.lazy(() => AppVolumeModel.extend({ app: RelatedAppModel, volumeBackups: RelatedVolumeBackupModel.array(), + monitoringSamples: RelatedAppVolumeMonitoringSampleModel.array(), + monitoringMemberships: RelatedAppVolumeMonitoringMembershipModel.array(), sharedVolume: RelatedAppVolumeModel.nullish(), sharedVolumes: RelatedAppVolumeModel.array(), })) diff --git a/src/shared/model/generated-zod/appvolumemonitoringmembership.ts b/src/shared/model/generated-zod/appvolumemonitoringmembership.ts new file mode 100644 index 00000000..a075f7ca --- /dev/null +++ b/src/shared/model/generated-zod/appvolumemonitoringmembership.ts @@ -0,0 +1,23 @@ +import * as z from "zod" + +import { CompleteAppVolumeMonitoringSample, RelatedAppVolumeMonitoringSampleModel, CompleteAppVolume, RelatedAppVolumeModel } from "./index" + +export const AppVolumeMonitoringMembershipModel = z.object({ + sampleId: z.string(), + volumeId: z.string(), +}) + +export interface CompleteAppVolumeMonitoringMembership extends z.infer { + sample: CompleteAppVolumeMonitoringSample + volume: CompleteAppVolume +} + +/** + * RelatedAppVolumeMonitoringMembershipModel contains all relations on your model in addition to the scalars + * + * NOTE: Lazy required in case of potential circular dependencies within schema + */ +export const RelatedAppVolumeMonitoringMembershipModel: z.ZodSchema = z.lazy(() => AppVolumeMonitoringMembershipModel.extend({ + sample: RelatedAppVolumeMonitoringSampleModel, + volume: RelatedAppVolumeModel, +})) diff --git a/src/shared/model/generated-zod/appvolumemonitoringsample.ts b/src/shared/model/generated-zod/appvolumemonitoringsample.ts new file mode 100644 index 00000000..b86a0336 --- /dev/null +++ b/src/shared/model/generated-zod/appvolumemonitoringsample.ts @@ -0,0 +1,29 @@ +import * as z from "zod" + +import { CompleteAppVolume, RelatedAppVolumeModel, CompleteAppVolumeMonitoringMembership, RelatedAppVolumeMonitoringMembershipModel } from "./index" + +export const AppVolumeMonitoringSampleModel = z.object({ + id: z.string(), + sampledAt: z.date(), + volumeId: z.string(), + status: z.string(), + usedBytes: z.number().nullish(), + availableBytes: z.number().nullish(), + capacityBytes: z.number().nullish(), + measuredAt: z.date().nullish(), +}) + +export interface CompleteAppVolumeMonitoringSample extends z.infer { + volume: CompleteAppVolume + memberships: CompleteAppVolumeMonitoringMembership[] +} + +/** + * RelatedAppVolumeMonitoringSampleModel contains all relations on your model in addition to the scalars + * + * NOTE: Lazy required in case of potential circular dependencies within schema + */ +export const RelatedAppVolumeMonitoringSampleModel: z.ZodSchema = z.lazy(() => AppVolumeMonitoringSampleModel.extend({ + volume: RelatedAppVolumeModel, + memberships: RelatedAppVolumeMonitoringMembershipModel.array(), +})) diff --git a/src/shared/model/generated-zod/index.ts b/src/shared/model/generated-zod/index.ts index 4f65c0da..9a157304 100644 --- a/src/shared/model/generated-zod/index.ts +++ b/src/shared/model/generated-zod/index.ts @@ -32,3 +32,6 @@ export * from "./parameter" export * from "./s3target" export * from "./volumebackup" export * from "./appbasicauth" +export * from "./appmonitoringsample" +export * from "./appvolumemonitoringsample" +export * from "./appvolumemonitoringmembership" diff --git a/src/shared/utils/constants.ts b/src/shared/utils/constants.ts index b3ce1d6a..6df0a825 100644 --- a/src/shared/utils/constants.ts +++ b/src/shared/utils/constants.ts @@ -1,4 +1,5 @@ export class Constants { + static readonly APP_MONITORING_HISTORY_RETENTION_HOURS = 'APP_MONITORING_HISTORY_RETENTION_HOURS'; static readonly TERMINAL_MAX_SESSIONS_PER_SOCKET = 4; static readonly TERMINAL_MAX_OPENS_PER_MINUTE = 20; static readonly TERMINAL_MAX_INPUT_BYTES = 16384; From b0758ae0fa865e4228666b3f8a5f4992c4c2dd5a Mon Sep 17 00:00:00 2001 From: biersoeckli Date: Sun, 8 Nov 2026 12:58:30 +0000 Subject: [PATCH 05/11] fix: show gaps when monitoring collection stops --- ...app-monitoring-history.integration.spec.ts | 3 +- .../deployments/history-actions.unit.spec.ts | 37 ++++ ...nitoring-history-actions.ts => actions.ts} | 0 .../monitoring-history-settings.tsx | 57 ++++-- .../custom/app-monitoring-history.tsx | 184 +++++++++++++----- .../app-monitoring-history.unit.spec.ts | 20 ++ .../app-monitoring-history-query.service.ts | 21 +- ...itoring-history-query.service.unit.spec.ts | 16 ++ .../app-monitoring-history.service.ts | 45 ++++- ...pp-monitoring-history.service.unit.spec.ts | 12 ++ .../model/app-monitoring-history.model.ts | 1 + 11 files changed, 318 insertions(+), 78 deletions(-) create mode 100644 src/app/project/[projectId]/apps/details/deployments/history-actions.unit.spec.ts rename src/app/settings/platform/operations/{monitoring-history-actions.ts => actions.ts} (100%) diff --git a/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts b/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts index bb665eb0..50a3f4c6 100644 --- a/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts +++ b/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts @@ -15,7 +15,8 @@ describe('App monitoring history persistence', () => { { appId: 'app', sampledAt: new Date(cutoff.getTime() - 1), cpuCores: 1, ramBytes: 2 }, { appId: 'app', sampledAt: cutoff, cpuCores: 3, ramBytes: 4 }, ], [], [], cutoff); - expect((await history.getHistory(['app'])).points).toMatchObject([{ cpuCores: 3, ramBytes: 4 }]); + expect((await history.getHistory(['app'])).points.filter(point => !point.isGap)) + .toMatchObject([{ cpuCores: 3, ramBytes: 4 }]); await db.app.delete({ where: { id: 'app' } }); expect((await history.getHistory(['app'])).points).toEqual([]); }); diff --git a/src/app/project/[projectId]/apps/details/deployments/history-actions.unit.spec.ts b/src/app/project/[projectId]/apps/details/deployments/history-actions.unit.spec.ts new file mode 100644 index 00000000..16227d84 --- /dev/null +++ b/src/app/project/[projectId]/apps/details/deployments/history-actions.unit.spec.ts @@ -0,0 +1,37 @@ +vi.mock('@/server/services/app-monitoring-history-query.service', () => ({ + default: { getHistory: vi.fn().mockResolvedValue({ points: [] }) }, +})); +vi.mock('@/server/services/app.service', () => ({ default: {} })); +vi.mock('@/server/services/build.service', () => ({ default: {} })); +vi.mock('@/server/services/deployment.service', () => ({ default: {} })); +vi.mock('@/server/services/monitoring.service', () => ({ default: {} })); +vi.mock('@/server/services/pod.service', () => ({ default: {} })); +vi.mock('@/server/utils/action-wrapper.utils', () => ({ + isAuthorizedReadForApp: vi.fn(), + isAuthorizedWriteForApp: vi.fn(), + isAuthorizedWriteForWorkload: vi.fn(), + simpleAction: async (callback: () => Promise) => { + try { return { status: 'success', data: await callback() }; } + catch { return { status: 'error' }; } + }, +})); +import history from '@/server/services/app-monitoring-history-query.service'; +import { isAuthorizedReadForApp } from '@/server/utils/action-wrapper.utils'; +import { getAppMonitoringHistory } from './actions'; + +beforeEach(() => vi.clearAllMocks()); + +it('returns history for an App when the user has read access', async () => { + vi.mocked(isAuthorizedReadForApp).mockResolvedValue({} as never); + const result = await getAppMonitoringHistory('allowed'); + expect(result.status).toBe('success'); + expect(isAuthorizedReadForApp).toHaveBeenCalledWith('allowed'); + expect(history.getHistory).toHaveBeenCalledWith(['allowed']); +}); + +it('does not query history when the user cannot read the App', async () => { + vi.mocked(isAuthorizedReadForApp).mockRejectedValue(new Error('not authorized')); + const result = await getAppMonitoringHistory('denied'); + expect(result.status).toBe('error'); + expect(history.getHistory).not.toHaveBeenCalled(); +}); diff --git a/src/app/settings/platform/operations/monitoring-history-actions.ts b/src/app/settings/platform/operations/actions.ts similarity index 100% rename from src/app/settings/platform/operations/monitoring-history-actions.ts rename to src/app/settings/platform/operations/actions.ts diff --git a/src/app/settings/platform/operations/monitoring-history-settings.tsx b/src/app/settings/platform/operations/monitoring-history-settings.tsx index 2be84d1c..9d21b7fb 100644 --- a/src/app/settings/platform/operations/monitoring-history-settings.tsx +++ b/src/app/settings/platform/operations/monitoring-history-settings.tsx @@ -8,7 +8,7 @@ import { Button } from '@/components/ui/button'; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'; import { Form, FormControl, FormField, FormItem, FormLabel, FormMessage } from '@/components/ui/form'; import { Input } from '@/components/ui/input'; -import { getMonitoringHistoryRetention, setMonitoringHistoryRetention } from './monitoring-history-actions'; +import { getMonitoringHistoryRetention, setMonitoringHistoryRetention } from './actions'; const schema = z.object({ hours: z.coerce.number().int().min(1).max(168) }); @@ -24,26 +24,51 @@ export default function MonitoringHistorySettings() { if (result.status === 'success' && result.data !== undefined) { form.reset({ hours: result.data }); setLoaded(true); - } else setMessage('Unable to load retention settings.'); + } else { + setMessage('Unable to load retention settings.'); + } }).catch(() => setMessage('Unable to load retention settings.')); }, [form]); return - App monitoring historyKeep measurements for 1 to 168 hours. Default: 24 hours. + + App monitoring history + Keep measurements for 1 to 168 hours. Default: 24 hours. +
- { - try { - const result = await setMonitoringHistoryRetention(hours); - setMessage(result.status === 'success' ? 'Retention saved. Older samples are removed during the next collection.' : result.message ?? 'Unable to save retention.'); - } catch { - setMessage('Unable to save retention.'); - } - })}> - - Retention (hours) - - - } /> + { + try { + const result = await setMonitoringHistoryRetention(hours); + setMessage(result.status === 'success' + ? 'Retention saved. Older samples are removed during the next collection.' + : result.message ?? 'Unable to save retention.'); + } catch { + setMessage('Unable to save retention.'); + } + })} + > + ( + + Retention (hours) + + + + + + )} + /> {message &&

{message}

} diff --git a/src/components/custom/app-monitoring-history.tsx b/src/components/custom/app-monitoring-history.tsx index c0d69a0f..f176780f 100644 --- a/src/components/custom/app-monitoring-history.tsx +++ b/src/components/custom/app-monitoring-history.tsx @@ -8,35 +8,68 @@ import { AppMonitoringHistory } from '@/shared/model/app-monitoring-history.mode import { getAppMonitoringHistory as getAllHistory } from '@/app/monitoring/actions'; import { getAppMonitoringHistory as getOneHistory } from '@/app/project/[projectId]/apps/details/deployments/actions'; -function HistoryChart({ title, data, dataKey, unit }: { title: string; data: object[]; dataKey: string; unit: string }) { - return
-

{title} ({unit})

-
- - - - new Date(value).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })} /> - - { - if (!active || !payload?.length) return null; - const item = payload[0]; - const sample = item.payload as { availableVolumes?: number; unavailableVolumes?: number }; - return
-

{new Date(Number(label)).toLocaleString()}

-

{item.value} {unit}

- {dataKey === 'storageBytes' &&

{sample.availableVolumes} available, {sample.unavailableVolumes} unavailable{sample.unavailableVolumes ? ' (incomplete total)' : ''}

} -
; - }} /> - - - +function HistoryChart({ title, data, dataKey, unit }: { + title: string; + data: object[]; + dataKey: string; + unit: string; +}) { + const visibleSamples = data.filter(point => (point as Record)[dataKey] != null); + return ( +
+

{title} ({unit})

+
+ + + + new Date(value).toLocaleTimeString([], { + hour: '2-digit', minute: '2-digit', + })} + /> + + { + if (!active || !payload?.length) return null; + const item = payload[0]; + const sample = item.payload as { + availableVolumes?: number; + unavailableVolumes?: number; + }; + return ( +
+

{new Date(Number(label)).toLocaleString()}

+

{item.value} {unit}

+ {dataKey === 'storageBytes' && ( +

+ {sample.availableVolumes} available, {sample.unavailableVolumes} unavailable + {sample.unavailableVolumes ? ' (incomplete total)' : ''} +

+ )} +
+ ); + }} /> + + + +
-
; + ); } function freshness(time: number | null) { if (!time) return 'Measurement unavailable'; - return `Latest successful measurement: ${new Date(time).toLocaleString()}${Date.now() - time > 10 * 60_000 ? ' (stale)' : ''}`; + const stale = Date.now() - time > 10 * 60_000 ? ' (stale)' : ''; + return `Latest successful measurement: ${new Date(time).toLocaleString()}${stale}`; } export default function AppMonitoringHistoryView({ appId }: { appId?: string }) { @@ -45,7 +78,9 @@ export default function AppMonitoringHistoryView({ appId }: { appId?: string }) const refresh = useCallback(async () => { try { const response = appId ? await getOneHistory(appId) : await getAllHistory(); - if (response.status !== 'success' || !response.data) throw new Error(response.message ?? 'Unable to load history'); + if (response.status !== 'success' || !response.data) { + throw new Error(response.message ?? 'Unable to load history'); + } setHistory(response.data); setError(false); } catch { @@ -53,31 +88,78 @@ export default function AppMonitoringHistoryView({ appId }: { appId?: string }) } }, [appId]); usePolling(refresh, { intervalMs: 60_000, runImmediately: true }); - const latest = history?.points[history.points.length - 1]; - return - {appId ? 'App history' : 'App totals history'} - - {error &&

Unable to load monitoring history.

} - {!history && !error &&

Loading monitoring history…

} - {history && history.points.length === 0 &&

No measurements yet. History is collected every five minutes.

} - {history && history.points.length > 0 && <> - {history.points.length === 1 &&

One sample collected. More samples will show a trend.

} -

CPU is an average over the Kubernetes metrics source window; short peaks may be missed.

- -

{freshness(history.latestCpuMeasurement)}

- -

{freshness(history.latestRamMeasurement)}

-

{freshness(history.latestVolumeMeasurement)}

- {!appId && <> - - {latest &&

Latest sample: {latest.availableVolumes} available and {latest.unavailableVolumes} unavailable volumes{latest.unavailableVolumes ? ' (incomplete total)' : ''}.

} + const latest = history?.points.findLast(point => !point.isGap); + const collectionStopped = history?.points.at(-1)?.isGap; + const sampleCount = history?.points.filter(point => !point.isGap).length ?? 0; + + return ( + + + {appId ? 'App history' : 'App totals history'} + + + {error &&

Unable to load monitoring history.

} + {!history && !error &&

Loading monitoring history…

} + {history && history.points.length === 0 && ( +

No measurements yet. History is collected every five minutes.

+ )} + {history && history.points.length > 0 && <> + {collectionStopped && ( +

Collection has stopped or is delayed. The chart shows a gap after the last sample.

+ )} + {sampleCount === 1 &&

One sample collected. More samples will show a trend.

} +

+ CPU is an average over the Kubernetes metrics source window; short peaks may be missed. +

+ +

{freshness(history.latestCpuMeasurement)}

+ +

{freshness(history.latestRamMeasurement)}

+

{freshness(history.latestVolumeMeasurement)}

+ {!appId && <> + + {latest && ( +

+ Latest sample: {latest.availableVolumes} available and {latest.unavailableVolumes} unavailable volumes + {latest.unavailableVolumes ? ' (incomplete total)' : ''}. +

+ )} + } + {appId && history.volumes.length === 0 &&

No App Volume measurements.

} + {appId && history.volumes.map(volume => { + const latestMeasurement = volume.points.reduce((latest, point) => { + if (point.usedBytes === null || !point.measuredAt) return latest; + return Math.max(latest ?? 0, point.measuredAt); + }, null); + return ( +
+ +

{freshness(latestMeasurement)}

+
+ ); + })} } - {appId && history.volumes.length === 0 &&

No App Volume measurements.

} - {appId && history.volumes.map(volume =>
- -

{freshness(volume.points.reduce((latest, point) => point.usedBytes !== null && point.measuredAt ? Math.max(latest ?? 0, point.measuredAt) : latest, null))}

-
)} - } - - ; + + + ); } diff --git a/src/components/custom/app-monitoring-history.unit.spec.ts b/src/components/custom/app-monitoring-history.unit.spec.ts index dc806277..6233e76b 100644 --- a/src/components/custom/app-monitoring-history.unit.spec.ts +++ b/src/components/custom/app-monitoring-history.unit.spec.ts @@ -29,3 +29,23 @@ it('shows an error when history cannot be loaded', async () => { render(React.createElement(AppMonitoringHistoryView)); expect((await screen.findByRole('alert')).textContent).toContain('Unable to load monitoring history.'); }); + +it('shows a stopped collection gap without treating it as a new volume sample', async () => { + const now = Date.now(); + vi.mocked(getAppMonitoringHistory).mockResolvedValue({ + status: 'success', + data: { + points: [ + { time: now - 900_000, cpuCores: 1, ramBytes: 2, storageBytes: 10, availableVolumes: 1, unavailableVolumes: 2 }, + { time: now, isGap: true, cpuCores: null, ramBytes: null, storageBytes: null, availableVolumes: 0, unavailableVolumes: 0 }, + ], + volumes: [], + latestCpuMeasurement: null, + latestRamMeasurement: null, + latestVolumeMeasurement: null, + }, + } as never); + render(React.createElement(AppMonitoringHistoryView)); + expect(await screen.findByText(/Collection has stopped or is delayed/)).toBeTruthy(); + expect(screen.getByText(/1 available and 2 unavailable volumes/)).toBeTruthy(); +}); diff --git a/src/server/services/app-monitoring-history-query.service.ts b/src/server/services/app-monitoring-history-query.service.ts index b197d57e..a143223d 100644 --- a/src/server/services/app-monitoring-history-query.service.ts +++ b/src/server/services/app-monitoring-history-query.service.ts @@ -27,7 +27,7 @@ class AppMonitoringHistoryQueryService { const volumeSeries = new Map(); for (const time of [...times].sort((a, b) => a - b)) { if (points.length && time - points[points.length - 1].time > 7 * 60_000) { - points.push({ time: points[points.length - 1].time + 5 * 60_000, cpuCores: null, ramBytes: null, storageBytes: null, availableVolumes: 0, unavailableVolumes: 0 }); + points.push({ time: points[points.length - 1].time + 5 * 60_000, isGap: true, cpuCores: null, ramBytes: null, storageBytes: null, availableVolumes: 0, unavailableVolumes: 0 }); } const currentApps = appsByTime.get(time) ?? []; const currentVolumes = volumesByTime.get(time) ?? []; @@ -64,6 +64,25 @@ class AppMonitoringHistoryQueryService { unavailableVolumes: uniqueVolumes.size - available.length, }); } + const now = Date.now(); + const lastPoint = points.at(-1); + if (lastPoint && now - lastPoint.time > 7 * 60_000) { + points.push({ + time: now, + isGap: true, + cpuCores: null, + ramBytes: null, + storageBytes: null, + availableVolumes: 0, + unavailableVolumes: 0, + }); + } + for (const volume of volumeSeries.values()) { + const lastSample = volume.points.at(-1); + if (lastSample && now - lastSample.time > 7 * 60_000) { + volume.points.push({ time: now, usedBytes: null, measuredAt: null }); + } + } return { points, volumes: [...volumeSeries.values()], latestCpuMeasurement, latestRamMeasurement, latestVolumeMeasurement }; } } diff --git a/src/server/services/app-monitoring-history-query.service.unit.spec.ts b/src/server/services/app-monitoring-history-query.service.unit.spec.ts index dafccf2e..bcbcc8ce 100644 --- a/src/server/services/app-monitoring-history-query.service.unit.spec.ts +++ b/src/server/services/app-monitoring-history-query.service.unit.spec.ts @@ -23,3 +23,19 @@ it('inserts a gap across collection downtime', async () => { const result = await historyService.getHistory(['app']); expect(result.points.map(point => point.cpuCores)).toEqual([1, null, 1]); }); + +it('shows a trailing gap while collection remains stopped', async () => { + const now = Date.now(); + const sampledAt = new Date(now - 30 * 60_000); + vi.mocked(adapter.getHistory).mockResolvedValue([[{ + id: 'one', appId: 'app', sampledAt, cpuCores: 1, ramBytes: 100, measuredAt: sampledAt, + }], [{ + id: 'volume-sample', volumeId: 'volume', sampledAt, status: 'available', usedBytes: 42, + availableBytes: 58, capacityBytes: 100, measuredAt: sampledAt, + volume: { appId: 'app', sharedVolumeId: null, containerMountPath: '/data', app: { projectId: 'project' } }, + }]]); + const result = await historyService.getHistory(['app']); + expect(result.points.at(-1)).toMatchObject({ time: expect.any(Number), cpuCores: null, ramBytes: null, storageBytes: null, isGap: true }); + expect(result.points.at(-1)!.time).toBeGreaterThanOrEqual(now); + expect(result.volumes[0].points.at(-1)).toMatchObject({ usedBytes: null }); +}); diff --git a/src/server/services/standalone-services/app-monitoring-history.service.ts b/src/server/services/standalone-services/app-monitoring-history.service.ts index 38163804..230bbbdc 100644 --- a/src/server/services/standalone-services/app-monitoring-history.service.ts +++ b/src/server/services/standalone-services/app-monitoring-history.service.ts @@ -1,15 +1,20 @@ import { randomUUID } from 'crypto'; +import { Prisma } from '@prisma/client'; import appHistoryAdapter from '@/server/adapter/app-history.adapter'; +import { KubeObjectNameUtils } from '@/server/utils/kube-object-name.utils'; import appHistoryMetricsService from './app-history-metrics.service'; import appVolumeFilesystemService from './app-volume-filesystem.service'; import scheduleService from './schedule.service'; -import { KubeObjectNameUtils } from '@/server/utils/kube-object-name.utils'; + +type VolumeSample = Prisma.AppVolumeMonitoringSampleCreateManyInput & { id: string }; class AppMonitoringHistoryService { private running = false; initialize() { - scheduleService.scheduleJob('app-monitoring-history', '*/5 * * * *', () => { void this.collect(); }); + scheduleService.scheduleJob('app-monitoring-history', '*/5 * * * *', () => { + void this.collect(); + }); void this.collect(); } @@ -20,26 +25,48 @@ class AppMonitoringHistoryService { const sampledAt = new Date(); const { apps, volumes } = await appHistoryAdapter.getPopulation(); const [metrics, filesystem, retention] = await Promise.all([ - appHistoryMetricsService.getAppMetrics(apps).catch(error => { console.error('App metrics collection failed', error); return new Map(); }), - appVolumeFilesystemService.getVolumeFilesystemUsage(volumes.map(volume => ({ ...volume, projectId: volume.app.projectId }))) - .catch(error => { console.error('Volume metrics collection failed', error); return new Map(); }), + appHistoryMetricsService.getAppMetrics(apps).catch(error => { + console.error('App metrics collection failed', error); + return new Map(); + }), + appVolumeFilesystemService.getVolumeFilesystemUsage( + volumes.map(volume => ({ ...volume, projectId: volume.app.projectId })), + ).catch(error => { + console.error('Volume metrics collection failed', error); + return new Map(); + }), appHistoryAdapter.getRetentionHours(), ]); - const samples = new Map(); - const memberships: { sampleId: string; volumeId: string }[] = []; + const samples = new Map(); + const memberships: Prisma.AppVolumeMonitoringMembershipCreateManyInput[] = []; for (const volume of volumes) { const underlyingId = volume.sharedVolumeId || volume.id; let sample = samples.get(underlyingId); if (!sample) { const pvcName = KubeObjectNameUtils.toPvcName(underlyingId); const usage = filesystem.get(`${volume.app.projectId}/${pvcName}`); - sample = { id: randomUUID(), volumeId: underlyingId, sampledAt, status: usage?.status ?? 'unavailable', usedBytes: usage?.usedBytes ?? null, availableBytes: usage?.availableBytes ?? null, capacityBytes: usage?.capacityBytes ?? null, measuredAt: usage?.measuredAt ? new Date(usage.measuredAt) : null }; + sample = { + id: randomUUID(), + volumeId: underlyingId, + sampledAt, + status: usage?.status ?? 'unavailable', + usedBytes: usage?.usedBytes ?? null, + availableBytes: usage?.availableBytes ?? null, + capacityBytes: usage?.capacityBytes ?? null, + measuredAt: usage?.measuredAt ? new Date(usage.measuredAt) : null, + }; samples.set(underlyingId, sample); } memberships.push({ sampleId: sample.id, volumeId: volume.id }); } await appHistoryAdapter.save( - apps.map(app => ({ appId: app.id, sampledAt, cpuCores: metrics.get(app.id)?.cpuCores ?? null, ramBytes: metrics.get(app.id)?.ramBytes ?? null, measuredAt: metrics.get(app.id)?.measuredAt ?? null })), + apps.map(app => ({ + appId: app.id, + sampledAt, + cpuCores: metrics.get(app.id)?.cpuCores ?? null, + ramBytes: metrics.get(app.id)?.ramBytes ?? null, + measuredAt: metrics.get(app.id)?.measuredAt ?? null, + })), [...samples.values()], memberships, new Date(sampledAt.getTime() - retention * 60 * 60 * 1000), diff --git a/src/server/services/standalone-services/app-monitoring-history.service.unit.spec.ts b/src/server/services/standalone-services/app-monitoring-history.service.unit.spec.ts index ac9a54b5..1a5e3913 100644 --- a/src/server/services/standalone-services/app-monitoring-history.service.unit.spec.ts +++ b/src/server/services/standalone-services/app-monitoring-history.service.unit.spec.ts @@ -50,3 +50,15 @@ it('stores one filesystem measurement for a shared Volume with a link for each m expect(samples).toHaveLength(1); expect(memberships).toHaveLength(2); }); + +it('collects promptly on startup and resumes collection after a restart', async () => { + vi.clearAllMocks(); + vi.mocked(adapter.getPopulation).mockResolvedValue({ apps: [], volumes: [] }); + vi.mocked(adapter.getRetentionHours).mockResolvedValue(24); + vi.mocked(metrics.getAppMetrics).mockResolvedValue(new Map()); + vi.mocked(filesystem.getVolumeFilesystemUsage).mockResolvedValue(new Map()); + history.initialize(); + await vi.waitFor(() => expect(adapter.save).toHaveBeenCalledTimes(1)); + history.initialize(); + await vi.waitFor(() => expect(adapter.save).toHaveBeenCalledTimes(2)); +}); diff --git a/src/shared/model/app-monitoring-history.model.ts b/src/shared/model/app-monitoring-history.model.ts index 2fb02b18..a7d2ad99 100644 --- a/src/shared/model/app-monitoring-history.model.ts +++ b/src/shared/model/app-monitoring-history.model.ts @@ -1,5 +1,6 @@ export interface AppHistoryPoint { time: number; + isGap?: boolean; cpuCores: number | null; ramBytes: number | null; storageBytes: number | null; From b3bf03bcd53e78e5d36657c860080fdeadf0f818 Mon Sep 17 00:00:00 2001 From: biersoeckli Date: Sun, 8 Nov 2026 14:38:27 +0000 Subject: [PATCH 06/11] refactor: migrate app history adapter to a service structure --- .../issues/01-storage-filesystem-usage.md | 54 ----- ...app-monitoring-history.integration.spec.ts | 2 +- src/app/monitoring/actions.ts | 13 +- src/app/monitoring/app-volumes-monitoring.tsx | 9 +- .../app-volumes-monitoring.unit.spec.ts | 2 +- .../monitoring/history-actions.unit.spec.ts | 16 -- src/app/monitoring/monitoring-client.tsx | 13 +- src/app/monitoring/monitoring-nodes.tsx | 228 +----------------- src/app/monitoring/page.tsx | 6 +- .../apps/details/app-details-drawer.tsx | 8 +- .../app-details-deployments-tab.tsx | 2 +- .../deployments/webhook-deployment.tsx | 25 +- .../apps/details/runtime/monitoring-app.tsx | 46 +--- .../backups/app-details-backups-tab.tsx | 4 +- .../apps/details/volumes/volumes-list.tsx | 6 +- .../settings/platform/operations/actions.ts | 6 +- .../custom/app-monitoring-history.tsx | 214 ++++++++-------- .../app-monitoring-history.unit.spec.ts | 40 +-- src/server/adapter/app-history.adapter.ts | 53 ---- .../app-monitoring-history-query.service.ts | 8 +- ...itoring-history-query.service.unit.spec.ts | 5 +- src/server/services/cluster.service.ts | 9 + .../services/cluster.service.unit.spec.ts | 17 +- ...rics.service.ts => app-history.service.ts} | 55 ++++- ...ec.ts => app-history.service.unit.spec.ts} | 4 +- .../app-monitoring-history.service.ts | 11 +- ...pp-monitoring-history.service.unit.spec.ts | 7 +- .../model/app-monitoring-history.model.ts | 1 + 28 files changed, 304 insertions(+), 560 deletions(-) delete mode 100644 .scratch/storage-filesystem-usage/issues/01-storage-filesystem-usage.md delete mode 100644 src/app/monitoring/history-actions.unit.spec.ts delete mode 100644 src/server/adapter/app-history.adapter.ts rename src/server/services/standalone-services/{app-history-metrics.service.ts => app-history.service.ts} (51%) rename src/server/services/standalone-services/{app-history-metrics.service.unit.spec.ts => app-history.service.unit.spec.ts} (94%) diff --git a/.scratch/storage-filesystem-usage/issues/01-storage-filesystem-usage.md b/.scratch/storage-filesystem-usage/issues/01-storage-filesystem-usage.md deleted file mode 100644 index 64a33b0b..00000000 --- a/.scratch/storage-filesystem-usage/issues/01-storage-filesystem-usage.md +++ /dev/null @@ -1,54 +0,0 @@ -Status: done - -# StorageClass-independent App Volume filesystem usage - -## Problem Statement - -As a QuickStack user, I want to see how full an App Volume's filesystem is. Existing screens use Longhorn actual size, which includes snapshot-related storage and does not describe filesystem usage. It also ties the feature to Longhorn. - -## Solution - -Read App Volume filesystem usage through the Kubernetes API and Kubelet Summary API. Show reliable filesystem usage in existing App Volume screens and the Monitoring overview, independent of the Longhorn API. Clearly show unavailable usage where a trustworthy measurement cannot be obtained. - -## User Stories - -1. As an App owner, I want to see my App Volume's used filesystem space, so that I can judge remaining space. -2. As an App owner, I want to see filesystem capacity and available space, so that the usage has context. -3. As an App owner, I want a percentage based on filesystem usage and capacity, so that I can recognize a nearly full volume. -4. As an App owner, I want the existing App Volume UI to use the new measurements, so that it no longer confuses snapshot storage with filesystem usage. -5. As an operator, I want the Monitoring overview to show App Volume usage, so that I can find full volumes across projects. -6. As an operator, I want a shared App Volume shown at each relevant App mount, so that its users can see its usage. -7. As an operator, I want shared volumes counted once in totals, so that totals do not double-count usage. -8. As a user, I want missing, stale, unsupported, and inaccessible measurements identified, so that they are not mistaken for zero usage. -9. As a user, I want valid usage from supported volumes even when other volumes cannot be measured, so that partial failures do not hide useful information. -10. As a maintainer, I want a driver-specific extension point, so that additional storage drivers can be supported after their metrics are verified. -11. As an operator, I want local-path and unverified drivers marked unavailable, so that host filesystem usage is not presented as exclusive PVC usage. -12. As an operator, I want volume measurements to work across cluster nodes, so that App placement does not hide metrics. - -## Implementation Decisions - -- Replace Longhorn-derived filesystem usage for all existing consumers with a shared monitoring-service interface backed by node Summary API data accessed through the Kubernetes API. Do not require direct Longhorn API or pod access for these measurements. Other Longhorn features are unaffected. -- Map Summary volume data by namespace and PVC reference, discover nodes and associate values with configured App Volumes, and resolve shared App Volumes to the underlying PVC. Deduplicate that identity for totals, including across pods and nodes. -- Introduce a common storage-driver interface with driver-specific implementations. Determine the effective driver from the bound persistent volume and its provisioner/driver metadata, not merely the StorageClass name. Support verified Longhorn filesystem semantics initially. Treat local-path, unknown and unverified drivers as unavailable; make it possible to add verified drivers later. -- Use reported filesystem used, available and capacity values with their measurement time. Do not substitute configured PVC capacity, Longhorn actual size, host-wide values, zero, or stale values as if they were a fresh filesystem measurement. Define and enforce a bounded freshness policy; communicate unavailable or stale data and preserve available results on partial failure. -- Include configured App Volumes even when unbound, unmounted, unsupported, or missing metrics. Handle zero or missing capacity safely. Sum unique volumes and identify incomplete totals rather than presenting them as complete. -- Account for the existing installation's broad node proxy access; avoid introducing broader access. Node proxy is privileged, so do not expose a generic proxy endpoint to users. -- Existing App Volume and Monitoring authorization applies. No Agent Volume monitoring is added. - -## Testing Decisions - -- Test observable behavior at the existing monitoring-service boundary using representative Kubernetes responses and database records; test UI behavior where necessary. Follow existing monitoring service unit-test and component-test patterns. -- Cover Longhorn metrics, multi-node lookup, app and shared-volume mapping, deduplication, unmounted/unbound volumes, local-path and unknown drivers, stale and missing fields, unavailable nodes, partial results and safe capacity calculations. -- Test externally visible results and error states, not internal helper call sequences. Validate against a real supported k3s cluster where available. - -## Out of Scope - -Historical metrics, Agent Volumes, supporting unverified storage drivers, Longhorn snapshot or replica storage accounting, alerting and export. - -## Further Notes - -The current cluster returned Longhorn filesystem metrics through the Summary API. A local-path example reported the host filesystem, and not all PVCs appeared in the response. The history issue builds on this feature. - -## Comments - -Implemented on `integration/storage-filesystem-usage` (commits `0a5666a0`, `181af6ce`). Targeted tests and lint pass; real-cluster validation remains outstanding. diff --git a/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts b/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts index 50a3f4c6..7dea5443 100644 --- a/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts +++ b/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts @@ -1,6 +1,6 @@ // @vitest-environment node import { createPrismaTestContext } from '@/__tests__/prisma-test.utils'; -import adapter from '@/server/adapter/app-history.adapter'; +import adapter from '@/server/services/standalone-services/app-history.service'; import history from '@/server/services/app-monitoring-history-query.service'; describe('App monitoring history persistence', () => { diff --git a/src/app/monitoring/actions.ts b/src/app/monitoring/actions.ts index c412a364..f2f302e9 100644 --- a/src/app/monitoring/actions.ts +++ b/src/app/monitoring/actions.ts @@ -1,7 +1,5 @@ 'use server' -import appHistoryAdapter from "@/server/adapter/app-history.adapter"; -import appMonitoringHistoryQueryService from "@/server/services/app-monitoring-history-query.service"; import monitoringService from "@/server/services/monitoring.service"; import clusterService from "@/server/services/cluster.service"; import { getAuthUserSession, simpleAction } from "@/server/utils/action-wrapper.utils"; @@ -32,10 +30,9 @@ export const getMonitoringForAllApps = async () => updatedNodeRessources = updatedNodeRessources?.filter((app) => UserGroupUtils.sessionHasReadAccessForApp(session, app.appId)); return updatedNodeRessources; }) as Promise>; -export const getAppMonitoringHistory = async () => + +export const getReservedVolumeStorage = async () => simpleAction(async () => { - const session = await getAuthUserSession(); - const { apps } = await appHistoryAdapter.getPopulation(); - const appIds = apps.filter(app => UserGroupUtils.sessionHasReadAccessForApp(session, app.id)).map(app => app.id); - return appMonitoringHistoryQueryService.getHistory(appIds); - }); + await getAuthUserSession(); + return await clusterService.getReservedVolumeStorage(); + }) as Promise>; diff --git a/src/app/monitoring/app-volumes-monitoring.tsx b/src/app/monitoring/app-volumes-monitoring.tsx index 4c0ba4b5..ca3ac1e4 100644 --- a/src/app/monitoring/app-volumes-monitoring.tsx +++ b/src/app/monitoring/app-volumes-monitoring.tsx @@ -71,8 +71,8 @@ export default function AppVolumeMonitoring({ {item.appName} {item.mountPath} - {item.status === 'available' && item.usedBytes !== undefined && item.capacityBytes !== undefined && item.capacityBytes > 0 && item.availableBytes !== undefined ? ( - + {item.status === 'available' && item.usedBytes !== undefined && item.capacityBytes !== undefined && item.capacityBytes > 0 ? ( + ) : Unavailable ({item.status})} @@ -91,16 +91,15 @@ export default function AppVolumeMonitoring({ ); } -function VolumeUsage({ usedBytes, capacityBytes, availableBytes }: { +function VolumeUsage({ usedBytes, capacityBytes }: { usedBytes: number; capacityBytes: number; - availableBytes: number; }) { const percentage = Math.round(usedBytes / capacityBytes * 100); return <> = 90 ? 'red' : percentage >= 80 ? 'orange' : undefined} />
- {KubeSizeConverter.convertBytesToReadableSize(usedBytes)} used ({percentage}%) / {KubeSizeConverter.convertBytesToReadableSize(capacityBytes)} | {KubeSizeConverter.convertBytesToReadableSize(availableBytes)} available + {KubeSizeConverter.convertBytesToReadableSize(usedBytes)} / {KubeSizeConverter.convertBytesToReadableSize(capacityBytes)}
; } diff --git a/src/app/monitoring/app-volumes-monitoring.unit.spec.ts b/src/app/monitoring/app-volumes-monitoring.unit.spec.ts index 654aef0d..eeee8ab3 100644 --- a/src/app/monitoring/app-volumes-monitoring.unit.spec.ts +++ b/src/app/monitoring/app-volumes-monitoring.unit.spec.ts @@ -19,5 +19,5 @@ it('counts shared volumes once and marks incomplete totals', () => { expect(screen.getByText(/Total used.*incomplete/).textContent).toContain('1 KiB'); expect(screen.getByText(/Total used.*incomplete/).textContent).toContain('4 KiB'); expect(screen.getByText('Unavailable (unavailable)')).toBeTruthy(); - expect(screen.getAllByText(/25%/)).toHaveLength(2); + expect(screen.getAllByText('1 KiB / 4 KiB')).toHaveLength(2); }); diff --git a/src/app/monitoring/history-actions.unit.spec.ts b/src/app/monitoring/history-actions.unit.spec.ts deleted file mode 100644 index 10b53dea..00000000 --- a/src/app/monitoring/history-actions.unit.spec.ts +++ /dev/null @@ -1,16 +0,0 @@ -vi.mock('@/server/utils/action-wrapper.utils', () => ({ - getAuthUserSession: vi.fn().mockResolvedValue({ userId: 'user' }), - simpleAction: async (callback: () => Promise) => ({ status: 'success', data: await callback() }), -})); -vi.mock('@/server/adapter/app-history.adapter', () => ({ default: { getPopulation: vi.fn().mockResolvedValue({ apps: [{ id: 'allowed' }, { id: 'denied' }], volumes: [] }) } })); -vi.mock('@/server/services/app-monitoring-history-query.service', () => ({ default: { getHistory: vi.fn().mockResolvedValue({}) } })); -vi.mock('@/shared/utils/role.utils', () => ({ UserGroupUtils: { sessionHasReadAccessForApp: (_session: unknown, appId: string) => appId === 'allowed' } })); -vi.mock('@/server/services/monitoring.service', () => ({ default: {} })); -vi.mock('@/server/services/cluster.service', () => ({ default: {} })); -import history from '@/server/services/app-monitoring-history-query.service'; -import { getAppMonitoringHistory } from './actions'; - -it('queries history only for Apps the user may read', async () => { - await getAppMonitoringHistory(); - expect(history.getHistory).toHaveBeenCalledWith(['allowed']); -}); diff --git a/src/app/monitoring/monitoring-client.tsx b/src/app/monitoring/monitoring-client.tsx index 59499fe4..97b03214 100644 --- a/src/app/monitoring/monitoring-client.tsx +++ b/src/app/monitoring/monitoring-client.tsx @@ -7,10 +7,9 @@ import { usePolling } from '@/frontend/hooks/use-polling'; import { AppMonitoringUsageModel } from '@/shared/model/app-monitoring-usage.model'; import { AppVolumeMonitoringUsageModel } from '@/shared/model/app-volume-monitoring-usage.model'; import { NodeResourceModel } from '@/shared/model/node-resource.model'; -import { getMonitoringForAllApps, getNodeResourceUsage, getVolumeMonitoringUsage } from './actions'; +import { getMonitoringForAllApps, getNodeResourceUsage, getReservedVolumeStorage, getVolumeMonitoringUsage } from './actions'; import AppRessourceMonitoring from './app-monitoring'; import AppVolumeMonitoring from './app-volumes-monitoring'; -import AppMonitoringHistoryView from '@/components/custom/app-monitoring-history'; import ResourcesNodes from './monitoring-nodes'; const MONITORING_POLL_INTERVAL_MS = 10000; @@ -24,24 +23,29 @@ export default function MonitoringClient({ resourcesNodes, appsRessourceUsage, volumesUsage, + reservedVolumeStorage, }: { resourcesNodes?: NodeResourceModel[]; appsRessourceUsage?: AppMonitoringUsageModel[]; volumesUsage?: AppVolumeMonitoringUsageModel[]; + reservedVolumeStorage?: number; }) { + const [updatedReservedStorage, setUpdatedReservedStorage] = useState(reservedVolumeStorage); const [updatedNodes, setUpdatedNodes] = useState(resourcesNodes); const [updatedApps, setUpdatedApps] = useState(appsRessourceUsage); const [updatedVolumes, setUpdatedVolumes] = useState(volumesUsage); const refreshMonitoringData = useCallback(async () => { try { - const [nodes, volumes, apps] = await Promise.all([ + const [nodes, volumes, apps, reservedStorage] = await Promise.all([ Actions.run(() => getNodeResourceUsage()), Actions.run(() => getVolumeMonitoringUsage()), Actions.run(() => getMonitoringForAllApps()), + Actions.run(() => getReservedVolumeStorage()).catch(() => undefined), ]); setUpdatedNodes(nodes); + setUpdatedReservedStorage(reservedStorage); setUpdatedVolumes(volumes?.filter((volume) => !!volume.isBaseVolume)); setUpdatedApps(apps); } catch (ex) { @@ -54,8 +58,7 @@ export default function MonitoringClient({ return (
- - +
diff --git a/src/app/monitoring/monitoring-nodes.tsx b/src/app/monitoring/monitoring-nodes.tsx index 6e2f90c4..fbd3f234 100644 --- a/src/app/monitoring/monitoring-nodes.tsx +++ b/src/app/monitoring/monitoring-nodes.tsx @@ -2,10 +2,6 @@ import { Label, - PolarGrid, - PolarRadiusAxis, - RadialBar, - RadialBarChart, Pie, PieChart, } from 'recharts'; @@ -22,19 +18,17 @@ import { useBreadcrumbs, } from '@/frontend/states/zustand.states'; import { useEffect, useMemo } from 'react'; -import ChartDiskRessources from './disk-chart'; import FullLoadingSpinner from '@/components/ui/full-loading-spinnter'; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table'; import { KubeSizeConverter } from '@/shared/utils/kubernetes-size-converter.utils'; import { Progress } from '@/components/ui/progress'; -import { Button } from '@/components/ui/button'; -import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle, SheetTrigger } from '@/components/ui/sheet'; -import { Activity, Cpu, HardDrive, MemoryStick } from 'lucide-react'; export default function ResourcesNodes({ resourcesNodes, + reservedVolumeStorage, }: { resourcesNodes?: NodeResourceModel[]; + reservedVolumeStorage?: number; }) { const getDiskUsageAbsolut = (node: NodeResourceModel) => node.diskUsageAbsolut ?? 0; @@ -167,12 +161,22 @@ export default function ResourcesNodes({ Cluster Storage - Node filesystem usage + Node filesystem usage and volume reservations {updatedNodeRessources.some(node => node.diskUsageCapacity === undefined) && (

Incomplete: some node filesystem metrics are unavailable.

)} +
+

Used: {clusterStats.diskCapacity > 0 ? KubeSizeConverter.convertBytesToReadableSize(clusterStats.diskUsageAbsolut) : 'Unavailable'}

+

Reserved by volumes: {reservedVolumeStorage === undefined ? 'Unavailable' : KubeSizeConverter.convertBytesToReadableSize(reservedVolumeStorage)}

+ {reservedVolumeStorage !== undefined && clusterStats.diskCapacity > 0 && ( +
+ +

{toPercent(reservedVolumeStorage, clusterStats.diskCapacity).toFixed(0)}% of node capacity

+
+ )} +
{clusterStats.diskCapacity > 0 ? ( @@ -207,7 +211,6 @@ export default function ResourcesNodes({ CPU RAM Storage - Actions @@ -252,9 +255,6 @@ export default function ResourcesNodes({ })()}
- - - ))} @@ -265,205 +265,3 @@ export default function ResourcesNodes({ ); } -function NodeDetailsSheet({ node }: { node: NodeResourceModel }) { - const chartData = [ - { browser: 'safari', usage: 1, fill: 'var(--color-safari)' }, - ]; - - const chartConfig = { - usage: { - label: 'Usage', - }, - safari: { - label: 'Safari', - color: 'var(--chart-2)', - }, - } satisfies ChartConfig; - - return ( - - View Details} /> - - - - - {node.name} - - - Detailed resource usage metrics - - - -
- {/* CPU Chart */} - - - - CPU Usage - - - - - - - - - - - - - - - {/* RAM Chart */} - - - - Memory Usage - - - - - - - - - - - - - - - {/* Disk Chart */} - - - - Storage Usage - - - - - - -
- - - ); -} diff --git a/src/app/monitoring/page.tsx b/src/app/monitoring/page.tsx index 3340edf3..196abf9f 100644 --- a/src/app/monitoring/page.tsx +++ b/src/app/monitoring/page.tsx @@ -12,10 +12,11 @@ export default async function ResourceNodesInfoPage() { const session = await getAuthUserSession(); - let [resourcesNode, volumesUsage, updatedNodeRessources] = await Promise.all([ + let [resourcesNode, volumesUsage, updatedNodeRessources, reservedVolumeStorage] = await Promise.all([ CatchUtils.resultOrUndefined(() => clusterService.getNodeResourceUsage()), CatchUtils.resultOrUndefined(() => monitoringService.getAllAppVolumesUsage()), - CatchUtils.resultOrUndefined(() => monitoringService.getMonitoringForAllApps()) + CatchUtils.resultOrUndefined(() => monitoringService.getMonitoringForAllApps()), + CatchUtils.resultOrUndefined(() => clusterService.getReservedVolumeStorage()) ]); // filter by role @@ -33,6 +34,7 @@ export default async function ResourceNodesInfoPage() {
diff --git a/src/app/project/[projectId]/apps/details/app-details-drawer.tsx b/src/app/project/[projectId]/apps/details/app-details-drawer.tsx index 50726e6a..81568319 100644 --- a/src/app/project/[projectId]/apps/details/app-details-drawer.tsx +++ b/src/app/project/[projectId]/apps/details/app-details-drawer.tsx @@ -264,20 +264,20 @@ export function AppDetailsDrawer({ - + - + {hasVolumes && ( - + - + )} {app.appType !== 'APP' && ( diff --git a/src/app/project/[projectId]/apps/details/deployments/app-details-deployments-tab.tsx b/src/app/project/[projectId]/apps/details/deployments/app-details-deployments-tab.tsx index c49194f3..4adbc38c 100644 --- a/src/app/project/[projectId]/apps/details/deployments/app-details-deployments-tab.tsx +++ b/src/app/project/[projectId]/apps/details/deployments/app-details-deployments-tab.tsx @@ -36,7 +36,7 @@ export default function DrawerDeploymentsTab({ const showWebhookCard = () => { openNestedDrawer({ title: 'Webhook', - description: ``, + description: 'Use this webhook URL to trigger deployments from external services (for example GitHub Actions or GitLab Pipelines).', content: ( - - - Webhook Deployment - Use this webhook URL to trigger deployments from external services (for example GitHub Actions or GitLab Pipelines). - - -
- {webhookUrl && } - {role === RolePermissionEnum.READWRITE && } -
- - - ; + return ( +
+ {webhookUrl && } + {role === RolePermissionEnum.READWRITE && } +
+ ); } diff --git a/src/app/project/[projectId]/apps/details/runtime/monitoring-app.tsx b/src/app/project/[projectId]/apps/details/runtime/monitoring-app.tsx index fa118152..17a3f04d 100644 --- a/src/app/project/[projectId]/apps/details/runtime/monitoring-app.tsx +++ b/src/app/project/[projectId]/apps/details/runtime/monitoring-app.tsx @@ -1,15 +1,9 @@ import AppMonitoringHistoryView from "@/components/custom/app-monitoring-history"; -import { CardContent } from "@/components/ui/card"; import { AppExtendedModel } from "@/shared/model/app-extended.model"; import { useCallback, useState } from "react"; import { getRessourceDataApp } from "../deployments/actions"; -import FullLoadingSpinner from "@/components/ui/full-loading-spinnter"; import { usePolling } from "@/frontend/hooks/use-polling"; import { PodsResourceInfoModel } from "@/shared/model/pods-resource-info.model"; -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; -import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; -import { KubeSizeConverter } from "@/shared/utils/kubernetes-size-converter.utils"; -import { cn } from "@/frontend/utils/utils"; export default function MonitoringTab({ app, @@ -26,12 +20,13 @@ export default function MonitoringTab({ const response = await getRessourceDataApp(app.projectId, app.id); if (response.status === 'success' && response.data) { setSelectedPod(response.data); - } else { + setSelectedPod(undefined); console.error(response); setError(response.message ?? 'An unknown error occurred.'); } } catch (ex) { + setSelectedPod(undefined); console.error(ex); setError('An unknown error occurred.'); } @@ -39,35 +34,10 @@ export default function MonitoringTab({ usePolling(updateValues, { intervalMs: 10000, runImmediately: true }); - return <> - - {error &&

Unable to load current App metrics: {error}

} - {!selectedPod && !error ? : selectedPod ? - - - - CPU % - RAM - - - - - - - - {selectedPod?.cpuPercent.toFixed(2)}} /> - -

{selectedPod?.cpuAbsolutCores.toFixed(10)} cores

- - - - - {KubeSizeConverter.convertBytesToReadableSize(selectedPod?.ramAbsolutBytes)} - - -
: null - } - - - ; + return ( + <> + {error &&

Unable to load current App metrics: {error}

} + + + ); } diff --git a/src/app/project/[projectId]/apps/details/volumes/backups/app-details-backups-tab.tsx b/src/app/project/[projectId]/apps/details/volumes/backups/app-details-backups-tab.tsx index b4b146d5..bfd14738 100644 --- a/src/app/project/[projectId]/apps/details/volumes/backups/app-details-backups-tab.tsx +++ b/src/app/project/[projectId]/apps/details/volumes/backups/app-details-backups-tab.tsx @@ -129,7 +129,7 @@ export function DrawerBackupsTab({ app, role, s3Targets, volumeBackups }: { // ScrollArea renders its vertical scrollbar over the right edge. Reserve space // so the split-button trigger stays reachable instead of sitting underneath it. - return
+ return
Backups
{!readonly && Backups could not be loaded Check the backup storage connection and try again. - : !backups ?
: backups.length === 0 ? + : !backups ?
: backups.length === 0 ? No backups yet diff --git a/src/app/project/[projectId]/apps/details/volumes/volumes-list.tsx b/src/app/project/[projectId]/apps/details/volumes/volumes-list.tsx index 049a6b72..82c22b79 100644 --- a/src/app/project/[projectId]/apps/details/volumes/volumes-list.tsx +++ b/src/app/project/[projectId]/apps/details/volumes/volumes-list.tsx @@ -28,7 +28,6 @@ type AppVolumeWithCapacity = (AppVolume & { usedBytes?: number; capacityBytes?: number; usedPercentage?: number; - availableBytes?: number; usageStatus?: string; }); @@ -52,7 +51,6 @@ export default function StorageList({ app, readonly, storageClasses }: { if (volume) { item.usedBytes = volume.usedBytes; item.capacityBytes = volume.capacityBytes; - item.availableBytes = volume.availableBytes; item.usageStatus = volume.status; item.usedPercentage = volume.usedBytes !== undefined && volume.capacityBytes ? Math.round(volume.usedBytes / volume.capacityBytes * 100) : undefined; } @@ -175,11 +173,11 @@ export default function StorageList({ app, readonly, storageClasses }: { {volume.containerMountPath} {volume.size} MiB - {volume.usedPercentage !== undefined && volume.usedBytes !== undefined && volume.capacityBytes !== undefined && volume.availableBytes !== undefined ? <> + {volume.usedPercentage !== undefined && volume.usedBytes !== undefined && volume.capacityBytes !== undefined ? <> = 90 ? 'red' : (volume.usedPercentage >= 80 ? 'orange' : undefined)} />
- {KubeSizeConverter.convertBytesToReadableSize(volume.usedBytes)} used / {KubeSizeConverter.convertBytesToReadableSize(volume.capacityBytes)} ({volume.usedPercentage}%) | {KubeSizeConverter.convertBytesToReadableSize(volume.availableBytes)} available + {KubeSizeConverter.convertBytesToReadableSize(volume.usedBytes)} / {KubeSizeConverter.convertBytesToReadableSize(volume.capacityBytes)}
: Unavailable{volume.usageStatus ? ` (${volume.usageStatus})` : ""}} diff --git a/src/app/settings/platform/operations/actions.ts b/src/app/settings/platform/operations/actions.ts index fb053b7c..943b6000 100644 --- a/src/app/settings/platform/operations/actions.ts +++ b/src/app/settings/platform/operations/actions.ts @@ -1,6 +1,6 @@ 'use server'; -import appHistoryAdapter from '@/server/adapter/app-history.adapter'; +import appHistoryService from '@/server/services/standalone-services/app-history.service'; import { getAdminUserSession, simpleAction } from '@/server/utils/action-wrapper.utils'; import { z } from 'zod'; @@ -8,10 +8,10 @@ const retentionSchema = z.number().int().min(1).max(168); export const getMonitoringHistoryRetention = async () => simpleAction(async () => { await getAdminUserSession(); - return appHistoryAdapter.getRetentionHours(); + return appHistoryService.getRetentionHours(); }); export const setMonitoringHistoryRetention = async (hours: number) => simpleAction(async () => { await getAdminUserSession(); - await appHistoryAdapter.setRetentionHours(retentionSchema.parse(hours)); + await appHistoryService.setRetentionHours(retentionSchema.parse(hours)); }); diff --git a/src/components/custom/app-monitoring-history.tsx b/src/components/custom/app-monitoring-history.tsx index f176780f..cfa9cf41 100644 --- a/src/components/custom/app-monitoring-history.tsx +++ b/src/components/custom/app-monitoring-history.tsx @@ -1,83 +1,104 @@ 'use client'; import { useCallback, useState } from 'react'; -import { CartesianGrid, Line, LineChart, ResponsiveContainer, Tooltip, XAxis, YAxis } from 'recharts'; +import { CartesianGrid, Line, LineChart, XAxis, YAxis } from 'recharts'; +import { ChartContainer, ChartTooltip, type ChartConfig } from '@/components/ui/chart'; import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip'; +import { Select, SelectContent, SelectGroup, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select'; +import { KubeSizeConverter } from '@/shared/utils/kubernetes-size-converter.utils'; import { usePolling } from '@/frontend/hooks/use-polling'; +import { PodsResourceInfoModel } from '@/shared/model/pods-resource-info.model'; import { AppMonitoringHistory } from '@/shared/model/app-monitoring-history.model'; -import { getAppMonitoringHistory as getAllHistory } from '@/app/monitoring/actions'; import { getAppMonitoringHistory as getOneHistory } from '@/app/project/[projectId]/apps/details/deployments/actions'; +import FullLoadingSpinner from '../ui/full-loading-spinnter'; -function HistoryChart({ title, data, dataKey, unit }: { +function HistoryChart({ title, data, dataKey, unit, currentValue, currentPercent, selectedRangeHours, useCurrentValue = false }: { title: string; data: object[]; dataKey: string; unit: string; + selectedRangeHours: number; + currentValue?: number; + currentPercent?: number; + useCurrentValue?: boolean; }) { const visibleSamples = data.filter(point => (point as Record)[dataKey] != null); + const latestValue = [...visibleSamples].reverse().map(point => (point as Record)[dataKey])[0]; + const formatValue = (value: number, axis = false) => unit === 'bytes' + ? KubeSizeConverter.convertBytesToReadableSize(value, axis ? 1 : 2) + : `${new Intl.NumberFormat(undefined, { maximumSignificantDigits: axis ? 2 : 6 }).format(value)}${axis ? '' : ' cores'}`; + const config = { [dataKey]: { label: title, color: "var(--chart-2)" } } satisfies ChartConfig; return ( -
-

{title} ({unit})

-
- - - + + + {title} + {currentPercent !== undefined && currentValue !== undefined ? ( + + + }> + {currentPercent.toFixed(2)}% + + {currentValue} cores + + + ) : ( +
+ {useCurrentValue + ? currentValue === undefined ? 'Unavailable' : formatValue(currentValue) + : latestValue === undefined ? 'Unavailable' : formatValue(latestValue)} +
+ )} + + + {visibleSamples.length === 0 ?

No measurements available.

: + + new Date(value).toLocaleTimeString([], { - hour: '2-digit', minute: '2-digit', + ...(selectedRangeHours > 24 ? { month: 'short', day: 'numeric' } as const : { hour: '2-digit', minute: '2-digit' } as const), })} /> - - { + formatValue(value, true)} /> + { if (!active || !payload?.length) return null; const item = payload[0]; - const sample = item.payload as { - availableVolumes?: number; - unavailableVolumes?: number; - }; return ( -
+

{new Date(Number(label)).toLocaleString()}

-

{item.value} {unit}

- {dataKey === 'storageBytes' && ( -

- {sample.availableVolumes} available, {sample.unavailableVolumes} unavailable - {sample.unavailableVolumes ? ' (incomplete total)' : ''} -

- )} +

{typeof item.value === "number" ? formatValue(item.value) : "Unavailable"}

); }} /> - -
-
+ } + + ); } -function freshness(time: number | null) { - if (!time) return 'Measurement unavailable'; - const stale = Date.now() - time > 10 * 60_000 ? ' (stale)' : ''; - return `Latest successful measurement: ${new Date(time).toLocaleString()}${stale}`; -} - -export default function AppMonitoringHistoryView({ appId }: { appId?: string }) { +export default function AppMonitoringHistoryView({ appId, currentUsage }: { appId: string; currentUsage?: PodsResourceInfoModel }) { const [history, setHistory] = useState(); const [error, setError] = useState(false); + const [range, setRange] = useState('1h'); const refresh = useCallback(async () => { try { - const response = appId ? await getOneHistory(appId) : await getAllHistory(); + const response = await getOneHistory(appId); if (response.status !== 'success' || !response.data) { throw new Error(response.message ?? 'Unable to load history'); } @@ -88,78 +109,81 @@ export default function AppMonitoringHistoryView({ appId }: { appId?: string }) } }, [appId]); usePolling(refresh, { intervalMs: 60_000, runImmediately: true }); - const latest = history?.points.findLast(point => !point.isGap); + const retentionHours = history?.retentionHours ?? 1; + const options = [ + { value: '1h', label: 'Last hour', hours: 1 }, + { value: '5h', label: 'Last 5 hours', hours: 5 }, + { value: 'today', label: 'Today', hours: 24 }, + { value: '48h', label: 'Last 48 hours', hours: 48 }, + { value: '7d', label: 'Last 7 days', hours: 168 }, + ].filter(option => option.hours <= retentionHours); + if (![1, 5, 48, 168].includes(retentionHours)) { + options.push({ value: 'all', label: `Last ${retentionHours} hours`, hours: retentionHours }); + } + const selectedRange = options.find(option => option.value === range) ?? options[0]; + const now = Date.now(); + const start = selectedRange.value === 'today' + ? new Date(new Date(now).setHours(0, 0, 0, 0)).getTime() + : now - selectedRange.hours * 60 * 60_000; + const visiblePoints = history?.points.filter(point => point.time >= start) ?? []; const collectionStopped = history?.points.at(-1)?.isGap; - const sampleCount = history?.points.filter(point => !point.isGap).length ?? 0; + const sampleCount = visiblePoints.filter(point => !point.isGap).length; return ( - - - {appId ? 'App history' : 'App totals history'} - - +
+
+ {history &&
+

Stats

+ +
} {error &&

Unable to load monitoring history.

} - {!history && !error &&

Loading monitoring history…

} + {!history && !error &&
} {history && history.points.length === 0 && (

No measurements yet. History is collected every five minutes.

)} - {history && history.points.length > 0 && <> + {history && <> {collectionStopped && (

Collection has stopped or is delayed. The chart shows a gap after the last sample.

)} {sampleCount === 1 &&

One sample collected. More samples will show a trend.

} -

- CPU is an average over the Kubernetes metrics source window; short peaks may be missed. -

- -

{freshness(history.latestCpuMeasurement)}

- -

{freshness(history.latestRamMeasurement)}

-

{freshness(history.latestVolumeMeasurement)}

- {!appId && <> +
+ - {latest && ( -

- Latest sample: {latest.availableVolumes} available and {latest.unavailableVolumes} unavailable volumes - {latest.unavailableVolumes ? ' (incomplete total)' : ''}. -

- )} - } - {appId && history.volumes.length === 0 &&

No App Volume measurements.

} - {appId && history.volumes.map(volume => { - const latestMeasurement = volume.points.reduce((latest, point) => { - if (point.usedBytes === null || !point.measuredAt) return latest; - return Math.max(latest ?? 0, point.measuredAt); - }, null); - return ( -
- -

{freshness(latestMeasurement)}

-
- ); - })} + {history.volumes.map(volume => { + return point.time >= start)} + dataKey="usedBytes" + unit="bytes" + selectedRangeHours={selectedRange.hours} + />; + })} +
} - - +
+
); } diff --git a/src/components/custom/app-monitoring-history.unit.spec.ts b/src/components/custom/app-monitoring-history.unit.spec.ts index 6233e76b..aad794f9 100644 --- a/src/components/custom/app-monitoring-history.unit.spec.ts +++ b/src/components/custom/app-monitoring-history.unit.spec.ts @@ -1,32 +1,25 @@ import React from 'react'; import { cleanup, render, screen } from '@testing-library/react'; import AppMonitoringHistoryView from './app-monitoring-history'; -import { getAppMonitoringHistory } from '@/app/monitoring/actions'; +import { getAppMonitoringHistory } from '@/app/project/[projectId]/apps/details/deployments/actions'; -vi.mock('@/app/monitoring/actions', () => ({ getAppMonitoringHistory: vi.fn() })); vi.mock('@/app/project/[projectId]/apps/details/deployments/actions', () => ({ getAppMonitoringHistory: vi.fn() })); vi.mock('recharts', () => ({ CartesianGrid: () => null, Line: () => null, LineChart: () => null, - ResponsiveContainer: () => null, Tooltip: () => null, XAxis: () => null, YAxis: () => null, + ResponsiveContainer: () => null, Tooltip: () => null, Legend: () => null, XAxis: () => null, YAxis: () => null, })); afterEach(() => { cleanup(); vi.clearAllMocks(); }); it('shows an empty state before the first sample', async () => { - vi.mocked(getAppMonitoringHistory).mockResolvedValue({ status: 'success', data: { points: [], volumes: [], latestCpuMeasurement: null, latestRamMeasurement: null, latestVolumeMeasurement: null } } as never); - render(React.createElement(AppMonitoringHistoryView)); + vi.mocked(getAppMonitoringHistory).mockResolvedValue({ status: 'success', data: { retentionHours: 24, points: [], volumes: [], latestCpuMeasurement: null, latestRamMeasurement: null, latestVolumeMeasurement: null } } as never); + render(React.createElement(AppMonitoringHistoryView, { appId: 'app' })); expect(await screen.findByText(/No measurements yet/)).toBeTruthy(); -}); - -it('labels partial App Volume totals with availability counts', async () => { - vi.mocked(getAppMonitoringHistory).mockResolvedValue({ status: 'success', data: { points: [{ time: Date.now(), cpuCores: null, ramBytes: null, storageBytes: 10, availableVolumes: 1, unavailableVolumes: 2 }], volumes: [], latestCpuMeasurement: null, latestRamMeasurement: null, latestVolumeMeasurement: null } } as never); - render(React.createElement(AppMonitoringHistoryView)); - expect(await screen.findByText(/1 available and 2 unavailable volumes \(incomplete total\)/)).toBeTruthy(); - expect(screen.getByText(/App CPU total/)).toBeTruthy(); + expect(screen.getByRole('combobox', { name: 'History time range' })).toBeTruthy(); }); it('shows an error when history cannot be loaded', async () => { vi.mocked(getAppMonitoringHistory).mockRejectedValue(new Error('unavailable')); - render(React.createElement(AppMonitoringHistoryView)); + render(React.createElement(AppMonitoringHistoryView, { appId: 'app' })); expect((await screen.findByRole('alert')).textContent).toContain('Unable to load monitoring history.'); }); @@ -45,7 +38,24 @@ it('shows a stopped collection gap without treating it as a new volume sample', latestVolumeMeasurement: null, }, } as never); - render(React.createElement(AppMonitoringHistoryView)); + render(React.createElement(AppMonitoringHistoryView, { appId: 'app' })); expect(await screen.findByText(/Collection has stopped or is delayed/)).toBeTruthy(); - expect(screen.getByText(/1 available and 2 unavailable volumes/)).toBeTruthy(); +}); + +it('updates app card values from current usage while keeping history', async () => { + vi.mocked(getAppMonitoringHistory).mockResolvedValue({ + status: 'success', + data: { + points: [{ time: Date.now(), cpuCores: 1, ramBytes: 1024, storageBytes: null, availableVolumes: 0, unavailableVolumes: 0 }], + volumes: [], latestCpuMeasurement: null, latestRamMeasurement: null, latestVolumeMeasurement: null, + }, + } as never); + const { rerender } = render(React.createElement(AppMonitoringHistoryView, { + appId: 'app', currentUsage: { cpuAbsolutCores: 2, cpuPercent: 12.345, ramAbsolutBytes: 2048, ramPercent: 0 }, + })); + expect(await screen.findByText('12.35%')).toBeTruthy(); + rerender(React.createElement(AppMonitoringHistoryView, { + appId: 'app', currentUsage: { cpuAbsolutCores: 3, cpuPercent: 25.6, ramAbsolutBytes: 4096, ramPercent: 0 }, + })); + expect(screen.getByText('25.60%')).toBeTruthy(); }); diff --git a/src/server/adapter/app-history.adapter.ts b/src/server/adapter/app-history.adapter.ts deleted file mode 100644 index 1404b9ac..00000000 --- a/src/server/adapter/app-history.adapter.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { Prisma } from '@prisma/client'; -import dataAccess from './db.client'; -import { Constants } from '@/shared/utils/constants'; - -class AppHistoryAdapter { - async getPopulation() { - const [apps, volumes] = await Promise.all([ - dataAccess.client.app.findMany({ select: { id: true, projectId: true } }), - dataAccess.client.appVolume.findMany({ include: { app: { select: { projectId: true } } } }), - ]); - return { apps, volumes }; - } - - async getRetentionHours() { - const value = (await dataAccess.client.parameter.findUnique({ where: { name: Constants.APP_MONITORING_HISTORY_RETENTION_HOURS } }))?.value; - const hours = Number(value); - return Number.isInteger(hours) && hours >= 1 && hours <= 168 ? hours : 24; - } - - async setRetentionHours(hours: number) { - await dataAccess.client.parameter.upsert({ - where: { name: Constants.APP_MONITORING_HISTORY_RETENTION_HOURS }, - create: { name: Constants.APP_MONITORING_HISTORY_RETENTION_HOURS, value: String(hours) }, - update: { value: String(hours) }, - }); - } - - async save(apps: Prisma.AppMonitoringSampleCreateManyInput[], volumes: Prisma.AppVolumeMonitoringSampleCreateManyInput[], memberships: Prisma.AppVolumeMonitoringMembershipCreateManyInput[], cutoff: Date) { - await dataAccess.client.$transaction(async tx => { - for (let i = 0; i < apps.length; i += 100) await tx.appMonitoringSample.createMany({ data: apps.slice(i, i + 100) }); - for (let i = 0; i < volumes.length; i += 100) await tx.appVolumeMonitoringSample.createMany({ data: volumes.slice(i, i + 100) }); - for (let i = 0; i < memberships.length; i += 100) await tx.appVolumeMonitoringMembership.createMany({ data: memberships.slice(i, i + 100) }); - await tx.appMonitoringSample.deleteMany({ where: { sampledAt: { lt: cutoff } } }); - await tx.appVolumeMonitoringSample.deleteMany({ where: { sampledAt: { lt: cutoff } } }); - }); - } - - async getHistory(appIds: string[], cutoff: Date) { - return Promise.all([ - dataAccess.client.appMonitoringSample.findMany({ where: { appId: { in: appIds }, sampledAt: { gte: cutoff } }, orderBy: { sampledAt: 'asc' } }), - dataAccess.client.appVolumeMonitoringMembership.findMany({ - where: { volume: { appId: { in: appIds } }, sample: { sampledAt: { gte: cutoff } } }, - include: { - sample: true, - volume: { select: { appId: true, sharedVolumeId: true, containerMountPath: true, app: { select: { projectId: true } } } }, - }, - orderBy: { sample: { sampledAt: 'asc' } }, - }).then(memberships => memberships.map(item => ({ ...item.sample, volumeId: item.volumeId, volume: item.volume }))), - ]); - } -} -const appHistoryAdapter = new AppHistoryAdapter(); -export default appHistoryAdapter; diff --git a/src/server/services/app-monitoring-history-query.service.ts b/src/server/services/app-monitoring-history-query.service.ts index a143223d..05ae8a81 100644 --- a/src/server/services/app-monitoring-history-query.service.ts +++ b/src/server/services/app-monitoring-history-query.service.ts @@ -1,10 +1,10 @@ -import appHistoryAdapter from '@/server/adapter/app-history.adapter'; +import appHistoryService from '@/server/services/standalone-services/app-history.service'; import { AppHistoryPoint, AppMonitoringHistory, AppVolumeHistory } from '@/shared/model/app-monitoring-history.model'; class AppMonitoringHistoryQueryService { async getHistory(appIds: string[]): Promise { - const retention = await appHistoryAdapter.getRetentionHours(); - const [apps, volumes] = await appHistoryAdapter.getHistory(appIds, new Date(Date.now() - retention * 3600_000)); + const retention = await appHistoryService.getRetentionHours(); + const [apps, volumes] = await appHistoryService.getHistory(appIds, new Date(Date.now() - retention * 3600_000)); const appsByTime = new Map(); const volumesByTime = new Map(); for (const item of apps) { @@ -83,7 +83,7 @@ class AppMonitoringHistoryQueryService { volume.points.push({ time: now, usedBytes: null, measuredAt: null }); } } - return { points, volumes: [...volumeSeries.values()], latestCpuMeasurement, latestRamMeasurement, latestVolumeMeasurement }; + return { retentionHours: retention, points, volumes: [...volumeSeries.values()], latestCpuMeasurement, latestRamMeasurement, latestVolumeMeasurement }; } } const appMonitoringHistoryQueryService = new AppMonitoringHistoryQueryService(); diff --git a/src/server/services/app-monitoring-history-query.service.unit.spec.ts b/src/server/services/app-monitoring-history-query.service.unit.spec.ts index bcbcc8ce..f41d170a 100644 --- a/src/server/services/app-monitoring-history-query.service.unit.spec.ts +++ b/src/server/services/app-monitoring-history-query.service.unit.spec.ts @@ -1,7 +1,7 @@ -vi.mock('@/server/adapter/app-history.adapter', () => ({ +vi.mock('@/server/services/standalone-services/app-history.service', () => ({ default: { getRetentionHours: vi.fn().mockResolvedValue(24), getHistory: vi.fn() }, })); -import adapter from '@/server/adapter/app-history.adapter'; +import adapter from '@/server/services/standalone-services/app-history.service'; import historyService from './app-monitoring-history-query.service'; it('shows incomplete App totals as gaps and counts shared volumes once', async () => { @@ -13,6 +13,7 @@ it('shows incomplete App totals as gaps and counts shared volumes once', async ( [volume('base', null, 'available', 20), volume('shared', 'base', 'available', 20), volume('missing', null, 'unavailable', null)], ]); const history = await historyService.getHistory(['one', 'two']); + expect(history.retentionHours).toBe(24); expect(history.points[0]).toMatchObject({ cpuCores: null, ramBytes: null, storageBytes: 20, availableVolumes: 1, unavailableVolumes: 1 }); }); diff --git a/src/server/services/cluster.service.ts b/src/server/services/cluster.service.ts index 0de75ab6..4d5f063a 100644 --- a/src/server/services/cluster.service.ts +++ b/src/server/services/cluster.service.ts @@ -1,3 +1,4 @@ +import volumeSummaryAdapter from "../adapter/volume-summary.adapter"; import k3s, { kubernetesPatchOptions } from "../adapter/kubernetes-api.adapter"; import * as k8s from '@kubernetes/client-node'; import { NodeInfoModel } from "@/shared/model/node-info.model"; @@ -125,6 +126,14 @@ class ClusterService { } } + async getReservedVolumeStorage(): Promise { + const claims = await volumeSummaryAdapter.getPersistentVolumeClaims(); + return claims.reduce((total, claim) => { + const size = claim.spec?.resources?.requests?.storage; + return total + (size ? KubeSizeConverter.fromKubeSizeToBytes(size) : 0); + }, 0); + } + async getNodeResourceUsage(): Promise { const topNodes = await k8s.topNodes(k3s.core); diff --git a/src/server/services/cluster.service.unit.spec.ts b/src/server/services/cluster.service.unit.spec.ts index 6239b82c..ea88810d 100644 --- a/src/server/services/cluster.service.unit.spec.ts +++ b/src/server/services/cluster.service.unit.spec.ts @@ -1,6 +1,6 @@ -const mocks = vi.hoisted(() => ({ topNodes: vi.fn(), getNodeMetrics: vi.fn(), getNodeSummary: vi.fn() })); +const mocks = vi.hoisted(() => ({ topNodes: vi.fn(), getNodeMetrics: vi.fn(), getNodeSummary: vi.fn(), getPersistentVolumeClaims: vi.fn() })); vi.mock('@kubernetes/client-node', async (importOriginal) => ({ ...await importOriginal(), topNodes: mocks.topNodes, @@ -9,6 +9,9 @@ vi.mock('@/server/adapter/kubernetes-api.adapter', () => ({ default: { core: {}, metrics: { getNodeMetrics: mocks.getNodeMetrics }, getNodeSummary: mocks.getNodeSummary }, kubernetesPatchOptions: vi.fn(), })); +vi.mock('@/server/adapter/volume-summary.adapter', () => ({ + default: { getPersistentVolumeClaims: mocks.getPersistentVolumeClaims }, +})); import clusterService from './cluster.service'; describe('cluster node storage', () => { @@ -41,3 +44,15 @@ describe('cluster node storage', () => { expect(nodes.map(node => node.diskUsageCapacity)).toEqual([undefined, undefined]); }); }); + +describe('reserved volume storage', () => { + it('sums requested storage across all claims', async () => { + mocks.getPersistentVolumeClaims.mockResolvedValue([ + { spec: { resources: { requests: { storage: '2Gi' } } } }, + { spec: { resources: { requests: { storage: '512Mi' } } } }, + { spec: {} }, + ]); + + expect(await clusterService.getReservedVolumeStorage()).toBe(2684354560); + }); +}); diff --git a/src/server/services/standalone-services/app-history-metrics.service.ts b/src/server/services/standalone-services/app-history.service.ts similarity index 51% rename from src/server/services/standalone-services/app-history-metrics.service.ts rename to src/server/services/standalone-services/app-history.service.ts index 7400cee1..fb5373bf 100644 --- a/src/server/services/standalone-services/app-history-metrics.service.ts +++ b/src/server/services/standalone-services/app-history.service.ts @@ -1,10 +1,59 @@ +import { Prisma } from '@prisma/client'; +import dataAccess from '@/server/adapter/db.client'; +import { Constants } from '@/shared/utils/constants'; import { quantityToScalar } from '@kubernetes/client-node/dist/util.js'; import metricsAdapter from '@/server/adapter/app-history-metrics.adapter'; type App = { id: string; projectId: string }; type Measurement = { cpuCores: number | null; ramBytes: number | null; measuredAt: Date | null }; -class AppHistoryMetricsService { +class AppHistoryService { + async getPopulation() { + const [apps, volumes] = await Promise.all([ + dataAccess.client.app.findMany({ select: { id: true, projectId: true } }), + dataAccess.client.appVolume.findMany({ include: { app: { select: { projectId: true } } } }), + ]); + return { apps, volumes }; + } + + async getRetentionHours() { + const value = (await dataAccess.client.parameter.findUnique({ where: { name: Constants.APP_MONITORING_HISTORY_RETENTION_HOURS } }))?.value; + const hours = Number(value); + return Number.isInteger(hours) && hours >= 1 && hours <= 168 ? hours : 24; + } + + async setRetentionHours(hours: number) { + await dataAccess.client.parameter.upsert({ + where: { name: Constants.APP_MONITORING_HISTORY_RETENTION_HOURS }, + create: { name: Constants.APP_MONITORING_HISTORY_RETENTION_HOURS, value: String(hours) }, + update: { value: String(hours) }, + }); + } + + async save(apps: Prisma.AppMonitoringSampleCreateManyInput[], volumes: Prisma.AppVolumeMonitoringSampleCreateManyInput[], memberships: Prisma.AppVolumeMonitoringMembershipCreateManyInput[], cutoff: Date) { + await dataAccess.client.$transaction(async tx => { + for (let i = 0; i < apps.length; i += 100) await tx.appMonitoringSample.createMany({ data: apps.slice(i, i + 100) }); + for (let i = 0; i < volumes.length; i += 100) await tx.appVolumeMonitoringSample.createMany({ data: volumes.slice(i, i + 100) }); + for (let i = 0; i < memberships.length; i += 100) await tx.appVolumeMonitoringMembership.createMany({ data: memberships.slice(i, i + 100) }); + await tx.appMonitoringSample.deleteMany({ where: { sampledAt: { lt: cutoff } } }); + await tx.appVolumeMonitoringSample.deleteMany({ where: { sampledAt: { lt: cutoff } } }); + }); + } + + async getHistory(appIds: string[], cutoff: Date) { + return Promise.all([ + dataAccess.client.appMonitoringSample.findMany({ where: { appId: { in: appIds }, sampledAt: { gte: cutoff } }, orderBy: { sampledAt: 'asc' } }), + dataAccess.client.appVolumeMonitoringMembership.findMany({ + where: { volume: { appId: { in: appIds } }, sample: { sampledAt: { gte: cutoff } } }, + include: { + sample: true, + volume: { select: { appId: true, sharedVolumeId: true, containerMountPath: true, app: { select: { projectId: true } } } }, + }, + orderBy: { sample: { sampledAt: 'asc' } }, + }).then(memberships => memberships.map(item => ({ ...item.sample, volumeId: item.volumeId, volume: item.volume }))), + ]); + } + async getAppMetrics(apps: App[]): Promise> { const [pods, metrics] = await Promise.all([metricsAdapter.getPods(), metricsAdapter.getMetrics()]); const metricsByPod = new Map(metrics.map(metric => [`${metric.metadata.namespace}/${metric.metadata.name}`, metric])); @@ -61,5 +110,5 @@ class AppHistoryMetricsService { } } -const appHistoryMetricsService = new AppHistoryMetricsService(); -export default appHistoryMetricsService; +const appHistoryService = new AppHistoryService(); +export default appHistoryService; diff --git a/src/server/services/standalone-services/app-history-metrics.service.unit.spec.ts b/src/server/services/standalone-services/app-history.service.unit.spec.ts similarity index 94% rename from src/server/services/standalone-services/app-history-metrics.service.unit.spec.ts rename to src/server/services/standalone-services/app-history.service.unit.spec.ts index 35f867ac..70674ed9 100644 --- a/src/server/services/standalone-services/app-history-metrics.service.unit.spec.ts +++ b/src/server/services/standalone-services/app-history.service.unit.spec.ts @@ -1,4 +1,6 @@ -import appHistoryMetricsService from './app-history-metrics.service'; +vi.mock('@/server/adapter/db.client', () => ({ default: { client: {} } })); + +import appHistoryMetricsService from './app-history.service'; vi.mock('@/server/adapter/app-history-metrics.adapter', () => ({ default: { getPods: vi.fn(), getMetrics: vi.fn() }, diff --git a/src/server/services/standalone-services/app-monitoring-history.service.ts b/src/server/services/standalone-services/app-monitoring-history.service.ts index 230bbbdc..64235152 100644 --- a/src/server/services/standalone-services/app-monitoring-history.service.ts +++ b/src/server/services/standalone-services/app-monitoring-history.service.ts @@ -1,8 +1,7 @@ import { randomUUID } from 'crypto'; import { Prisma } from '@prisma/client'; -import appHistoryAdapter from '@/server/adapter/app-history.adapter'; +import appHistoryService from '@/server/services/standalone-services/app-history.service'; import { KubeObjectNameUtils } from '@/server/utils/kube-object-name.utils'; -import appHistoryMetricsService from './app-history-metrics.service'; import appVolumeFilesystemService from './app-volume-filesystem.service'; import scheduleService from './schedule.service'; @@ -23,9 +22,9 @@ class AppMonitoringHistoryService { this.running = true; try { const sampledAt = new Date(); - const { apps, volumes } = await appHistoryAdapter.getPopulation(); + const { apps, volumes } = await appHistoryService.getPopulation(); const [metrics, filesystem, retention] = await Promise.all([ - appHistoryMetricsService.getAppMetrics(apps).catch(error => { + appHistoryService.getAppMetrics(apps).catch(error => { console.error('App metrics collection failed', error); return new Map(); }), @@ -35,7 +34,7 @@ class AppMonitoringHistoryService { console.error('Volume metrics collection failed', error); return new Map(); }), - appHistoryAdapter.getRetentionHours(), + appHistoryService.getRetentionHours(), ]); const samples = new Map(); const memberships: Prisma.AppVolumeMonitoringMembershipCreateManyInput[] = []; @@ -59,7 +58,7 @@ class AppMonitoringHistoryService { } memberships.push({ sampleId: sample.id, volumeId: volume.id }); } - await appHistoryAdapter.save( + await appHistoryService.save( apps.map(app => ({ appId: app.id, sampledAt, diff --git a/src/server/services/standalone-services/app-monitoring-history.service.unit.spec.ts b/src/server/services/standalone-services/app-monitoring-history.service.unit.spec.ts index 1a5e3913..108b13d0 100644 --- a/src/server/services/standalone-services/app-monitoring-history.service.unit.spec.ts +++ b/src/server/services/standalone-services/app-monitoring-history.service.unit.spec.ts @@ -1,10 +1,9 @@ -vi.mock('@/server/adapter/app-history.adapter', () => ({ default: { getPopulation: vi.fn(), getRetentionHours: vi.fn(), save: vi.fn() } })); -vi.mock('./app-history-metrics.service', () => ({ default: { getAppMetrics: vi.fn() } })); +vi.mock('./app-history.service', () => ({ default: { getPopulation: vi.fn(), getRetentionHours: vi.fn(), save: vi.fn(), getAppMetrics: vi.fn() } })); vi.mock('./app-volume-filesystem.service', () => ({ default: { getVolumeFilesystemUsage: vi.fn() } })); vi.mock('./schedule.service', () => ({ default: { scheduleJob: vi.fn() } })); import history from './app-monitoring-history.service'; -import adapter from '@/server/adapter/app-history.adapter'; -import metrics from './app-history-metrics.service'; +import adapter from '@/server/services/standalone-services/app-history.service'; +import metrics from './app-history.service'; import filesystem from './app-volume-filesystem.service'; it('records a gap for failed App metrics while preserving successful volume data', async () => { diff --git a/src/shared/model/app-monitoring-history.model.ts b/src/shared/model/app-monitoring-history.model.ts index a7d2ad99..e03936b3 100644 --- a/src/shared/model/app-monitoring-history.model.ts +++ b/src/shared/model/app-monitoring-history.model.ts @@ -13,6 +13,7 @@ export interface AppVolumeHistory { points: { time: number; usedBytes: number | null; measuredAt: number | null }[]; } export interface AppMonitoringHistory { + retentionHours: number; points: AppHistoryPoint[]; volumes: AppVolumeHistory[]; latestCpuMeasurement: number | null; From 9c94e0b3694a36baec2e447a55db1d0048a565a6 Mon Sep 17 00:00:00 2001 From: biersoeckli Date: Sun, 8 Nov 2026 14:44:04 +0000 Subject: [PATCH 07/11] feat: enhance disk usage chart with reserved storage metrics and improve accessibility --- src/app/monitoring/monitoring-nodes.tsx | 62 +++++++++++++++---------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/src/app/monitoring/monitoring-nodes.tsx b/src/app/monitoring/monitoring-nodes.tsx index fbd3f234..ded34d52 100644 --- a/src/app/monitoring/monitoring-nodes.tsx +++ b/src/app/monitoring/monitoring-nodes.tsx @@ -167,32 +167,44 @@ export default function ResourcesNodes({ {updatedNodeRessources.some(node => node.diskUsageCapacity === undefined) && (

Incomplete: some node filesystem metrics are unavailable.

)} -
-

Used: {clusterStats.diskCapacity > 0 ? KubeSizeConverter.convertBytesToReadableSize(clusterStats.diskUsageAbsolut) : 'Unavailable'}

-

Reserved by volumes: {reservedVolumeStorage === undefined ? 'Unavailable' : KubeSizeConverter.convertBytesToReadableSize(reservedVolumeStorage)}

- {reservedVolumeStorage !== undefined && clusterStats.diskCapacity > 0 && ( -
- -

{toPercent(reservedVolumeStorage, clusterStats.diskCapacity).toFixed(0)}% of node capacity

-
- )} -
{clusterStats.diskCapacity > 0 ? ( - - - KubeSizeConverter.convertBytesToReadableSize(value as number)} />} /> - - - - + <> + + + ( + {name === 'reserved' ? 'Reserved beyond node usage' : name === 'used' ? 'Used' : 'Remaining'}: {KubeSizeConverter.convertBytesToReadableSize(value as number)} + )} />} /> + + + + +

+ Node filesystem: {KubeSizeConverter.convertBytesToReadableSize(clusterStats.diskUsageAbsolut)} used. + Volume reservation: {reservedVolumeStorage === undefined ? 'reservation unavailable' : `${KubeSizeConverter.convertBytesToReadableSize(reservedVolumeStorage)} reserved`}. + Total node capacity: {KubeSizeConverter.convertBytesToReadableSize(clusterStats.diskCapacity)}. +

+ ) :

Storage metrics unavailable

} From a3220d9ce5a6a05dbb413f2429e034506ee110a7 Mon Sep 17 00:00:00 2001 From: biersoeckli Date: Sun, 8 Nov 2026 14:49:17 +0000 Subject: [PATCH 08/11] feat: refactor app monitoring history services and introduce app volume identity utilities --- .prettierrc.json | 15 +++ ...app-monitoring-history.integration.spec.ts | 35 ++++- src/app/monitoring/app-volumes-monitoring.tsx | 3 +- .../apps/details/deployments/actions.ts | 2 +- .../deployments/history-actions.unit.spec.ts | 4 +- .../settings/platform/operations/actions.ts | 2 +- src/server/adapter/app-history.adapter.ts | 55 ++++++++ .../app-monitoring-history-query.service.ts | 90 ------------- src/server/services/monitoring.service.ts | 10 +- .../app-history.service.ts | 49 ------- ...itoring-history-query.service.unit.spec.ts | 6 +- .../app-monitoring-history.service.ts | 120 ++++++++++++++++-- ...pp-monitoring-history.service.unit.spec.ts | 9 +- .../app-volume-filesystem.service.ts | 27 ++-- ...app-volume-filesystem.service.unit.spec.ts | 32 +++++ .../services/volume-filesystem-drivers.ts | 13 -- src/shared/utils/app-volume-identity.utils.ts | 13 ++ 17 files changed, 295 insertions(+), 190 deletions(-) create mode 100644 .prettierrc.json create mode 100644 src/server/adapter/app-history.adapter.ts delete mode 100644 src/server/services/app-monitoring-history-query.service.ts rename src/server/services/{ => standalone-services}/app-monitoring-history-query.service.unit.spec.ts (92%) create mode 100644 src/server/services/standalone-services/app-volume-filesystem.service.unit.spec.ts delete mode 100644 src/server/services/volume-filesystem-drivers.ts create mode 100644 src/shared/utils/app-volume-identity.utils.ts diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 00000000..088517c2 --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,15 @@ +{ + "endOfLine": "lf", + "semi": true, + "singleQuote": true, + "jsxSingleQuote": true, + "tabWidth": 2, + "useTabs": false, + "trailingComma": "none", + "printWidth": 160, + "arrowParens": "always", + "objectWrap": "collapse", + "plugins": ["prettier-plugin-tailwindcss"], + "tailwindStylesheet": "app/globals.css", + "tailwindFunctions": ["cn", "cva"] +} diff --git a/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts b/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts index 7dea5443..7555564c 100644 --- a/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts +++ b/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts @@ -1,7 +1,9 @@ // @vitest-environment node import { createPrismaTestContext } from '@/__tests__/prisma-test.utils'; -import adapter from '@/server/services/standalone-services/app-history.service'; -import history from '@/server/services/app-monitoring-history-query.service'; +import adapter from '@/server/adapter/app-history.adapter'; +import history from '@/server/services/standalone-services/app-monitoring-history.service'; +import metrics from '@/server/services/standalone-services/app-history.service'; +import filesystem from '@/server/services/standalone-services/app-volume-filesystem.service'; describe('App monitoring history persistence', () => { const ctx = createPrismaTestContext('app-monitoring-history'); @@ -54,6 +56,35 @@ describe('App monitoring history persistence', () => { }); + it('collects shared Volume measurements and reads them through each App', async () => { + const db = ctx.getDataAccess().client; + await db.project.create({ data: { id: 'project', name: 'Project' } }); + await db.app.createMany({ data: [ + { id: 'one', name: 'One', projectId: 'project' }, + { id: 'two', name: 'Two', projectId: 'project' }, + ] }); + await db.appVolume.create({ data: { id: 'base', appId: 'one', containerMountPath: '/data', size: 1 } }); + await db.appVolume.create({ data: { id: 'shared', appId: 'two', containerMountPath: '/share', size: 1, sharedVolumeId: 'base' } }); + const measuredAt = new Date(); + const appMetrics = vi.spyOn(metrics, 'getAppMetrics').mockResolvedValue(new Map([ + ['one', { cpuCores: 1, ramBytes: 100, measuredAt }], + ['two', { cpuCores: 2, ramBytes: 200, measuredAt }], + ])); + const volumeMetrics = vi.spyOn(filesystem, 'getVolumeFilesystemUsage').mockResolvedValue(new Map([ + ['base', { pvcName: 'pvc-base', status: 'available', usedBytes: 42, measuredAt: measuredAt.toISOString() }], + ['shared', { pvcName: 'pvc-base', status: 'available', usedBytes: 42, measuredAt: measuredAt.toISOString() }], + ])); + try { + await history.collect(); + expect(await db.appVolumeMonitoringSample.count()).toBe(1); + expect((await history.getHistory(['one', 'two'])).points[0]).toMatchObject({ cpuCores: 3, ramBytes: 300, storageBytes: 42, availableVolumes: 1 }); + expect((await history.getHistory(['two'])).volumes.map(volume => volume.label)).toEqual(['/share']); + } finally { + appMetrics.mockRestore(); + volumeMetrics.mockRestore(); + } + }); + it('reads a day of five-minute history for 100 Apps', async () => { const db = ctx.getDataAccess().client; await db.project.create({ data: { id: 'project', name: 'Project' } }); diff --git a/src/app/monitoring/app-volumes-monitoring.tsx b/src/app/monitoring/app-volumes-monitoring.tsx index ca3ac1e4..f3e5182e 100644 --- a/src/app/monitoring/app-volumes-monitoring.tsx +++ b/src/app/monitoring/app-volumes-monitoring.tsx @@ -17,6 +17,7 @@ import Link from 'next/link'; import { Button } from '@/components/ui/button'; import { Progress } from "@/components/ui/progress" import { PathBuilderUtils } from '@/shared/utils/path-builder.utils'; +import { AppVolumeIdentityUtils } from '@/shared/utils/app-volume-identity.utils'; export default function AppVolumeMonitoring({ volumesUsage @@ -27,7 +28,7 @@ export default function AppVolumeMonitoring({ const updatedVolumeUsage = volumesUsage; const totals = useMemo(() => { const unique = new Map(); - for (const volume of volumesUsage ?? []) unique.set(`${volume.projectId}/${volume.pvcName}`, volume); + for (const volume of volumesUsage ?? []) unique.set(AppVolumeIdentityUtils.observationKey(volume.projectId, volume.pvcName), volume); const values = [...unique.values()]; return { used: values.reduce((sum, value) => sum + (value.usedBytes ?? 0), 0), diff --git a/src/app/project/[projectId]/apps/details/deployments/actions.ts b/src/app/project/[projectId]/apps/details/deployments/actions.ts index 1d98ab2f..115f3615 100644 --- a/src/app/project/[projectId]/apps/details/deployments/actions.ts +++ b/src/app/project/[projectId]/apps/details/deployments/actions.ts @@ -1,6 +1,6 @@ 'use server' -import appMonitoringHistoryQueryService from '@/server/services/app-monitoring-history-query.service'; +import appMonitoringHistoryQueryService from '@/server/services/standalone-services/app-monitoring-history.service'; import { SuccessActionResult } from "@/shared/model/server-action-error-return.model"; import appService from "@/server/services/app.service"; diff --git a/src/app/project/[projectId]/apps/details/deployments/history-actions.unit.spec.ts b/src/app/project/[projectId]/apps/details/deployments/history-actions.unit.spec.ts index 16227d84..1270603c 100644 --- a/src/app/project/[projectId]/apps/details/deployments/history-actions.unit.spec.ts +++ b/src/app/project/[projectId]/apps/details/deployments/history-actions.unit.spec.ts @@ -1,4 +1,4 @@ -vi.mock('@/server/services/app-monitoring-history-query.service', () => ({ +vi.mock('@/server/services/standalone-services/app-monitoring-history.service', () => ({ default: { getHistory: vi.fn().mockResolvedValue({ points: [] }) }, })); vi.mock('@/server/services/app.service', () => ({ default: {} })); @@ -15,7 +15,7 @@ vi.mock('@/server/utils/action-wrapper.utils', () => ({ catch { return { status: 'error' }; } }, })); -import history from '@/server/services/app-monitoring-history-query.service'; +import history from '@/server/services/standalone-services/app-monitoring-history.service'; import { isAuthorizedReadForApp } from '@/server/utils/action-wrapper.utils'; import { getAppMonitoringHistory } from './actions'; diff --git a/src/app/settings/platform/operations/actions.ts b/src/app/settings/platform/operations/actions.ts index 943b6000..5ac4237c 100644 --- a/src/app/settings/platform/operations/actions.ts +++ b/src/app/settings/platform/operations/actions.ts @@ -1,6 +1,6 @@ 'use server'; -import appHistoryService from '@/server/services/standalone-services/app-history.service'; +import appHistoryService from '@/server/services/standalone-services/app-monitoring-history.service'; import { getAdminUserSession, simpleAction } from '@/server/utils/action-wrapper.utils'; import { z } from 'zod'; diff --git a/src/server/adapter/app-history.adapter.ts b/src/server/adapter/app-history.adapter.ts new file mode 100644 index 00000000..2511ff09 --- /dev/null +++ b/src/server/adapter/app-history.adapter.ts @@ -0,0 +1,55 @@ +import { Prisma } from '@prisma/client'; +import dataAccess from '@/server/adapter/db.client'; +import { Constants } from '@/shared/utils/constants'; + +class AppHistoryAdapter { + async getPopulation() { + const [apps, volumes] = await Promise.all([ + dataAccess.client.app.findMany({ select: { id: true, projectId: true } }), + dataAccess.client.appVolume.findMany({ include: { app: { select: { projectId: true } } } }), + ]); + return { apps, volumes }; + } + + async getRetentionHours() { + const value = (await dataAccess.client.parameter.findUnique({ where: { name: Constants.APP_MONITORING_HISTORY_RETENTION_HOURS } }))?.value; + const hours = Number(value); + return Number.isInteger(hours) && hours >= 1 && hours <= 168 ? hours : 24; + } + + async setRetentionHours(hours: number) { + await dataAccess.client.parameter.upsert({ + where: { name: Constants.APP_MONITORING_HISTORY_RETENTION_HOURS }, + create: { name: Constants.APP_MONITORING_HISTORY_RETENTION_HOURS, value: String(hours) }, + update: { value: String(hours) }, + }); + } + + async save(apps: Prisma.AppMonitoringSampleCreateManyInput[], volumes: Prisma.AppVolumeMonitoringSampleCreateManyInput[], memberships: Prisma.AppVolumeMonitoringMembershipCreateManyInput[], cutoff: Date) { + await dataAccess.client.$transaction(async tx => { + for (let i = 0; i < apps.length; i += 100) await tx.appMonitoringSample.createMany({ data: apps.slice(i, i + 100) }); + for (let i = 0; i < volumes.length; i += 100) await tx.appVolumeMonitoringSample.createMany({ data: volumes.slice(i, i + 100) }); + for (let i = 0; i < memberships.length; i += 100) await tx.appVolumeMonitoringMembership.createMany({ data: memberships.slice(i, i + 100) }); + await tx.appMonitoringSample.deleteMany({ where: { sampledAt: { lt: cutoff } } }); + await tx.appVolumeMonitoringSample.deleteMany({ where: { sampledAt: { lt: cutoff } } }); + }); + } + + async getHistory(appIds: string[], cutoff: Date) { + return Promise.all([ + dataAccess.client.appMonitoringSample.findMany({ where: { appId: { in: appIds }, sampledAt: { gte: cutoff } }, orderBy: { sampledAt: 'asc' } }), + dataAccess.client.appVolumeMonitoringMembership.findMany({ + where: { volume: { appId: { in: appIds } }, sample: { sampledAt: { gte: cutoff } } }, + include: { + sample: true, + volume: { select: { appId: true, sharedVolumeId: true, containerMountPath: true, app: { select: { projectId: true } } } }, + }, + orderBy: { sample: { sampledAt: 'asc' } }, + }).then(memberships => memberships.map(item => ({ ...item.sample, volumeId: item.volumeId, volume: item.volume }))), + ]); + } + +} + +const appHistoryAdapter = new AppHistoryAdapter(); +export default appHistoryAdapter; diff --git a/src/server/services/app-monitoring-history-query.service.ts b/src/server/services/app-monitoring-history-query.service.ts deleted file mode 100644 index 05ae8a81..00000000 --- a/src/server/services/app-monitoring-history-query.service.ts +++ /dev/null @@ -1,90 +0,0 @@ -import appHistoryService from '@/server/services/standalone-services/app-history.service'; -import { AppHistoryPoint, AppMonitoringHistory, AppVolumeHistory } from '@/shared/model/app-monitoring-history.model'; - -class AppMonitoringHistoryQueryService { - async getHistory(appIds: string[]): Promise { - const retention = await appHistoryService.getRetentionHours(); - const [apps, volumes] = await appHistoryService.getHistory(appIds, new Date(Date.now() - retention * 3600_000)); - const appsByTime = new Map(); - const volumesByTime = new Map(); - for (const item of apps) { - const time = item.sampledAt.getTime(); - const group = appsByTime.get(time) ?? []; - group.push(item); - appsByTime.set(time, group); - } - for (const item of volumes) { - const time = item.sampledAt.getTime(); - const group = volumesByTime.get(time) ?? []; - group.push(item); - volumesByTime.set(time, group); - } - const times = new Set([...appsByTime.keys(), ...volumesByTime.keys()]); - const points: AppHistoryPoint[] = []; - let latestCpuMeasurement: number | null = null; - let latestRamMeasurement: number | null = null; - let latestVolumeMeasurement: number | null = null; - const volumeSeries = new Map(); - for (const time of [...times].sort((a, b) => a - b)) { - if (points.length && time - points[points.length - 1].time > 7 * 60_000) { - points.push({ time: points[points.length - 1].time + 5 * 60_000, isGap: true, cpuCores: null, ramBytes: null, storageBytes: null, availableVolumes: 0, unavailableVolumes: 0 }); - } - const currentApps = appsByTime.get(time) ?? []; - const currentVolumes = volumesByTime.get(time) ?? []; - const uniqueVolumes = new Map(); - for (const item of currentVolumes) { - const key = `${item.volume.app.projectId}/${item.volume.sharedVolumeId ?? item.volumeId}`; - if (!uniqueVolumes.has(key) || item.status === 'available') uniqueVolumes.set(key, item); - let series = volumeSeries.get(item.volumeId); - if (!series) { - series = { id: item.volumeId, label: item.volume.containerMountPath, points: [] }; - volumeSeries.set(item.volumeId, series); - } - if (series.points.length && time - series.points[series.points.length - 1].time > 7 * 60_000) { - series.points.push({ time: series.points[series.points.length - 1].time + 5 * 60_000, usedBytes: null, measuredAt: null }); - } - series.points.push({ time, usedBytes: item.status === 'available' ? item.usedBytes : null, measuredAt: item.measuredAt?.getTime() ?? null }); - } - if (currentApps.length && currentApps.every(item => item.cpuCores !== null && item.measuredAt)) { - latestCpuMeasurement = Math.max(latestCpuMeasurement ?? 0, Math.min(...currentApps.map(item => item.measuredAt!.getTime()))); - } - if (currentApps.length && currentApps.every(item => item.ramBytes !== null && item.measuredAt)) { - latestRamMeasurement = Math.max(latestRamMeasurement ?? 0, Math.min(...currentApps.map(item => item.measuredAt!.getTime()))); - } - const available = [...uniqueVolumes.values()].filter(item => item.status === 'available' && item.usedBytes !== null); - if (available.length && available.every(item => item.measuredAt)) { - latestVolumeMeasurement = Math.max(latestVolumeMeasurement ?? 0, Math.min(...available.map(item => item.measuredAt!.getTime()))); - } - points.push({ - time, - cpuCores: currentApps.length && currentApps.every(item => item.cpuCores !== null) ? currentApps.reduce((sum, item) => sum + item.cpuCores!, 0) : null, - ramBytes: currentApps.length && currentApps.every(item => item.ramBytes !== null) ? currentApps.reduce((sum, item) => sum + item.ramBytes!, 0) : null, - storageBytes: available.length ? available.reduce((sum, item) => sum + item.usedBytes!, 0) : null, - availableVolumes: available.length, - unavailableVolumes: uniqueVolumes.size - available.length, - }); - } - const now = Date.now(); - const lastPoint = points.at(-1); - if (lastPoint && now - lastPoint.time > 7 * 60_000) { - points.push({ - time: now, - isGap: true, - cpuCores: null, - ramBytes: null, - storageBytes: null, - availableVolumes: 0, - unavailableVolumes: 0, - }); - } - for (const volume of volumeSeries.values()) { - const lastSample = volume.points.at(-1); - if (lastSample && now - lastSample.time > 7 * 60_000) { - volume.points.push({ time: now, usedBytes: null, measuredAt: null }); - } - } - return { retentionHours: retention, points, volumes: [...volumeSeries.values()], latestCpuMeasurement, latestRamMeasurement, latestVolumeMeasurement }; - } -} -const appMonitoringHistoryQueryService = new AppMonitoringHistoryQueryService(); -export default appMonitoringHistoryQueryService; diff --git a/src/server/services/monitoring.service.ts b/src/server/services/monitoring.service.ts index ada3e542..16ecf526 100644 --- a/src/server/services/monitoring.service.ts +++ b/src/server/services/monitoring.service.ts @@ -9,7 +9,7 @@ import { VolumeFilesystemUsage } from "@/shared/model/app-volume-monitoring-usag import dataAccess from "../adapter/db.client"; import appVolumeFilesystemService from "./standalone-services/app-volume-filesystem.service"; import pvcService from "./pvc.service"; -import { KubeObjectNameUtils } from "../utils/kube-object-name.utils"; +import { AppVolumeIdentityUtils } from "@/shared/utils/app-volume-identity.utils"; import projectService from "./project.service"; import { AppMonitoringUsageModel } from "@/shared/model/app-monitoring-usage.model"; @@ -25,9 +25,8 @@ class MonitorService { () => pvcService.getAllPvc(), ); return appVolumes.map(volume => { - const pvcName = KubeObjectNameUtils.toPvcName(volume.sharedVolumeId || volume.id); return { - ...usages.get(`${volume.app.projectId}/${pvcName}`)!, + ...usages.get(volume.id)!, projectId: volume.app.projectId, projectName: volume.app.project.name, appName: volume.app.name, @@ -156,7 +155,10 @@ class MonitorService { select: { id: true, sharedVolumeId: true }, }); const usages = await appVolumeFilesystemService.getVolumeFilesystemUsage(volumes.map(volume => ({ ...volume, projectId })), () => pvcService.getAllPvc()); - return Array.from(usages.values()); + const uniqueUsages = new Map( + Array.from(usages.values(), usage => [AppVolumeIdentityUtils.observationKey(projectId, usage.pvcName), usage]), + ); + return Array.from(uniqueUsages.values()); } } diff --git a/src/server/services/standalone-services/app-history.service.ts b/src/server/services/standalone-services/app-history.service.ts index fb5373bf..dee94b65 100644 --- a/src/server/services/standalone-services/app-history.service.ts +++ b/src/server/services/standalone-services/app-history.service.ts @@ -1,6 +1,3 @@ -import { Prisma } from '@prisma/client'; -import dataAccess from '@/server/adapter/db.client'; -import { Constants } from '@/shared/utils/constants'; import { quantityToScalar } from '@kubernetes/client-node/dist/util.js'; import metricsAdapter from '@/server/adapter/app-history-metrics.adapter'; @@ -8,52 +5,6 @@ type App = { id: string; projectId: string }; type Measurement = { cpuCores: number | null; ramBytes: number | null; measuredAt: Date | null }; class AppHistoryService { - async getPopulation() { - const [apps, volumes] = await Promise.all([ - dataAccess.client.app.findMany({ select: { id: true, projectId: true } }), - dataAccess.client.appVolume.findMany({ include: { app: { select: { projectId: true } } } }), - ]); - return { apps, volumes }; - } - - async getRetentionHours() { - const value = (await dataAccess.client.parameter.findUnique({ where: { name: Constants.APP_MONITORING_HISTORY_RETENTION_HOURS } }))?.value; - const hours = Number(value); - return Number.isInteger(hours) && hours >= 1 && hours <= 168 ? hours : 24; - } - - async setRetentionHours(hours: number) { - await dataAccess.client.parameter.upsert({ - where: { name: Constants.APP_MONITORING_HISTORY_RETENTION_HOURS }, - create: { name: Constants.APP_MONITORING_HISTORY_RETENTION_HOURS, value: String(hours) }, - update: { value: String(hours) }, - }); - } - - async save(apps: Prisma.AppMonitoringSampleCreateManyInput[], volumes: Prisma.AppVolumeMonitoringSampleCreateManyInput[], memberships: Prisma.AppVolumeMonitoringMembershipCreateManyInput[], cutoff: Date) { - await dataAccess.client.$transaction(async tx => { - for (let i = 0; i < apps.length; i += 100) await tx.appMonitoringSample.createMany({ data: apps.slice(i, i + 100) }); - for (let i = 0; i < volumes.length; i += 100) await tx.appVolumeMonitoringSample.createMany({ data: volumes.slice(i, i + 100) }); - for (let i = 0; i < memberships.length; i += 100) await tx.appVolumeMonitoringMembership.createMany({ data: memberships.slice(i, i + 100) }); - await tx.appMonitoringSample.deleteMany({ where: { sampledAt: { lt: cutoff } } }); - await tx.appVolumeMonitoringSample.deleteMany({ where: { sampledAt: { lt: cutoff } } }); - }); - } - - async getHistory(appIds: string[], cutoff: Date) { - return Promise.all([ - dataAccess.client.appMonitoringSample.findMany({ where: { appId: { in: appIds }, sampledAt: { gte: cutoff } }, orderBy: { sampledAt: 'asc' } }), - dataAccess.client.appVolumeMonitoringMembership.findMany({ - where: { volume: { appId: { in: appIds } }, sample: { sampledAt: { gte: cutoff } } }, - include: { - sample: true, - volume: { select: { appId: true, sharedVolumeId: true, containerMountPath: true, app: { select: { projectId: true } } } }, - }, - orderBy: { sample: { sampledAt: 'asc' } }, - }).then(memberships => memberships.map(item => ({ ...item.sample, volumeId: item.volumeId, volume: item.volume }))), - ]); - } - async getAppMetrics(apps: App[]): Promise> { const [pods, metrics] = await Promise.all([metricsAdapter.getPods(), metricsAdapter.getMetrics()]); const metricsByPod = new Map(metrics.map(metric => [`${metric.metadata.namespace}/${metric.metadata.name}`, metric])); diff --git a/src/server/services/app-monitoring-history-query.service.unit.spec.ts b/src/server/services/standalone-services/app-monitoring-history-query.service.unit.spec.ts similarity index 92% rename from src/server/services/app-monitoring-history-query.service.unit.spec.ts rename to src/server/services/standalone-services/app-monitoring-history-query.service.unit.spec.ts index f41d170a..293d4a4d 100644 --- a/src/server/services/app-monitoring-history-query.service.unit.spec.ts +++ b/src/server/services/standalone-services/app-monitoring-history-query.service.unit.spec.ts @@ -1,8 +1,8 @@ -vi.mock('@/server/services/standalone-services/app-history.service', () => ({ +vi.mock('@/server/adapter/app-history.adapter', () => ({ default: { getRetentionHours: vi.fn().mockResolvedValue(24), getHistory: vi.fn() }, })); -import adapter from '@/server/services/standalone-services/app-history.service'; -import historyService from './app-monitoring-history-query.service'; +import adapter from '@/server/adapter/app-history.adapter'; +import historyService from '@/server/services/standalone-services/app-monitoring-history.service'; it('shows incomplete App totals as gaps and counts shared volumes once', async () => { const now = new Date(); diff --git a/src/server/services/standalone-services/app-monitoring-history.service.ts b/src/server/services/standalone-services/app-monitoring-history.service.ts index 64235152..690bf991 100644 --- a/src/server/services/standalone-services/app-monitoring-history.service.ts +++ b/src/server/services/standalone-services/app-monitoring-history.service.ts @@ -1,28 +1,126 @@ import { randomUUID } from 'crypto'; import { Prisma } from '@prisma/client'; -import appHistoryService from '@/server/services/standalone-services/app-history.service'; -import { KubeObjectNameUtils } from '@/server/utils/kube-object-name.utils'; +import appHistoryService from './app-history.service'; +import appHistoryAdapter from '@/server/adapter/app-history.adapter'; +import { AppHistoryPoint, AppMonitoringHistory, AppVolumeHistory } from '@/shared/model/app-monitoring-history.model'; +import { AppVolumeIdentityUtils } from '@/shared/utils/app-volume-identity.utils'; import appVolumeFilesystemService from './app-volume-filesystem.service'; import scheduleService from './schedule.service'; type VolumeSample = Prisma.AppVolumeMonitoringSampleCreateManyInput & { id: string }; +const COLLECTION_INTERVAL_MINUTES = 5; +const COLLECTION_INTERVAL_MS = COLLECTION_INTERVAL_MINUTES * 60_000; +const GAP_THRESHOLD_MS = COLLECTION_INTERVAL_MS + 2 * 60_000; + class AppMonitoringHistoryService { private running = false; initialize() { - scheduleService.scheduleJob('app-monitoring-history', '*/5 * * * *', () => { + scheduleService.scheduleJob('app-monitoring-history', `*/${COLLECTION_INTERVAL_MINUTES} * * * *`, () => { void this.collect(); }); void this.collect(); } + async getHistory(appIds: string[]): Promise { + const retention = await appHistoryAdapter.getRetentionHours(); + const [apps, volumes] = await appHistoryAdapter.getHistory(appIds, new Date(Date.now() - retention * 3600_000)); + const appsByTime = new Map(); + const volumesByTime = new Map(); + for (const item of apps) { + const time = item.sampledAt.getTime(); + const group = appsByTime.get(time) ?? []; + group.push(item); + appsByTime.set(time, group); + } + for (const item of volumes) { + const time = item.sampledAt.getTime(); + const group = volumesByTime.get(time) ?? []; + group.push(item); + volumesByTime.set(time, group); + } + const times = new Set([...appsByTime.keys(), ...volumesByTime.keys()]); + const points: AppHistoryPoint[] = []; + let latestCpuMeasurement: number | null = null; + let latestRamMeasurement: number | null = null; + let latestVolumeMeasurement: number | null = null; + const volumeSeries = new Map(); + for (const time of [...times].sort((a, b) => a - b)) { + if (points.length && time - points[points.length - 1].time > GAP_THRESHOLD_MS) { + points.push({ time: points[points.length - 1].time + COLLECTION_INTERVAL_MS, isGap: true, cpuCores: null, ramBytes: null, storageBytes: null, availableVolumes: 0, unavailableVolumes: 0 }); + } + const currentApps = appsByTime.get(time) ?? []; + const currentVolumes = volumesByTime.get(time) ?? []; + const uniqueVolumes = new Map(); + for (const item of currentVolumes) { + const key = AppVolumeIdentityUtils.key(item.volume.app.projectId, item.volumeId, item.volume.sharedVolumeId); + if (!uniqueVolumes.has(key) || item.status === 'available') uniqueVolumes.set(key, item); + let series = volumeSeries.get(item.volumeId); + if (!series) { + series = { id: item.volumeId, label: item.volume.containerMountPath, points: [] }; + volumeSeries.set(item.volumeId, series); + } + if (series.points.length && time - series.points[series.points.length - 1].time > GAP_THRESHOLD_MS) { + series.points.push({ time: series.points[series.points.length - 1].time + COLLECTION_INTERVAL_MS, usedBytes: null, measuredAt: null }); + } + series.points.push({ time, usedBytes: item.status === 'available' ? item.usedBytes : null, measuredAt: item.measuredAt?.getTime() ?? null }); + } + if (currentApps.length && currentApps.every(item => item.cpuCores !== null && item.measuredAt)) { + latestCpuMeasurement = Math.max(latestCpuMeasurement ?? 0, Math.min(...currentApps.map(item => item.measuredAt!.getTime()))); + } + if (currentApps.length && currentApps.every(item => item.ramBytes !== null && item.measuredAt)) { + latestRamMeasurement = Math.max(latestRamMeasurement ?? 0, Math.min(...currentApps.map(item => item.measuredAt!.getTime()))); + } + const available = [...uniqueVolumes.values()].filter(item => item.status === 'available' && item.usedBytes !== null); + if (available.length && available.every(item => item.measuredAt)) { + latestVolumeMeasurement = Math.max(latestVolumeMeasurement ?? 0, Math.min(...available.map(item => item.measuredAt!.getTime()))); + } + points.push({ + time, + cpuCores: currentApps.length && currentApps.every(item => item.cpuCores !== null) ? currentApps.reduce((sum, item) => sum + item.cpuCores!, 0) : null, + ramBytes: currentApps.length && currentApps.every(item => item.ramBytes !== null) ? currentApps.reduce((sum, item) => sum + item.ramBytes!, 0) : null, + storageBytes: available.length ? available.reduce((sum, item) => sum + item.usedBytes!, 0) : null, + availableVolumes: available.length, + unavailableVolumes: uniqueVolumes.size - available.length, + }); + } + const now = Date.now(); + const lastPoint = points.at(-1); + if (lastPoint && now - lastPoint.time > GAP_THRESHOLD_MS) { + points.push({ + time: now, + isGap: true, + cpuCores: null, + ramBytes: null, + storageBytes: null, + availableVolumes: 0, + unavailableVolumes: 0, + }); + } + for (const volume of volumeSeries.values()) { + const lastSample = volume.points.at(-1); + if (lastSample && now - lastSample.time > GAP_THRESHOLD_MS) { + volume.points.push({ time: now, usedBytes: null, measuredAt: null }); + } + } + return { retentionHours: retention, points, volumes: [...volumeSeries.values()], latestCpuMeasurement, latestRamMeasurement, latestVolumeMeasurement }; + } + + async getRetentionHours() { + return appHistoryAdapter.getRetentionHours(); + } + + async setRetentionHours(hours: number) { + await appHistoryAdapter.setRetentionHours(hours); + } + async collect() { if (this.running) return; this.running = true; try { const sampledAt = new Date(); - const { apps, volumes } = await appHistoryService.getPopulation(); + const { apps, volumes } = await appHistoryAdapter.getPopulation(); const [metrics, filesystem, retention] = await Promise.all([ appHistoryService.getAppMetrics(apps).catch(error => { console.error('App metrics collection failed', error); @@ -34,16 +132,16 @@ class AppMonitoringHistoryService { console.error('Volume metrics collection failed', error); return new Map(); }), - appHistoryService.getRetentionHours(), + appHistoryAdapter.getRetentionHours(), ]); const samples = new Map(); const memberships: Prisma.AppVolumeMonitoringMembershipCreateManyInput[] = []; for (const volume of volumes) { - const underlyingId = volume.sharedVolumeId || volume.id; - let sample = samples.get(underlyingId); + const underlyingId = AppVolumeIdentityUtils.underlyingId(volume.id, volume.sharedVolumeId); + const identity = AppVolumeIdentityUtils.key(volume.app.projectId, volume.id, volume.sharedVolumeId); + let sample = samples.get(identity); if (!sample) { - const pvcName = KubeObjectNameUtils.toPvcName(underlyingId); - const usage = filesystem.get(`${volume.app.projectId}/${pvcName}`); + const usage = filesystem.get(volume.id); sample = { id: randomUUID(), volumeId: underlyingId, @@ -54,11 +152,11 @@ class AppMonitoringHistoryService { capacityBytes: usage?.capacityBytes ?? null, measuredAt: usage?.measuredAt ? new Date(usage.measuredAt) : null, }; - samples.set(underlyingId, sample); + samples.set(identity, sample); } memberships.push({ sampleId: sample.id, volumeId: volume.id }); } - await appHistoryService.save( + await appHistoryAdapter.save( apps.map(app => ({ appId: app.id, sampledAt, diff --git a/src/server/services/standalone-services/app-monitoring-history.service.unit.spec.ts b/src/server/services/standalone-services/app-monitoring-history.service.unit.spec.ts index 108b13d0..f6ad3e00 100644 --- a/src/server/services/standalone-services/app-monitoring-history.service.unit.spec.ts +++ b/src/server/services/standalone-services/app-monitoring-history.service.unit.spec.ts @@ -1,8 +1,9 @@ -vi.mock('./app-history.service', () => ({ default: { getPopulation: vi.fn(), getRetentionHours: vi.fn(), save: vi.fn(), getAppMetrics: vi.fn() } })); +vi.mock('@/server/adapter/app-history.adapter', () => ({ default: { getPopulation: vi.fn(), getRetentionHours: vi.fn(), save: vi.fn() } })); +vi.mock('./app-history.service', () => ({ default: { getAppMetrics: vi.fn() } })); vi.mock('./app-volume-filesystem.service', () => ({ default: { getVolumeFilesystemUsage: vi.fn() } })); vi.mock('./schedule.service', () => ({ default: { scheduleJob: vi.fn() } })); import history from './app-monitoring-history.service'; -import adapter from '@/server/services/standalone-services/app-history.service'; +import adapter from '@/server/adapter/app-history.adapter'; import metrics from './app-history.service'; import filesystem from './app-volume-filesystem.service'; @@ -10,7 +11,7 @@ it('records a gap for failed App metrics while preserving successful volume data vi.mocked(adapter.getPopulation).mockResolvedValue({ apps: [{ id: 'app', projectId: 'p' }], volumes: [{ id: 'v', appId: 'app', sharedVolumeId: null, app: { projectId: 'p' } }] } as never); vi.mocked(adapter.getRetentionHours).mockResolvedValue(24); vi.mocked(metrics.getAppMetrics).mockRejectedValue(new Error('metrics unavailable')); - vi.mocked(filesystem.getVolumeFilesystemUsage).mockResolvedValue(new Map([['p/pvc-v', { pvcName: 'pvc-v', status: 'available', usedBytes: 4 }]])); + vi.mocked(filesystem.getVolumeFilesystemUsage).mockResolvedValue(new Map([['v', { pvcName: 'pvc-v', status: 'available', usedBytes: 4 }]])); await history.collect(); expect(adapter.save).toHaveBeenCalledWith( [expect.objectContaining({ appId: 'app', cpuCores: null, ramBytes: null })], @@ -43,7 +44,7 @@ it('stores one filesystem measurement for a shared Volume with a link for each m ] } as never); vi.mocked(adapter.getRetentionHours).mockResolvedValue(24); vi.mocked(metrics.getAppMetrics).mockResolvedValue(new Map()); - vi.mocked(filesystem.getVolumeFilesystemUsage).mockResolvedValue(new Map([['p/pvc-v', { pvcName: 'pvc-v', status: 'available', usedBytes: 4 }]])); + vi.mocked(filesystem.getVolumeFilesystemUsage).mockResolvedValue(new Map([['v', { pvcName: 'pvc-v', status: 'available', usedBytes: 4 }]])); await history.collect(); const [, samples, memberships] = vi.mocked(adapter.save).mock.calls[0]; expect(samples).toHaveLength(1); diff --git a/src/server/services/standalone-services/app-volume-filesystem.service.ts b/src/server/services/standalone-services/app-volume-filesystem.service.ts index 9018d357..eddc59e8 100644 --- a/src/server/services/standalone-services/app-volume-filesystem.service.ts +++ b/src/server/services/standalone-services/app-volume-filesystem.service.ts @@ -1,6 +1,6 @@ import { V1PersistentVolumeClaim } from '@kubernetes/client-node'; import volumeSummaryAdapter from '@/server/adapter/volume-summary.adapter'; -import { supportsVolumeFilesystemMetrics } from '../volume-filesystem-drivers'; +import { AppVolumeIdentityUtils } from '@/shared/utils/app-volume-identity.utils'; import { VolumeFilesystemUsage } from '@/shared/model/app-volume-monitoring-usage.model'; import { KubeObjectNameUtils } from '@/server/utils/kube-object-name.utils'; @@ -17,19 +17,26 @@ class AppVolumeFilesystemService { const pvcByKey = new Map(pvcs.map(pvc => [`${pvc.metadata?.namespace}/${pvc.metadata?.name}`, pvc])); const pvByName = new Map(persistentVolumes.map(pv => [pv.metadata?.name, pv])); const result = new Map(); + const observations = new Map(); const now = Date.now(); for (const volume of volumes) { - const pvcName = KubeObjectNameUtils.toPvcName(volume.sharedVolumeId || volume.id); - const key = `${volume.projectId}/${pvcName}`; - if (result.has(key)) continue; + const pvcName = KubeObjectNameUtils.toPvcName(AppVolumeIdentityUtils.underlyingId(volume.id, volume.sharedVolumeId)); + const key = AppVolumeIdentityUtils.observationKey(volume.projectId, pvcName); + const existing = observations.get(key); + if (existing) { + result.set(volume.id, existing); + continue; + } const pvc = pvcByKey.get(key); const pv = pvc?.spec?.volumeName ? pvByName.get(pvc.spec.volumeName) : undefined; if (!pv) { - result.set(key, { pvcName, status: 'unavailable' }); + observations.set(key, { pvcName, status: 'unavailable' }); + result.set(volume.id, observations.get(key)!); continue; } - if (!supportsVolumeFilesystemMetrics(pv)) { - result.set(key, { pvcName, status: 'unsupported' }); + if (pv.spec?.csi?.driver !== 'driver.longhorn.io') { + observations.set(key, { pvcName, status: 'unsupported' }); + result.set(volume.id, observations.get(key)!); continue; } const candidates = samples @@ -43,10 +50,11 @@ class AppVolumeFilesystemService { const sample = candidates[0]; const age = sample ? now - Date.parse(sample.time!) : Infinity; if (!sample || age > 5 * 60 * 1000 || age < -60 * 1000) { - result.set(key, { pvcName, status: sample ? 'stale' : 'unavailable' }); + observations.set(key, { pvcName, status: sample ? 'stale' : 'unavailable' }); + result.set(volume.id, observations.get(key)!); continue; } - result.set(key, { + observations.set(key, { pvcName, status: 'available', usedBytes: sample.usedBytes, @@ -54,6 +62,7 @@ class AppVolumeFilesystemService { capacityBytes: sample.capacityBytes, measuredAt: sample.time, }); + result.set(volume.id, observations.get(key)!); } return result; } diff --git a/src/server/services/standalone-services/app-volume-filesystem.service.unit.spec.ts b/src/server/services/standalone-services/app-volume-filesystem.service.unit.spec.ts new file mode 100644 index 00000000..4c4467c8 --- /dev/null +++ b/src/server/services/standalone-services/app-volume-filesystem.service.unit.spec.ts @@ -0,0 +1,32 @@ +vi.mock('@/server/adapter/volume-summary.adapter', () => ({ + default: { getPersistentVolumeClaims: vi.fn(), getPersistentVolumes: vi.fn(), getSamples: vi.fn() }, +})); + +import adapter from '@/server/adapter/volume-summary.adapter'; +import filesystem from './app-volume-filesystem.service'; + +it('associates shared mounts with their project observation and preserves missing and stale status', async () => { + const now = Date.now(); + vi.mocked(adapter.getPersistentVolumeClaims).mockResolvedValue([ + { metadata: { namespace: 'one', name: 'pvc-base' }, spec: { volumeName: 'pv-one' } }, + { metadata: { namespace: 'two', name: 'pvc-base' }, spec: { volumeName: 'pv-two' } }, + ] as never); + vi.mocked(adapter.getPersistentVolumes).mockResolvedValue([ + { metadata: { name: 'pv-one' }, spec: { csi: { driver: 'driver.longhorn.io' } } }, + { metadata: { name: 'pv-two' }, spec: { csi: { driver: 'driver.longhorn.io' } } }, + ] as never); + vi.mocked(adapter.getSamples).mockResolvedValue([ + { pvcRef: { namespace: 'one', name: 'pvc-base' }, time: new Date(now).toISOString(), usedBytes: 4, availableBytes: 6, capacityBytes: 10 }, + { pvcRef: { namespace: 'two', name: 'pvc-base' }, time: new Date(now - 600_000).toISOString(), usedBytes: 8, availableBytes: 2, capacityBytes: 10 }, + ]); + const result = await filesystem.getVolumeFilesystemUsage([ + { id: 'base', sharedVolumeId: null, projectId: 'one' }, + { id: 'shared', sharedVolumeId: 'base', projectId: 'one' }, + { id: 'other', sharedVolumeId: 'base', projectId: 'two' }, + { id: 'missing', sharedVolumeId: null, projectId: 'one' }, + ]); + expect(result.get('base')).toMatchObject({ status: 'available', usedBytes: 4 }); + expect(result.get('shared')).toEqual(result.get('base')); + expect(result.get('other')).toMatchObject({ status: 'stale' }); + expect(result.get('missing')).toMatchObject({ status: 'unavailable' }); +}); diff --git a/src/server/services/volume-filesystem-drivers.ts b/src/server/services/volume-filesystem-drivers.ts deleted file mode 100644 index ab290fb9..00000000 --- a/src/server/services/volume-filesystem-drivers.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { V1PersistentVolume } from '@kubernetes/client-node'; - -interface VolumeFilesystemDriver { - supports(volume: V1PersistentVolume): boolean; -} - -const drivers: VolumeFilesystemDriver[] = [ - { supports: volume => volume.spec?.csi?.driver === 'driver.longhorn.io' }, -]; - -export function supportsVolumeFilesystemMetrics(volume: V1PersistentVolume): boolean { - return drivers.some(driver => driver.supports(volume)); -} diff --git a/src/shared/utils/app-volume-identity.utils.ts b/src/shared/utils/app-volume-identity.utils.ts new file mode 100644 index 00000000..d3011fcf --- /dev/null +++ b/src/shared/utils/app-volume-identity.utils.ts @@ -0,0 +1,13 @@ +export class AppVolumeIdentityUtils { + static underlyingId(id: string, sharedVolumeId: string | null): string { + return sharedVolumeId || id; + } + + static key(projectId: string, id: string, sharedVolumeId: string | null): string { + return `${projectId}/${this.underlyingId(id, sharedVolumeId)}`; + } + + static observationKey(projectId: string, pvcName: string): string { + return `${projectId}/${pvcName}`; + } +} From 4ae8b694c242fcecb885ba109734ed716360ba93 Mon Sep 17 00:00:00 2001 From: biersoeckli Date: Sun, 8 Nov 2026 15:06:33 +0000 Subject: [PATCH 09/11] feat: implement caching for node summary retrieval and enhance app volume filesystem service --- src/server/adapter/volume-summary.adapter.ts | 4 +-- src/server/services/cluster.service.ts | 3 ++- .../services/cluster.service.unit.spec.ts | 2 ++ src/server/services/monitoring.service.ts | 4 ++- .../services/monitoring.service.unit.spec.ts | 2 ++ .../services/node-summary-cache.service.ts | 18 +++++++++++++ .../node-summary-cache.service.unit.spec.ts | 19 ++++++++++++++ .../app-volume-filesystem.service.ts | 25 +++++++++++-------- 8 files changed, 63 insertions(+), 14 deletions(-) create mode 100644 src/server/services/node-summary-cache.service.ts create mode 100644 src/server/services/node-summary-cache.service.unit.spec.ts diff --git a/src/server/adapter/volume-summary.adapter.ts b/src/server/adapter/volume-summary.adapter.ts index e1227b5f..43e89624 100644 --- a/src/server/adapter/volume-summary.adapter.ts +++ b/src/server/adapter/volume-summary.adapter.ts @@ -21,11 +21,11 @@ class VolumeSummaryAdapter { return (await k3s.core.listPersistentVolume()).items; } - async getSamples(): Promise { + async getSamples(getNodeSummary: (nodeName: string) => Promise = nodeName => k3s.getNodeSummary(nodeName)): Promise { const nodes = await k3s.core.listNode(); const summaries = await Promise.allSettled(nodes.items.map(async node => { if (!node.metadata?.name) return []; - const summary = await k3s.getNodeSummary(node.metadata.name) as NodeSummary; + const summary = await getNodeSummary(node.metadata.name) as NodeSummary; return summary.pods?.flatMap(pod => pod.volume ?? []) ?? []; })); return summaries.flatMap(result => result.status === 'fulfilled' ? result.value : []); diff --git a/src/server/services/cluster.service.ts b/src/server/services/cluster.service.ts index 4d5f063a..5a6ae8dc 100644 --- a/src/server/services/cluster.service.ts +++ b/src/server/services/cluster.service.ts @@ -1,3 +1,4 @@ +import { getCachedNodeSummary } from './node-summary-cache.service'; import volumeSummaryAdapter from "../adapter/volume-summary.adapter"; import k3s, { kubernetesPatchOptions } from "../adapter/kubernetes-api.adapter"; import * as k8s from '@kubernetes/client-node'; @@ -155,7 +156,7 @@ class ClusterService { let filesystem: { usedBytes?: number; availableBytes?: number; capacityBytes?: number; time?: string } | undefined; try { - const summary = await k3s.getNodeSummary(node.Node.metadata?.name!) as { + const summary = await getCachedNodeSummary(node.Node.metadata?.name!) as { node?: { fs?: typeof filesystem }; }; const fs = summary.node?.fs; diff --git a/src/server/services/cluster.service.unit.spec.ts b/src/server/services/cluster.service.unit.spec.ts index ea88810d..70d23cc2 100644 --- a/src/server/services/cluster.service.unit.spec.ts +++ b/src/server/services/cluster.service.unit.spec.ts @@ -1,3 +1,5 @@ +vi.mock('next/cache', () => ({ unstable_cache: (fn: unknown) => fn, revalidateTag: vi.fn() })); + const mocks = vi.hoisted(() => ({ topNodes: vi.fn(), getNodeMetrics: vi.fn(), getNodeSummary: vi.fn(), getPersistentVolumeClaims: vi.fn() })); diff --git a/src/server/services/monitoring.service.ts b/src/server/services/monitoring.service.ts index 16ecf526..cf907bd8 100644 --- a/src/server/services/monitoring.service.ts +++ b/src/server/services/monitoring.service.ts @@ -1,3 +1,4 @@ +import { getCachedNodeSummary } from './node-summary-cache.service'; import k3s from "../adapter/kubernetes-api.adapter"; import * as k8s from '@kubernetes/client-node'; import standalonePodService from "./standalone-services/standalone-pod.service"; @@ -23,6 +24,7 @@ class MonitorService { const usages = await appVolumeFilesystemService.getVolumeFilesystemUsage( appVolumes.map(volume => ({ ...volume, projectId: volume.app.projectId })), () => pvcService.getAllPvc(), + getCachedNodeSummary, ); return appVolumes.map(volume => { return { @@ -154,7 +156,7 @@ class MonitorService { where: { appId }, select: { id: true, sharedVolumeId: true }, }); - const usages = await appVolumeFilesystemService.getVolumeFilesystemUsage(volumes.map(volume => ({ ...volume, projectId })), () => pvcService.getAllPvc()); + const usages = await appVolumeFilesystemService.getVolumeFilesystemUsage(volumes.map(volume => ({ ...volume, projectId })), () => pvcService.getAllPvc(), getCachedNodeSummary); const uniqueUsages = new Map( Array.from(usages.values(), usage => [AppVolumeIdentityUtils.observationKey(projectId, usage.pvcName), usage]), ); diff --git a/src/server/services/monitoring.service.unit.spec.ts b/src/server/services/monitoring.service.unit.spec.ts index b625e9eb..ed68363c 100644 --- a/src/server/services/monitoring.service.unit.spec.ts +++ b/src/server/services/monitoring.service.unit.spec.ts @@ -1,3 +1,5 @@ +vi.mock('next/cache', () => ({ unstable_cache: (fn: unknown) => fn, revalidateTag: vi.fn() })); + import * as k8s from '@kubernetes/client-node'; const k8sMocks = vi.hoisted(() => ({ diff --git a/src/server/services/node-summary-cache.service.ts b/src/server/services/node-summary-cache.service.ts new file mode 100644 index 00000000..a6562ee4 --- /dev/null +++ b/src/server/services/node-summary-cache.service.ts @@ -0,0 +1,18 @@ +import { unstable_cache } from 'next/cache'; +import k3s from '../adapter/kubernetes-api.adapter'; + +const cachedSummary = unstable_cache( + (nodeName: string) => k3s.getNodeSummary(nodeName), + ['node-summary'], + { revalidate: 20 }, +); +const inFlight = new Map>(); + +export function getCachedNodeSummary(nodeName: string): Promise { + const existing = inFlight.get(nodeName); + if (existing) return existing; + + const request = cachedSummary(nodeName).finally(() => inFlight.delete(nodeName)); + inFlight.set(nodeName, request); + return request; +} diff --git a/src/server/services/node-summary-cache.service.unit.spec.ts b/src/server/services/node-summary-cache.service.unit.spec.ts new file mode 100644 index 00000000..d659403e --- /dev/null +++ b/src/server/services/node-summary-cache.service.unit.spec.ts @@ -0,0 +1,19 @@ +vi.mock('next/cache', () => ({ unstable_cache: (fn: unknown) => fn })); +vi.mock('../adapter/kubernetes-api.adapter', () => ({ default: { getNodeSummary: vi.fn() } })); + +import k3s from '../adapter/kubernetes-api.adapter'; +import { getCachedNodeSummary } from './node-summary-cache.service'; + +it('deduplicates concurrent requests per node and retries failures', async () => { + let rejectRequest!: (error: Error) => void; + vi.mocked(k3s.getNodeSummary).mockImplementationOnce(() => new Promise((_, reject) => { rejectRequest = reject; })); + const first = getCachedNodeSummary('node-a'); + const second = getCachedNodeSummary('node-a'); + expect(first).toBe(second); + expect(k3s.getNodeSummary).toHaveBeenCalledTimes(1); + rejectRequest(new Error('failed')); + await expect(first).rejects.toThrow('failed'); + vi.mocked(k3s.getNodeSummary).mockResolvedValueOnce({ node: {} }); + await expect(getCachedNodeSummary('node-a')).resolves.toEqual({ node: {} }); + expect(k3s.getNodeSummary).toHaveBeenCalledTimes(2); +}); diff --git a/src/server/services/standalone-services/app-volume-filesystem.service.ts b/src/server/services/standalone-services/app-volume-filesystem.service.ts index eddc59e8..0e3d5f30 100644 --- a/src/server/services/standalone-services/app-volume-filesystem.service.ts +++ b/src/server/services/standalone-services/app-volume-filesystem.service.ts @@ -8,17 +8,30 @@ class AppVolumeFilesystemService { async getVolumeFilesystemUsage( volumes: { id: string; sharedVolumeId: string | null; projectId: string }[], getClaims: () => Promise = () => volumeSummaryAdapter.getPersistentVolumeClaims(), + getNodeSummary?: (nodeName: string) => Promise, ): Promise> { const [pvcs, persistentVolumes, samples] = await Promise.all([ getClaims().catch(() => []), volumeSummaryAdapter.getPersistentVolumes().catch(() => []), - volumeSummaryAdapter.getSamples().catch(() => []), + volumeSummaryAdapter.getSamples(getNodeSummary).catch(() => []), ]); const pvcByKey = new Map(pvcs.map(pvc => [`${pvc.metadata?.namespace}/${pvc.metadata?.name}`, pvc])); const pvByName = new Map(persistentVolumes.map(pv => [pv.metadata?.name, pv])); const result = new Map(); const observations = new Map(); const now = Date.now(); + const latestSampleByKey = new Map(); + for (const sample of samples) { + const namespace = sample.pvcRef?.namespace; + const name = sample.pvcRef?.name; + const timestamp = Date.parse(sample.time ?? ''); + const values = [sample.usedBytes, sample.availableBytes, sample.capacityBytes]; + if (!namespace || !name || !Number.isFinite(timestamp) || !sample.capacityBytes || + !values.every(value => typeof value === 'number' && Number.isFinite(value) && value >= 0)) continue; + const key = AppVolumeIdentityUtils.observationKey(namespace, name); + const previous = latestSampleByKey.get(key); + if (!previous || timestamp > Date.parse(previous.time!)) latestSampleByKey.set(key, sample); + } for (const volume of volumes) { const pvcName = KubeObjectNameUtils.toPvcName(AppVolumeIdentityUtils.underlyingId(volume.id, volume.sharedVolumeId)); const key = AppVolumeIdentityUtils.observationKey(volume.projectId, pvcName); @@ -39,15 +52,7 @@ class AppVolumeFilesystemService { result.set(volume.id, observations.get(key)!); continue; } - const candidates = samples - .filter(sample => sample.pvcRef?.namespace === volume.projectId && sample.pvcRef.name === pvcName) - .filter(sample => { - const values = [sample.usedBytes, sample.availableBytes, sample.capacityBytes]; - return values.every(value => typeof value === 'number' && Number.isFinite(value) && value >= 0) - && !!sample.capacityBytes && Number.isFinite(Date.parse(sample.time ?? '')); - }) - .sort((a, b) => Date.parse(b.time!) - Date.parse(a.time!)); - const sample = candidates[0]; + const sample = latestSampleByKey.get(key); const age = sample ? now - Date.parse(sample.time!) : Infinity; if (!sample || age > 5 * 60 * 1000 || age < -60 * 1000) { observations.set(key, { pvcName, status: sample ? 'stale' : 'unavailable' }); From 919832d08e5a8042dbdec5a2832e0a867f9b1917 Mon Sep 17 00:00:00 2001 From: biersoeckli Date: Sun, 8 Nov 2026 15:15:39 +0000 Subject: [PATCH 10/11] feat: add mock for kubernetes-api.adapter in app monitoring history query tests --- .../app-monitoring-history-query.service.unit.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server/services/standalone-services/app-monitoring-history-query.service.unit.spec.ts b/src/server/services/standalone-services/app-monitoring-history-query.service.unit.spec.ts index 293d4a4d..0ea61ced 100644 --- a/src/server/services/standalone-services/app-monitoring-history-query.service.unit.spec.ts +++ b/src/server/services/standalone-services/app-monitoring-history-query.service.unit.spec.ts @@ -1,6 +1,7 @@ vi.mock('@/server/adapter/app-history.adapter', () => ({ default: { getRetentionHours: vi.fn().mockResolvedValue(24), getHistory: vi.fn() }, })); +vi.mock('@/server/adapter/kubernetes-api.adapter', () => ({ default: {} })); import adapter from '@/server/adapter/app-history.adapter'; import historyService from '@/server/services/standalone-services/app-monitoring-history.service'; From f44d94e4a4b9016f6239abc523deb4b4a6cf5789 Mon Sep 17 00:00:00 2001 From: biersoeckli Date: Sun, 8 Nov 2026 15:37:59 +0000 Subject: [PATCH 11/11] feat: enhance runtime class retrieval with unstable caching mechanism and fix tests --- ...app-monitoring-history.integration.spec.ts | 2 ++ .../agent-template.service.unit.spec.ts | 1 + src/server/services/cluster.service.ts | 22 +++---------------- 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts b/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts index 7555564c..f084f948 100644 --- a/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts +++ b/src/__tests__/integration/server/services/app-monitoring-history.integration.spec.ts @@ -1,4 +1,6 @@ // @vitest-environment node +vi.mock('@/server/adapter/kubernetes-api.adapter', () => ({ default: {} })); + import { createPrismaTestContext } from '@/__tests__/prisma-test.utils'; import adapter from '@/server/adapter/app-history.adapter'; import history from '@/server/services/standalone-services/app-monitoring-history.service'; diff --git a/src/server/services/agent-template.service.unit.spec.ts b/src/server/services/agent-template.service.unit.spec.ts index fd67793c..07f2d7dd 100644 --- a/src/server/services/agent-template.service.unit.spec.ts +++ b/src/server/services/agent-template.service.unit.spec.ts @@ -1,5 +1,6 @@ vi.mock("next/cache", () => ({ revalidateTag: vi.fn(), + unstable_cache: (fn: (...args: any[]) => any) => fn, })); const dbProjectMocks = vi.hoisted(() => ({ diff --git a/src/server/services/cluster.service.ts b/src/server/services/cluster.service.ts index 5a6ae8dc..b485465a 100644 --- a/src/server/services/cluster.service.ts +++ b/src/server/services/cluster.service.ts @@ -68,8 +68,8 @@ class ClusterService { } async getRuntimeClasses(): Promise { - - const allowedRuntimes = ['gvisor', 'kata']; + return await unstable_cache(async () => { + const allowedRuntimes = ['gvisor', 'kata']; const runtimeClasses = await k3s.customObjects.listClusterCustomObject({ group: 'node.k8s.io', @@ -83,27 +83,11 @@ class ClusterService { .sort((a, b) => a.localeCompare(b)); return runtimeNames.filter(rn => allowedRuntimes.includes(rn.toLocaleLowerCase())); - /*return await unstable_cache(async () => { - - const allowedRuntimes = ['gVisor', 'kata']; - - const runtimeClasses = await k3s.customObjects.listClusterCustomObject({ - group: 'node.k8s.io', - version: 'v1', - plural: 'runtimeclasses', - }) as { items?: { metadata?: { name?: string } }[] }; - - const runtimeNames = (runtimeClasses.items ?? []) - .map((runtimeClass) => runtimeClass.metadata?.name) - .filter((name): name is string => !!name) - .sort((a, b) => a.localeCompare(b)); - - return runtimeNames.filter(rn => allowedRuntimes.includes(rn)); }, [Tags.runtimeClasses()], { revalidate: 60, tags: [Tags.runtimeClasses()] - })();*/ + })(); } async setNodeStatus(nodeName: string, schedulable: boolean) {